From d3f868efe820d3a1187218537c9ed7f83bc3ed3d Mon Sep 17 00:00:00 2001 From: Zhang ChuanJin Date: Sat, 5 Sep 2026 22:43:20 +0800 Subject: [PATCH] docs: refresh model examples across provider pages --- docs/providers/azure_ai.md | 7 +- docs/providers/bedrock.md | 12 ++- docs/providers/cerebras.md | 10 +- docs/providers/cohere.md | 28 ++--- docs/providers/deepinfra.md | 22 ++-- docs/providers/deepseek.md | 8 +- docs/providers/fireworks_ai.md | 20 ++-- docs/providers/gemini.md | 27 +++-- docs/providers/mistral.md | 6 +- docs/providers/moonshot.md | 20 ++-- docs/providers/nvidia_nim.md | 62 +++-------- docs/providers/sambanova.md | 9 +- docs/providers/togetherai.md | 190 ++------------------------------- docs/providers/vertex.md | 21 ++-- docs/providers/xai.md | 24 ++--- 15 files changed, 131 insertions(+), 335 deletions(-) diff --git a/docs/providers/azure_ai.md b/docs/providers/azure_ai.md index 78a135270..0ace05a02 100644 --- a/docs/providers/azure_ai.md +++ b/docs/providers/azure_ai.md @@ -307,6 +307,7 @@ LiteLLM supports **ALL** azure ai models. Here's a few examples: | Model Name | Function Call | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Claude Opus 5 | `completion(model="azure_ai/claude-opus-5", messages)` | | Cohere command-r-plus | `completion(model="azure_ai/command-r-plus", messages)` | | Cohere command-r | `completion(model="azure_ai/command-r", messages)` | | mistral-large-latest | `completion(model="azure_ai/mistral-large-latest", messages)` | @@ -354,7 +355,7 @@ export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic" ```yaml model_list: - - model_name: claude-4-azure + - model_name: azure-claude litellm_params: model: azure_ai/{{anthropic_large}} api_key: os.environ/AZURE_AI_API_KEY @@ -374,11 +375,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer $LITELLM_KEY' \ --data '{ - "model": "claude-4-azure", + "model": "azure-claude", "messages": [ { "role": "user", - "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?" + "content": "How do I use Claude Opus via Azure Anthropic in LiteLLM?" } ], "max_tokens": 1024 diff --git a/docs/providers/bedrock.md b/docs/providers/bedrock.md index 82463cdc7..6d09c01d5 100644 --- a/docs/providers/bedrock.md +++ b/docs/providers/bedrock.md @@ -56,7 +56,7 @@ response = completion( ```yaml model_list: - - model_name: bedrock-claude-sonnet-4-5 + - model_name: bedrock-claude-sonnet-5 litellm_params: model: bedrock/us.anthropic.{{anthropic}} api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK @@ -93,7 +93,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server ```yaml model_list: - - model_name: bedrock-claude-sonnet-4-5 + - model_name: bedrock-claude-sonnet-5 litellm_params: model: bedrock/us.anthropic.{{anthropic}} aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID @@ -2020,13 +2020,15 @@ LiteLLM supports ALL Bedrock models. Here's an example of using a bedrock model with LiteLLM. For a complete list, refer to the [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) -| Model Name | Command | -|----------------------------|------------------------------------------------------------------| +| Model Name | Command | Required OS Variables | +|------------|---------|-----------------------| +| Anthropic Claude Opus 5 | `completion(model='bedrock/us.anthropic.claude-opus-5', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | +| Amazon Nova 2 Lite | `completion(model='bedrock/amazon.nova-2-lite-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | GPT-OSS 20B | `completion(model='bedrock/converse/openai.gpt-oss-20b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | -| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | +| Anthropic Claude Haiku 4.5 | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | diff --git a/docs/providers/cerebras.md b/docs/providers/cerebras.md index 33bef5e10..571ef63b9 100644 --- a/docs/providers/cerebras.md +++ b/docs/providers/cerebras.md @@ -23,19 +23,18 @@ import os os.environ['CEREBRAS_API_KEY'] = "" response = completion( - model="cerebras/llama3-70b-instruct", + model="cerebras/gpt-oss-120b", messages=[ { "role": "user", "content": "What's the weather like in Boston today in Fahrenheit? (Write in JSON)", } ], - max_tokens=10, + max_tokens=4096, # The prompt should include JSON if 'json_object' is selected; otherwise, you will get error code 400. response_format={ "type": "json_object" }, seed=123, - stop=["\n\n"], temperature=0.2, top_p=0.9, tool_choice="auto", @@ -52,7 +51,7 @@ import os os.environ['CEREBRAS_API_KEY'] = "" response = completion( - model="cerebras/llama3-70b-instruct", + model="cerebras/gpt-oss-120b", messages=[ { "role": "user", @@ -60,12 +59,11 @@ response = completion( } ], stream=True, - max_tokens=10, + max_tokens=4096, # The prompt should include JSON if 'json_object' is selected; otherwise, you will get error code 400. response_format={ "type": "json_object" }, seed=123, - stop=["\n\n"], temperature=0.2, top_p=0.9, tool_choice="auto", diff --git a/docs/providers/cohere.md b/docs/providers/cohere.md index 1c3181d18..c9fb26b4a 100644 --- a/docs/providers/cohere.md +++ b/docs/providers/cohere.md @@ -25,7 +25,7 @@ os.environ["COHERE_API_KEY"] = "cohere key" # cohere v2 call response = completion( - model="cohere_chat/command-a-03-2025", + model="cohere_chat/command-a-plus-05-2026", messages = [{ "content": "Hello, how are you?","role": "user"}] ) ``` @@ -42,7 +42,7 @@ os.environ["COHERE_API_KEY"] = "cohere key" # cohere v1 call response = completion( - model="cohere_chat/v1/command-a-03-2025", + model="cohere_chat/v1/command-a-plus-05-2026", messages = [{ "content": "Hello, how are you?","role": "user"}] ) ``` @@ -59,7 +59,7 @@ os.environ["COHERE_API_KEY"] = "cohere key" # cohere v2 streaming response = completion( - model="cohere_chat/command-a-03-2025", + model="cohere_chat/command-a-plus-05-2026", messages = [{ "content": "Hello, how are you?","role": "user"}], stream=True ) @@ -79,7 +79,7 @@ os.environ["COHERE_API_KEY"] = "cohere key" # cohere v1 streaming response = completion( - model="cohere_chat/v1/command-a-03-2025", + model="cohere_chat/v1/command-a-plus-05-2026", messages = [{ "content": "Hello, how are you?","role": "user"}], stream=True ) @@ -106,18 +106,18 @@ Define the cohere models you want to use in the config.yaml **For Cohere v1 models:** ```yaml showLineNumbers model_list: - - model_name: command-a-03-2025 + - model_name: command-a-plus-05-2026 litellm_params: - model: cohere_chat/v1/command-a-03-2025 + model: cohere_chat/v1/command-a-plus-05-2026 api_key: "os.environ/COHERE_API_KEY" ``` **For Cohere v2 models:** ```yaml showLineNumbers model_list: - - model_name: command-a-03-2025-v2 + - model_name: command-a-plus-05-2026-v2 litellm_params: - model: cohere_chat/command-a-03-2025 + model: cohere_chat/command-a-plus-05-2026 api_key: "os.environ/COHERE_API_KEY" ``` @@ -136,7 +136,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer ' \ --data ' { - "model": "command-a-03-2025", + "model": "command-a-plus-05-2026", "messages": [ { "role": "user", @@ -154,7 +154,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer ' \ --data ' { - "model": "command-a-03-2025-v2", + "model": "command-a-plus-05-2026-v2", "messages": [ { "role": "user", @@ -175,7 +175,7 @@ client = openai.OpenAI( ) # request sent to cohere v1 model -response = client.chat.completions.create(model="command-a-03-2025", messages = [ +response = client.chat.completions.create(model="command-a-plus-05-2026", messages = [ { "role": "user", "content": "this is a test request, write a short poem" @@ -195,7 +195,7 @@ client = openai.OpenAI( ) # request sent to cohere v2 model -response = client.chat.completions.create(model="command-a-03-2025-v2", messages = [ +response = client.chat.completions.create(model="command-a-plus-05-2026-v2", messages = [ { "role": "user", "content": "this is a test request, write a short poem" @@ -211,7 +211,7 @@ print(response) ## Supported Models | Model Name | Function Call | |------------|----------------| -| command-a-03-2025 | `litellm.completion('command-a-03-2025', messages)` | +| command-a-plus-05-2026 | `litellm.completion('cohere_chat/command-a-plus-05-2026', messages)` | | command-r-plus-08-2024 | `litellm.completion('command-r-plus-08-2024', messages)` | | command-r-08-2024 | `litellm.completion('command-r-08-2024', messages)` | | command-r-plus | `litellm.completion('command-r-plus', messages)` | @@ -348,4 +348,4 @@ curl http://0.0.0.0:4000/rerank \ ``` - \ No newline at end of file + diff --git a/docs/providers/deepinfra.md b/docs/providers/deepinfra.md index ddf6122ca..1a83196ac 100644 --- a/docs/providers/deepinfra.md +++ b/docs/providers/deepinfra.md @@ -29,7 +29,7 @@ import os os.environ['DEEPINFRA_API_KEY'] = "" response = completion( - model="deepinfra/meta-llama/Llama-2-70b-chat-hf", + model="deepinfra/moonshotai/Kimi-K3", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}] ) ``` @@ -41,7 +41,7 @@ import os os.environ['DEEPINFRA_API_KEY'] = "" response = completion( - model="deepinfra/meta-llama/Llama-2-70b-chat-hf", + model="deepinfra/moonshotai/Kimi-K3", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}], stream=True ) @@ -51,16 +51,14 @@ for chunk in response: ``` ## Chat Models -| Model Name | Function Call | -|------------------|--------------------------------------| -| meta-llama/Meta-Llama-3-8B-Instruct | `completion(model="deepinfra/meta-llama/Meta-Llama-3-8B-Instruct", messages)` | -| meta-llama/Meta-Llama-3-70B-Instruct | `completion(model="deepinfra/meta-llama/Meta-Llama-3-70B-Instruct", messages)` | -| meta-llama/Llama-2-70b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-70b-chat-hf", messages)` | -| meta-llama/Llama-2-7b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-7b-chat-hf", messages)` | -| meta-llama/Llama-2-13b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-13b-chat-hf", messages)` | -| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` | -| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` | -| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` | + +Use `deepinfra/` with a model from the [DeepInfra catalog](https://deepinfra.com/models). These are representative chat models: + +| Model | LiteLLM model ID | +|-------|------------------| +| GLM-5.3 | `deepinfra/zai-org/GLM-5.3` | +| Kimi K3 | `deepinfra/moonshotai/Kimi-K3` | +| GPT OSS 120B | `deepinfra/openai/gpt-oss-120b` | ## Rerank Endpoint diff --git a/docs/providers/deepseek.md b/docs/providers/deepseek.md index fbf3942e3..2fc353559 100644 --- a/docs/providers/deepseek.md +++ b/docs/providers/deepseek.md @@ -19,7 +19,7 @@ import os os.environ['DEEPSEEK_API_KEY'] = "" response = completion( - model="deepseek/deepseek-chat", + model="deepseek/deepseek-v4-pro", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -34,7 +34,7 @@ import os os.environ['DEEPSEEK_API_KEY'] = "" response = completion( - model="deepseek/deepseek-chat", + model="deepseek/deepseek-v4-pro", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -51,7 +51,7 @@ We support ALL Deepseek models, just set `deepseek/` as a prefix when sending co | Model Name | Function Call | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| deepseek-chat | `completion(model="deepseek/deepseek-chat", messages)` | +| deepseek-v4-pro | `completion(model="deepseek/deepseek-v4-pro", messages)` | | deepseek-coder | `completion(model="deepseek/deepseek-coder", messages)` | @@ -170,4 +170,4 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ - \ No newline at end of file + diff --git a/docs/providers/fireworks_ai.md b/docs/providers/fireworks_ai.md index ec7f15fd0..6ac921311 100644 --- a/docs/providers/fireworks_ai.md +++ b/docs/providers/fireworks_ai.md @@ -38,7 +38,7 @@ import os os.environ['FIREWORKS_AI_API_KEY'] = "" response = completion( - model="fireworks_ai/glm-5p2", + model="fireworks_ai/glm-5p3-flash", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -46,7 +46,7 @@ response = completion( print(response) ``` -A bare serverless slug like `glm-5p2` is expanded to `accounts/fireworks/models/glm-5p2` for you, so you can pass either the short slug or the full resource id. +A bare serverless slug like `glm-5p3-flash` is expanded to `accounts/fireworks/models/glm-5p3-flash` for you, so you can pass either the short slug or the full resource id. ## Sample Usage - Serverless Models - Streaming ```python @@ -55,7 +55,7 @@ import os os.environ['FIREWORKS_AI_API_KEY'] = "" response = completion( - model="fireworks_ai/glm-5p2", + model="fireworks_ai/glm-5p3-flash", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -126,9 +126,9 @@ The full resource id (`fireworks_ai/accounts/fireworks/routers/glm-latest`) is s ```yaml model_list: - - model_name: fireworks-glm-5p2 + - model_name: fireworks-glm-5p3-flash litellm_params: - model: fireworks_ai/glm-5p2 + model: fireworks_ai/glm-5p3-flash api_key: "os.environ/FIREWORKS_AI_API_KEY" ``` @@ -148,7 +148,7 @@ litellm --config config.yaml curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --data ' { - "model": "fireworks-glm-5p2", + "model": "fireworks-glm-5p3-flash", "messages": [ { "role": "user", @@ -169,7 +169,7 @@ client = openai.OpenAI( ) # request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create(model="fireworks-glm-5p2", messages = [ +response = client.chat.completions.create(model="fireworks-glm-5p3-flash", messages = [ { "role": "user", "content": "this is a test request, write a short poem" @@ -193,7 +193,7 @@ from langchain.schema import HumanMessage, SystemMessage chat = ChatOpenAI( openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy - model = "fireworks-glm-5p2", + model = "fireworks-glm-5p3-flash", temperature=0.1 ) @@ -449,7 +449,7 @@ We support ALL Fireworks AI models, just set `fireworks_ai/` as a prefix when se | Model Name | Function Call | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| glm-5p2 | `completion(model="fireworks_ai/glm-5p2", messages)` | +| glm-5p3-flash | `completion(model="fireworks_ai/glm-5p3-flash", messages)` | | deepseek-v4-pro | `completion(model="fireworks_ai/deepseek-v4-pro", messages)` | | kimi-k3 | `completion(model="fireworks_ai/kimi-k3", messages)` | | qwen3p8-max | `completion(model="fireworks_ai/qwen3p8-max", messages)` | @@ -611,4 +611,4 @@ curl http://0.0.0.0:4000/rerank \ | Model Name | Function Call | |------------|---------------| -| fireworks/qwen3-reranker-8b | `rerank(model="fireworks_ai/fireworks/qwen3-reranker-8b", query=query, documents=documents)` | \ No newline at end of file +| fireworks/qwen3-reranker-8b | `rerank(model="fireworks_ai/fireworks/qwen3-reranker-8b", query=query, documents=documents)` | diff --git a/docs/providers/gemini.md b/docs/providers/gemini.md index 7ed2bced8..1b36e2cfc 100644 --- a/docs/providers/gemini.md +++ b/docs/providers/gemini.md @@ -19,9 +19,9 @@ import TabItem from '@theme/TabItem'; :::tip Gemini API vs Vertex AI | Model Format | Provider | Auth Required | |-------------|----------|---------------| -| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) | -| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project | -| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project | +| `gemini/{{gemini_flash}}` | Gemini API | `GEMINI_API_KEY` (simple API key) | +| `vertex_ai/{{gemini_flash}}` | Vertex AI | GCP credentials + project | +| `{{gemini_flash}}` (no prefix) | Vertex AI | GCP credentials + project | **If you just want to use an API key** (like OpenAI), use the `gemini/` prefix. @@ -2257,18 +2257,15 @@ response = litellm.completion( **We support ALL Gemini models, just set `model=gemini/` as a prefix when sending litellm requests** ::: -| Model Name | Function Call | Required OS Variables | -|-----------------------|--------------------------------------------------------|--------------------------------| -| gemini-pro | `completion(model='gemini/gemini-pro', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-1.5-pro-latest | `completion(model='gemini/gemini-1.5-pro-latest', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-2.0-flash | `completion(model='gemini/gemini-2.0-flash', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-2.0-flash-exp | `completion(model='gemini/gemini-2.0-flash-exp', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-2.0-flash-lite-preview-02-05 | `completion(model='gemini/gemini-2.0-flash-lite-preview-02-05', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-2.5-flash-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-2.5-flash-lite-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-lite-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-3.1-flash-lite-preview | `completion(model='gemini/gemini-3.1-flash-lite-preview', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-flash-latest | `completion(model='gemini/gemini-flash-latest', messages)` | `os.environ['GEMINI_API_KEY']` | -| gemini-flash-lite-latest | `completion(model='gemini/gemini-flash-lite-latest', messages)` | `os.environ['GEMINI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|------------|---------------|-----------------------| +| gemini-3.8-flash | `completion(model='gemini/gemini-3.8-flash', messages)` | `os.environ['GEMINI_API_KEY']` | +| gemini-3.1-pro-preview | `completion(model='gemini/gemini-3.1-pro-preview', messages)` | `os.environ['GEMINI_API_KEY']` | +| gemini-3.1-flash-lite-preview | `completion(model='gemini/gemini-3.1-flash-lite-preview', messages)` | `os.environ['GEMINI_API_KEY']` | +| gemini-flash-latest | `completion(model='gemini/gemini-flash-latest', messages)` | `os.environ['GEMINI_API_KEY']` | +| gemini-flash-lite-latest | `completion(model='gemini/gemini-flash-lite-latest', messages)` | `os.environ['GEMINI_API_KEY']` | + +This table is a representative selection. See the [Gemini models documentation](https://ai.google.dev/gemini-api/docs/models) for the current catalog. diff --git a/docs/providers/mistral.md b/docs/providers/mistral.md index 8355cd246..76144cb0a 100644 --- a/docs/providers/mistral.md +++ b/docs/providers/mistral.md @@ -17,7 +17,7 @@ import os os.environ['MISTRAL_API_KEY'] = "" response = completion( - model="mistral/mistral-tiny", + model="mistral/mistral-large-2512", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -32,7 +32,7 @@ import os os.environ['MISTRAL_API_KEY'] = "" response = completion( - model="mistral/mistral-tiny", + model="mistral/mistral-large-2512", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -148,7 +148,7 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported. |----------------|--------------------------------------------------------------|-------------------| | Mistral Small | `completion(model="mistral/mistral-small-latest", messages)` | No | | Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`| No | -| Mistral Large 2 | `completion(model="mistral/mistral-large-2407", messages)` | No | +| Mistral Large 3 | `completion(model="mistral/mistral-large-2512", messages)` | No | | Mistral Large Latest | `completion(model="mistral/mistral-large-latest", messages)` | No | | **Magistral Small** | `completion(model="mistral/magistral-small-2506", messages)` | Yes | | **Magistral Medium** | `completion(model="mistral/magistral-medium-2506", messages)`| Yes | diff --git a/docs/providers/moonshot.md b/docs/providers/moonshot.md index 9e5e5bec0..f815048f6 100644 --- a/docs/providers/moonshot.md +++ b/docs/providers/moonshot.md @@ -53,7 +53,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}] # Moonshot call response = completion( - model="moonshot/moonshot-v1-8k", + model="moonshot/kimi-k3", messages=messages ) @@ -73,7 +73,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}] # Moonshot call with streaming response = completion( - model="moonshot/moonshot-v1-8k", + model="moonshot/kimi-k3", messages=messages, stream=True ) @@ -88,9 +88,9 @@ Add the following to your LiteLLM Proxy configuration file: ```yaml showLineNumbers title="config.yaml" model_list: - - model_name: moonshot-v1-8k + - model_name: kimi-k3 litellm_params: - model: moonshot/moonshot-v1-8k + model: moonshot/kimi-k3 api_key: os.environ/MOONSHOT_API_KEY - model_name: moonshot-v1-32k @@ -126,7 +126,7 @@ client = OpenAI( # Non-streaming response response = client.chat.completions.create( - model="moonshot-v1-8k", + model="kimi-k3", messages=[{"role": "user", "content": "hello from litellm"}] ) @@ -144,7 +144,7 @@ client = OpenAI( # Streaming response response = client.chat.completions.create( - model="moonshot-v1-8k", + model="kimi-k3", messages=[{"role": "user", "content": "hello from litellm"}], stream=True ) @@ -163,7 +163,7 @@ import litellm # Configure LiteLLM to use your proxy response = litellm.completion( - model="litellm_proxy/moonshot-v1-8k", + model="litellm_proxy/kimi-k3", messages=[{"role": "user", "content": "hello from litellm"}], api_base="http://localhost:4000", api_key="your-proxy-api-key" @@ -177,7 +177,7 @@ import litellm # Configure LiteLLM to use your proxy with streaming response = litellm.completion( - model="litellm_proxy/moonshot-v1-8k", + model="litellm_proxy/kimi-k3", messages=[{"role": "user", "content": "hello from litellm"}], api_base="http://localhost:4000", api_key="your-proxy-api-key", @@ -198,7 +198,7 @@ curl http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer your-proxy-api-key" \ -d '{ - "model": "moonshot-v1-8k", + "model": "kimi-k3", "messages": [{"role": "user", "content": "hello from litellm"}] }' ``` @@ -208,7 +208,7 @@ curl http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer your-proxy-api-key" \ -d '{ - "model": "moonshot-v1-8k", + "model": "kimi-k3", "messages": [{"role": "user", "content": "hello from litellm"}], "stream": true }' diff --git a/docs/providers/nvidia_nim.md b/docs/providers/nvidia_nim.md index 9dbfc80f4..2e11a6e85 100644 --- a/docs/providers/nvidia_nim.md +++ b/docs/providers/nvidia_nim.md @@ -32,19 +32,18 @@ import os os.environ['NVIDIA_NIM_API_KEY'] = "" response = completion( - model="nvidia_nim/meta/llama3-70b-instruct", + model="nvidia_nim/nvidia/nemotron-3-super-120b-a12b", messages=[ { "role": "user", "content": "What's the weather like in Boston today in Fahrenheit?", } ], - temperature=0.2, # optional - top_p=0.9, # optional + temperature=1.0, # optional + top_p=0.95, # optional frequency_penalty=0.1, # optional presence_penalty=0.1, # optional - max_tokens=10, # optional - stop=["\n\n"], # optional + max_tokens=4096, # optional ) print(response) ``` @@ -56,7 +55,7 @@ import os os.environ['NVIDIA_NIM_API_KEY'] = "" response = completion( - model="nvidia_nim/meta/llama3-70b-instruct", + model="nvidia_nim/nvidia/nemotron-3-super-120b-a12b", messages=[ { "role": "user", @@ -64,12 +63,11 @@ response = completion( } ], stream=True, - temperature=0.2, # optional - top_p=0.9, # optional + temperature=1.0, # optional + top_p=0.95, # optional frequency_penalty=0.1, # optional presence_penalty=0.1, # optional - max_tokens=10, # optional - stop=["\n\n"], # optional + max_tokens=4096, # optional ) for chunk in response: @@ -168,39 +166,11 @@ Here's how to call an Nvidia NIM Endpoint with the LiteLLM Proxy Server -## Supported Models - 💥 ALL Nvidia NIM Models Supported! -We support ALL `nvidia_nim` models, just set `nvidia_nim/` as a prefix when sending completion requests - -| Model Name | Function Call | -|------------|---------------| -| nvidia/nemotron-4-340b-reward | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-reward", messages)` | -| 01-ai/yi-large | `completion(model="nvidia_nim/01-ai/yi-large", messages)` | -| aisingapore/sea-lion-7b-instruct | `completion(model="nvidia_nim/aisingapore/sea-lion-7b-instruct", messages)` | -| databricks/dbrx-instruct | `completion(model="nvidia_nim/databricks/dbrx-instruct", messages)` | -| google/gemma-7b | `completion(model="nvidia_nim/google/gemma-7b", messages)` | -| google/gemma-2b | `completion(model="nvidia_nim/google/gemma-2b", messages)` | -| google/codegemma-1.1-7b | `completion(model="nvidia_nim/google/codegemma-1.1-7b", messages)` | -| google/codegemma-7b | `completion(model="nvidia_nim/google/codegemma-7b", messages)` | -| google/recurrentgemma-2b | `completion(model="nvidia_nim/google/recurrentgemma-2b", messages)` | -| ibm/granite-34b-code-instruct | `completion(model="nvidia_nim/ibm/granite-34b-code-instruct", messages)` | -| ibm/granite-8b-code-instruct | `completion(model="nvidia_nim/ibm/granite-8b-code-instruct", messages)` | -| mediatek/breeze-7b-instruct | `completion(model="nvidia_nim/mediatek/breeze-7b-instruct", messages)` | -| meta/codellama-70b | `completion(model="nvidia_nim/meta/codellama-70b", messages)` | -| meta/llama2-70b | `completion(model="nvidia_nim/meta/llama2-70b", messages)` | -| meta/llama3-8b | `completion(model="nvidia_nim/meta/llama3-8b", messages)` | -| meta/llama3-70b | `completion(model="nvidia_nim/meta/llama3-70b", messages)` | -| microsoft/phi-3-medium-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-medium-4k-instruct", messages)` | -| microsoft/phi-3-mini-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-128k-instruct", messages)` | -| microsoft/phi-3-mini-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-4k-instruct", messages)` | -| microsoft/phi-3-small-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-128k-instruct", messages)` | -| microsoft/phi-3-small-8k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-8k-instruct", messages)` | -| mistralai/codestral-22b-instruct-v0.1 | `completion(model="nvidia_nim/mistralai/codestral-22b-instruct-v0.1", messages)` | -| mistralai/mistral-7b-instruct | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct", messages)` | -| mistralai/mistral-7b-instruct-v0.3 | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct-v0.3", messages)` | -| mistralai/mixtral-8x7b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x7b-instruct", messages)` | -| mistralai/mixtral-8x22b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x22b-instruct", messages)` | -| mistralai/mistral-large | `completion(model="nvidia_nim/mistralai/mistral-large", messages)` | -| nvidia/nemotron-4-340b-instruct | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-instruct", messages)` | -| seallms/seallm-7b-v2.5 | `completion(model="nvidia_nim/seallms/seallm-7b-v2.5", messages)` | -| snowflake/arctic | `completion(model="nvidia_nim/snowflake/arctic", messages)` | -| upstage/solar-10.7b-instruct | `completion(model="nvidia_nim/upstage/solar-10.7b-instruct", messages)` | \ No newline at end of file +## Supported Models + +Use `nvidia_nim/` with a chat model available on your NVIDIA endpoint. These are representative models from the [NVIDIA model catalog](https://build.nvidia.com/models): + +| Model | LiteLLM model ID | +|-------|------------------| +| Nemotron 3 Super | `nvidia_nim/nvidia/nemotron-3-super-120b-a12b` | +| Nemotron 3.5 Lightning | `nvidia_nim/nvidia/nemotron-3.5-lightning-30b-a3b` | diff --git a/docs/providers/sambanova.md b/docs/providers/sambanova.md index bf2342357..f7df63496 100644 --- a/docs/providers/sambanova.md +++ b/docs/providers/sambanova.md @@ -23,14 +23,14 @@ import os os.environ['SAMBANOVA_API_KEY'] = "" response = completion( - model="sambanova/Llama-4-Maverick-17B-128E-Instruct", + model="sambanova/MiniMax-M2.7", messages=[ { "role": "user", "content": "What do you know about SambaNova Systems", } ], - max_tokens=10, + max_tokens=4096, stop=[], temperature=0.2, top_p=0.9, @@ -46,7 +46,7 @@ import os os.environ['SAMBANOVA_API_KEY'] = "" response = completion( - model="sambanova/Llama-4-Maverick-17B-128E-Instruct", + model="sambanova/MiniMax-M2.7", messages=[ { "role": "user", @@ -54,8 +54,7 @@ response = completion( } ], stream=True, - max_tokens=10, - response_format={ "type": "json_object" }, + max_tokens=4096, stop=[], temperature=0.2, top_p=0.9, diff --git a/docs/providers/togetherai.md b/docs/providers/togetherai.md index 5f258ae5a..4d215f07a 100644 --- a/docs/providers/togetherai.md +++ b/docs/providers/togetherai.md @@ -19,193 +19,23 @@ os.environ["TOGETHERAI_API_KEY"] = "your-api-key" messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] -completion(model="together_ai/togethercomputer/Llama-2-7B-32K-Instruct", messages=messages) +completion(model="together_ai/zai-org/GLM-5.3-Flash", messages=messages) ``` ## Together AI Models -liteLLM supports `non-streaming` and `streaming` requests to all models on https://api.together.xyz/ - -Example TogetherAI Usage - Note: liteLLM supports all models deployed on TogetherAI - - -### Llama LLMs - Chat -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|-------------------------------------------------------------------------|------------------------------------| -| togethercomputer/llama-2-70b-chat | `completion('together_ai/togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Llama LLMs - Language / Instruct -| Model Name | Function Call | Required OS Variables | -|------------------------------------------|--------------------------------------------------------------------------------|------------------------------------| -| togethercomputer/llama-2-70b | `completion('together_ai/togethercomputer/llama-2-70b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/LLaMA-2-7B-32K | `completion('together_ai/togethercomputer/LLaMA-2-7B-32K', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/Llama-2-7B-32K-Instruct | `completion('together_ai/togethercomputer/Llama-2-7B-32K-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/llama-2-7b | `completion('together_ai/togethercomputer/llama-2-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Falcon LLMs -| Model Name | Function Call | Required OS Variables | -|--------------------------------------|----------------------------------------------------------------------------|------------------------------------| -| togethercomputer/falcon-40b-instruct | `completion('together_ai/togethercomputer/falcon-40b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/falcon-7b-instruct | `completion('together_ai/togethercomputer/falcon-7b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Alpaca LLMs -| Model Name | Function Call | Required OS Variables | -|----------------------------|------------------------------------------------------------------|------------------------------------| -| togethercomputer/alpaca-7b | `completion('together_ai/togethercomputer/alpaca-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Other Chat LLMs -| Model Name | Function Call | Required OS Variables | -|------------------------------|--------------------------------------------------------------------|------------------------------------| -| HuggingFaceH4/starchat-alpha | `completion('together_ai/HuggingFaceH4/starchat-alpha', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Code LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------------|-------------------------------------------------------------------------------|------------------------------------| -| togethercomputer/CodeLlama-34b | `completion('together_ai/togethercomputer/CodeLlama-34b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/CodeLlama-34b-Instruct | `completion('together_ai/togethercomputer/CodeLlama-34b-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/CodeLlama-34b-Python | `completion('together_ai/togethercomputer/CodeLlama-34b-Python', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| defog/sqlcoder | `completion('together_ai/defog/sqlcoder', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| NumbersStation/nsql-llama-2-7B | `completion('together_ai/NumbersStation/nsql-llama-2-7B', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardCoder-15B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-15B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardCoder-Python-34B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-Python-34B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | - -### Language LLMs -| Model Name | Function Call | Required OS Variables | -|-------------------------------------|---------------------------------------------------------------------------|------------------------------------| -| NousResearch/Nous-Hermes-Llama2-13b | `completion('together_ai/NousResearch/Nous-Hermes-Llama2-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| Austism/chronos-hermes-13b | `completion('together_ai/Austism/chronos-hermes-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| upstage/SOLAR-0-70b-16bit | `completion('together_ai/upstage/SOLAR-0-70b-16bit', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardLM-70B-V1.0 | `completion('together_ai/WizardLM/WizardLM-70B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +Use `together_ai/` for streaming and non-streaming chat requests. These are representative models; check the [Together AI catalog](https://docs.together.ai/docs/serverless-models) for availability and deployment requirements. -## Prompt Templates - -Using a chat model on Together AI with it's own prompt format? - -### Using Llama2 Instruct models -If you're using Together AI's Llama2 variants( `model=togethercomputer/llama-2..-instruct`), LiteLLM can automatically translate between the OpenAI prompt format and the TogetherAI Llama2 one (`[INST]..[/INST]`). - -```python -from litellm import completion - -# set env variable -os.environ["TOGETHERAI_API_KEY"] = "" +| Model | LiteLLM model ID | +|-------|------------------| +| GLM-5.3-Flash | `together_ai/zai-org/GLM-5.3-Flash` | +| Kimi K3 | `together_ai/moonshotai/Kimi-K3` | +| GPT OSS 120B | `together_ai/openai/gpt-oss-120b` | -messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] - -completion(model="together_ai/togethercomputer/Llama-2-7B-32K-Instruct", messages=messages) -``` - -### Using another model - -You can create a custom prompt template on LiteLLM (and we [welcome PRs](https://github.com/BerriAI/litellm) to add them to the main repo 🤗) - -Let's make one for `OpenAssistant/llama2-70b-oasst-sft-v10`! - -The accepted template format is: [Reference](https://huggingface.co/OpenAssistant/llama2-70b-oasst-sft-v10-) -``` -""" -<|im_start|>system -{system_message}<|im_end|> -<|im_start|>user -{prompt}<|im_end|> -<|im_start|>assistant -""" -``` - -Let's register our custom prompt template: [Implementation Code](https://github.com/BerriAI/litellm/blob/64f3d3c56ef02ac5544983efc78293de31c1c201/litellm/llms/prompt_templates/factory.py#L77) -```python -import litellm - -litellm.register_prompt_template( - model="OpenAssistant/llama2-70b-oasst-sft-v10", - roles={ - "system": { - "pre_message": "[<|im_start|>system", - "post_message": "\n" - }, - "user": { - "pre_message": "<|im_start|>user", - "post_message": "\n" - }, - "assistant": { - "pre_message": "<|im_start|>assistant", - "post_message": "\n" - } - } - ) -``` -Let's use it! - -```python -from litellm import completion - -# set env variable -os.environ["TOGETHERAI_API_KEY"] = "" - -messages=[{"role":"user", "content": "Write me a poem about the blue sky"}] - -completion(model="together_ai/OpenAssistant/llama2-70b-oasst-sft-v10", messages=messages) -``` - -**Complete Code** - -```python -import litellm -from litellm import completion - -# set env variable -os.environ["TOGETHERAI_API_KEY"] = "" - -litellm.register_prompt_template( - model="OpenAssistant/llama2-70b-oasst-sft-v10", - roles={ - "system": { - "pre_message": "[<|im_start|>system", - "post_message": "\n" - }, - "user": { - "pre_message": "<|im_start|>user", - "post_message": "\n" - }, - "assistant": { - "pre_message": "<|im_start|>assistant", - "post_message": "\n" - } - } - ) - -messages=[{"role":"user", "content": "Write me a poem about the blue sky"}] - -response = completion(model="together_ai/OpenAssistant/llama2-70b-oasst-sft-v10", messages=messages) - -print(response) -``` +## Prompt Templates -**Output** -```json -{ - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": ".\n\nThe sky is a canvas of blue,\nWith clouds that drift and move,", - "role": "assistant", - "logprobs": null - } - } - ], - "created": 1693941410.482018, - "model": "OpenAssistant/llama2-70b-oasst-sft-v10", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 16, - "total_tokens": 23 - }, - "litellm_call_id": "f21315db-afd6-4c1e-b43a-0b5682de4b06" -} -``` +Chat models accept OpenAI-style messages. For a deployment that requires a custom text prompt, see [prompt formatting](../completion/prompt_formatting.md#format-prompt-yourself). ## Reasoning controls via `chat_template_kwargs` @@ -360,4 +190,4 @@ curl http://0.0.0.0:4000/rerank \ ``` - \ No newline at end of file + diff --git a/docs/providers/vertex.md b/docs/providers/vertex.md index 694288e22..7de7e868e 100644 --- a/docs/providers/vertex.md +++ b/docs/providers/vertex.md @@ -17,9 +17,9 @@ import TabItem from '@theme/TabItem'; :::tip Vertex AI vs Gemini API | Model Format | Provider | Auth Required | |-------------|----------|---------------| -| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project | -| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project | -| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) | +| `vertex_ai/{{gemini_flash}}` | Vertex AI | GCP credentials + project | +| `{{gemini_flash}}` (no prefix) | Vertex AI | GCP credentials + project | +| `gemini/{{gemini_flash}}` | Gemini API | `GEMINI_API_KEY` (simple API key) | **If you just want to use an API key** (like OpenAI), use the `gemini/` prefix instead. See [Gemini - Google AI Studio](./gemini.md). @@ -1755,13 +1755,14 @@ os.environ["VERTEXAI_LOCATION"] = "us-central1" # Your Location litellm.vertex_location = "us-central1" # Your Location ``` -## Gemini Pro -| Model Name | Function Call | -|------------------|--------------------------------------| -| gemini-2.5-pro | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` | -| gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` | -| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` | -| gemini-3.1-flash-lite-preview | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` | +## Gemini Models {#gemini-pro} +| Model Name | Function Call | +|------------|---------------| +| gemini-3.8-flash | `completion('gemini-3.8-flash', messages)`, `completion('vertex_ai/gemini-3.8-flash', messages)` | +| gemini-3.1-pro-preview | `completion('gemini-3.1-pro-preview', messages)`, `completion('vertex_ai/gemini-3.1-pro-preview', messages)` | +| gemini-3.1-flash-lite-preview | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` | + +This table is a representative selection. See the [Vertex AI model catalog](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models) for the current catalog. ## PayGo / Priority Cost Tracking diff --git a/docs/providers/xai.md b/docs/providers/xai.md index 083cf46df..0ae4587b8 100644 --- a/docs/providers/xai.md +++ b/docs/providers/xai.md @@ -13,18 +13,18 @@ https://docs.x.ai/docs ## Supported Models -**Grok 4.5** - Frontier model for coding, agentic tasks, and knowledge work with 500K context, reasoning (low/medium/high), vision, tools, web search, and prompt caching. +**Grok 4.6** - Frontier model for coding, agentic tasks, and knowledge work with 500K context, reasoning (low/medium/high), vision, tools, web search, and prompt caching. | Model | Context | Features | |-------|---------|----------| -| `xai/grok-4.5` | 500K tokens | **Reasoning**, Function calling, Vision, Web search, Caching | +| `xai/grok-4.6` | 500K tokens | **Reasoning**, Function calling, Vision, Web search, Caching | **Example:** ```python from litellm import completion response = completion( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[{"role": "user", "content": "Find and fix the bug, then explain it."}], reasoning_effort="high", # low | medium | high (default high) ) @@ -54,14 +54,14 @@ import os os.environ['XAI_API_KEY'] = "" response = completion( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[ { "role": "user", "content": "What's the weather like in Boston today in Fahrenheit?", } ], - max_tokens=10, + max_tokens=4096, response_format={ "type": "json_object" }, seed=123, temperature=0.2, @@ -81,7 +81,7 @@ import os os.environ['XAI_API_KEY'] = "" response = completion( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[ { "role": "user", @@ -89,7 +89,7 @@ response = completion( } ], stream=True, - max_tokens=10, + max_tokens=4096, response_format={ "type": "json_object" }, seed=123, temperature=0.2, @@ -112,7 +112,7 @@ from litellm import completion os.environ["XAI_API_KEY"] = "your-api-key" response = completion( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[ { "role": "user", @@ -211,10 +211,10 @@ LiteLLM supports reasoning usage for xAI models. -```python showLineNumbers title="reasoning with xai/grok-4.5" +```python showLineNumbers title="reasoning with xai/grok-4.6" import litellm response = litellm.completion( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[{"role": "user", "content": "What is 101*3?"}], reasoning_effort="low", # low | medium | high ) @@ -235,7 +235,7 @@ print(response.usage.completion_tokens_details.reasoning_tokens) -```python showLineNumbers title="reasoning with xai/grok-4.5" +```python showLineNumbers title="reasoning with xai/grok-4.6" import openai client = openai.OpenAI( api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys @@ -243,7 +243,7 @@ client = openai.OpenAI( ) response = client.chat.completions.create( - model="xai/grok-4.5", + model="xai/grok-4.6", messages=[{"role": "user", "content": "What is 101*3?"}], reasoning_effort="low", # low | medium | high )