From e644ed3882f39515f151aaa68c6aedae5ce4088c Mon Sep 17 00:00:00 2001 From: AJ-ing Date: Sat, 25 Jul 2026 23:57:41 +1000 Subject: [PATCH 1/2] feat(openinfer): add OpenInfer as OpenAI-compatible provider Register OpenInfer via the JSON provider registry so users can call `openinfer/` with OPENINFER_API_KEY. Adds enum/constants routing, URL autodetection, endpoint matrix entry, a placeholder model price row, and unit tests. Fixes #34357 Co-authored-by: Cursor --- litellm/constants.py | 2 + .../get_llm_provider_logic.py | 3 + litellm/llms/openai_like/providers.json | 10 ++ ...odel_prices_and_context_window_backup.json | 11 +++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 11 +++ provider_endpoints_support.json | 17 ++++ .../openai_like/test_openinfer_provider.py | 91 +++++++++++++++++++ 8 files changed, 146 insertions(+) create mode 100644 tests/test_litellm/llms/openai_like/test_openinfer_provider.py diff --git a/litellm/constants.py b/litellm/constants.py index b9b9c0ba604..bbb30a49a8e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -728,6 +728,7 @@ "https://api.libertai.io/v1", "https://pinstripes.io/v1", "https://api.meta.ai/v1", + "https://api.openinfer.ai/v1", ] @@ -793,6 +794,7 @@ "docker_model_runner", "ragflow", "pinstripes", # Pinstripes - JSON-configured provider + "openinfer", # OpenInfer - JSON-configured provider "darkbloom", "meta", # Meta Model API (Muse Spark) - JSON-configured provider ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 487a7b7e25f..77659264f3d 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -346,6 +346,9 @@ def get_llm_provider( elif endpoint == "https://pinstripes.io/v1": custom_llm_provider = "pinstripes" dynamic_api_key = get_secret_str("PINSTRIPES_API_KEY") + elif endpoint == "https://api.openinfer.ai/v1": + custom_llm_provider = "openinfer" + dynamic_api_key = get_secret_str("OPENINFER_API_KEY") elif endpoint == "https://api.meta.ai/v1": custom_llm_provider = "meta" dynamic_api_key = get_secret_str("META_API_KEY") diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 164100d4194..feb7fbc5c77 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -183,5 +183,15 @@ "max_completion_tokens": "max_tokens" }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] + }, + "openinfer": { + "base_url": "https://api.openinfer.ai/v1", + "api_key_env": "OPENINFER_API_KEY", + "api_base_env": "OPENINFER_API_BASE", + "base_class": "openai_gpt", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d43eda39b1f..e1a6a348830 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46190,5 +46190,16 @@ } } ] + }, + "openinfer/llama-3.1-8b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openinfer", + "mode": "chat", + "supports_function_calling": true, + "source": "https://api.openinfer.ai/v1/models" } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e4dfac48141..47ea4b81364 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3508,6 +3508,7 @@ class LlmProviders(str, Enum): TENSORMESH = "tensormesh" LIBERTAI = "libertai" PINSTRIPES = "pinstripes" + OPENINFER = "openinfer" DARKBLOOM = "darkbloom" META = "meta" LITELLM_AGENT = "litellm_agent" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 749b2566c2a..fbf7724bef9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46312,5 +46312,16 @@ } } ] + }, + "openinfer/llama-3.1-8b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openinfer", + "mode": "chat", + "supports_function_calling": true, + "source": "https://api.openinfer.ai/v1/models" } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 65db63dc045..45515548dcc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2018,6 +2018,23 @@ "a2a": false } }, + "openinfer": { + "display_name": "OpenInfer (`openinfer`)", + "url": "https://openinfer.io", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "poe": { "display_name": "Poe (`poe`)", "endpoints": { diff --git a/tests/test_litellm/llms/openai_like/test_openinfer_provider.py b/tests/test_litellm/llms/openai_like/test_openinfer_provider.py new file mode 100644 index 00000000000..76af19c3670 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_openinfer_provider.py @@ -0,0 +1,91 @@ +""" +Tests for the OpenInfer LLM provider configuration and integration. +""" + +import litellm + + +class TestOpenInferProviderConfig: + def test_openinfer_in_provider_list(self): + from litellm import LlmProviders + + assert LlmProviders.OPENINFER.value == "openinfer" + assert "openinfer" in litellm.provider_list + + def test_openinfer_json_config(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("openinfer") + assert provider is not None + assert provider.base_url == "https://api.openinfer.ai/v1" + assert provider.api_key_env == "OPENINFER_API_KEY" + assert provider.api_base_env == "OPENINFER_API_BASE" + assert not JSONProviderRegistry.supports_responses_api("openinfer") + + def test_openinfer_in_openai_compatible_providers(self): + from litellm.constants import openai_compatible_providers + + assert "openinfer" in openai_compatible_providers + + def test_provider_prefixed_model_routes_to_openinfer(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="openinfer/llama-3.1-8b-instruct", + custom_llm_provider=None, + api_base=None, + api_key="sk-test", + ) + + assert model == "llama-3.1-8b-instruct" + assert provider == "openinfer" + assert api_key == "sk-test" + assert api_base == "https://api.openinfer.ai/v1" + + def test_api_key_and_base_resolved_from_env(self, monkeypatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("OPENINFER_API_KEY", "sk-env-key") + monkeypatch.setenv("OPENINFER_API_BASE", "https://proxy.internal/v1") + + _, provider, api_key, api_base = get_llm_provider( + model="openinfer/llama-3.1-8b-instruct", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert provider == "openinfer" + assert api_key == "sk-env-key" + assert api_base == "https://proxy.internal/v1" + + def test_url_autodetection_from_api_base(self, monkeypatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("OPENINFER_API_KEY", "sk-env-key") + + _, provider, api_key, api_base = get_llm_provider( + model="llama-3.1-8b-instruct", + custom_llm_provider=None, + api_base="https://api.openinfer.ai/v1", + api_key=None, + ) + + assert provider == "openinfer" + assert api_key == "sk-env-key" + + def test_chat_completions_url(self): + config = litellm.ProviderConfigManager.get_provider_chat_config( + model="llama-3.1-8b-instruct", provider=litellm.LlmProviders.OPENINFER + ) + assert config is not None + assert ( + config.get_complete_url( + api_base=None, + api_key="sk-test", + model="llama-3.1-8b-instruct", + optional_params={}, + litellm_params={}, + ) + == "https://api.openinfer.ai/v1/chat/completions" + ) From 7394f8d73fb06b256ad75827900e71800bc6820e Mon Sep 17 00:00:00 2001 From: AJ-ing Date: Sun, 26 Jul 2026 18:09:42 +1000 Subject: [PATCH 2/2] fix(openinfer): align max_tokens with max_output_tokens Co-authored-by: Cursor --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e1a6a348830..35affb047eb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46192,7 +46192,7 @@ ] }, "openinfer/llama-3.1-8b-instruct": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 0.0, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fbf7724bef9..33729c284fb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46314,7 +46314,7 @@ ] }, "openinfer/llama-3.1-8b-instruct": { - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 0.0,