diff --git a/tests/entrypoints/openai/responses/test_sampling_params.py b/tests/entrypoints/openai/responses/test_sampling_params.py index 87910271dd75..5a68e3a9c0d4 100644 --- a/tests/entrypoints/openai/responses/test_sampling_params.py +++ b/tests/entrypoints/openai/responses/test_sampling_params.py @@ -132,6 +132,21 @@ def test_structured_outputs_passed_through(self): assert sampling_params.structured_outputs is not None assert sampling_params.structured_outputs.grammar == "root ::= 'hello'" + def test_text_format_json_object_enables_structured_outputs(self): + """text.format json_object enables structured outputs for sampling.""" + request = ResponsesRequest( + model="test-model", + input="test input", + text=ResponseTextConfig.model_validate({"format": {"type": "json_object"}}), + ) + + sampling_params = request.to_sampling_params(default_max_tokens=1000) + + assert sampling_params.structured_outputs is not None + assert sampling_params.structured_outputs.json_object is True + assert sampling_params.structured_outputs.json is None + assert request.structured_outputs is None + def test_structured_outputs_and_json_schema_conflict(self): """Test that specifying both structured_outputs and json_schema raises.""" structured_outputs = StructuredOutputsParams(grammar="root ::= 'hello'") diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 76a03dd72028..8c1694bbf73e 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -3,7 +3,6 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py -import json import time from typing import Annotated, Any, ClassVar, Literal @@ -14,7 +13,6 @@ from pydantic import Field, PrivateAttr, model_serializer, model_validator from vllm.config import ModelConfig -from vllm.config.utils import replace from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, @@ -24,13 +22,12 @@ DeltaMessage, FunctionCall, FunctionDefinition, - LegacyStructuralTagResponseFormat, OpenAIBaseModel, PerRequestTimingMetrics, StreamOptions, - StructuralTagResponseFormat, ToolCall, UsageInfo, + structured_outputs_from_response_format, validate_structural_tag_response_format, validate_structured_outputs_structural_tag, ) @@ -607,6 +604,13 @@ def to_beam_search_params( include_stop_str_in_output=self.include_stop_str_in_output, ) + def extract_structured_outputs(self) -> StructuredOutputsParams | None: + """Normalize request constraints into ``StructuredOutputsParams``.""" + return structured_outputs_from_response_format( + self.structured_outputs, + self.response_format, + ) + def to_sampling_params( self, max_tokens: int, @@ -651,38 +655,6 @@ def to_sampling_params( if prompt_logprobs is None and self.echo: prompt_logprobs = self.top_logprobs - response_format = self.response_format - if response_format is not None: - structured_outputs_kwargs = dict[str, Any]() - - # Set structured output params for response format - if response_format.type == "json_object": - structured_outputs_kwargs["json_object"] = True - elif response_format.type == "json_schema": - json_schema = response_format.json_schema - assert json_schema is not None - structured_outputs_kwargs["json"] = json_schema.json_schema - elif response_format.type == "structural_tag": - structural_tag = response_format - assert structural_tag is not None and isinstance( - structural_tag, - ( - LegacyStructuralTagResponseFormat, - StructuralTagResponseFormat, - ), - ) - s_tag_obj = structural_tag.model_dump(by_alias=True) - structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj) - - # If structured outputs wasn't already enabled, - # we must enable it for these features to work - if len(structured_outputs_kwargs) > 0: - self.structured_outputs = ( - StructuredOutputsParams(**structured_outputs_kwargs) - if self.structured_outputs is None - else replace(self.structured_outputs, **structured_outputs_kwargs) - ) - extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} if self.kv_transfer_params: # Pass in kv_transfer_params via extra_args @@ -718,7 +690,7 @@ def to_sampling_params( output_kind=RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY, - structured_outputs=self.structured_outputs, + structured_outputs=self.extract_structured_outputs(), logit_bias=self.logit_bias, bad_words=self.bad_words, thinking_token_budget=self.thinking_token_budget, diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 1784d0f5364c..73677b16af9b 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -3,7 +3,6 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py -import json import time from typing import Annotated, Any, Literal @@ -11,15 +10,13 @@ import vllm.envs as envs from vllm.config import ModelConfig -from vllm.config.utils import replace from vllm.entrypoints.openai.engine.protocol import ( AnyResponseFormat, - LegacyStructuralTagResponseFormat, OpenAIBaseModel, PerRequestTimingMetrics, StreamOptions, - StructuralTagResponseFormat, UsageInfo, + structured_outputs_from_response_format, validate_structural_tag_response_format, validate_structured_outputs_structural_tag, ) @@ -281,6 +278,13 @@ def to_beam_search_params( include_stop_str_in_output=self.include_stop_str_in_output, ) + def extract_structured_outputs(self) -> StructuredOutputsParams | None: + """Normalize request constraints into ``StructuredOutputsParams``.""" + return structured_outputs_from_response_format( + self.structured_outputs, + self.response_format, + ) + def to_sampling_params( self, max_tokens: int, @@ -330,38 +334,6 @@ def to_sampling_params( echo_without_generation = self.echo and self.max_tokens == 0 - response_format = self.response_format - if response_format is not None: - structured_outputs_kwargs = dict[str, Any]() - - # Set structured output params for response format - if response_format.type == "json_object": - structured_outputs_kwargs["json_object"] = True - elif response_format.type == "json_schema": - json_schema = response_format.json_schema - assert json_schema is not None - structured_outputs_kwargs["json"] = json_schema.json_schema - elif response_format.type == "structural_tag": - structural_tag = response_format - assert isinstance( - structural_tag, - ( - LegacyStructuralTagResponseFormat, - StructuralTagResponseFormat, - ), - ) - s_tag_obj = structural_tag.model_dump(by_alias=True) - structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj) - - # If structured outputs wasn't already enabled, - # we must enable it for these features to work - if len(structured_outputs_kwargs) > 0: - self.structured_outputs = ( - StructuredOutputsParams(**structured_outputs_kwargs) - if self.structured_outputs is None - else replace(self.structured_outputs, **structured_outputs_kwargs) - ) - extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} if self.kv_transfer_params: # Pass in kv_transfer_params via extra_args @@ -393,7 +365,7 @@ def to_sampling_params( output_kind=RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY, - structured_outputs=self.structured_outputs, + structured_outputs=self.extract_structured_outputs(), logit_bias=self.logit_bias, allowed_token_ids=self.allowed_token_ids, bad_words=self.bad_words, diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 2c32fcf20c61..85717d4340a4 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -3,6 +3,7 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py +import json import time from http import HTTPStatus from typing import Any, ClassVar, Literal, TypeAlias @@ -16,9 +17,11 @@ model_validator, ) +from vllm.config.utils import replace from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger +from vllm.sampling_params import StructuredOutputsParams from vllm.utils import random_uuid from vllm.utils.import_utils import resolve_obj_by_qualname @@ -172,6 +175,39 @@ class ResponseFormat(OpenAIBaseModel): ) +def structured_outputs_from_response_format( + structured_outputs: StructuredOutputsParams | None, + response_format: AnyResponseFormat | None, +) -> StructuredOutputsParams | None: + """Apply ``response_format`` overrides to ``structured_outputs``.""" + if response_format is None or response_format.type == "text": + return structured_outputs + + overrides: dict[str, Any] + if response_format.type == "json_object": + overrides = {"json_object": True} + elif response_format.type == "json_schema": + json_schema = response_format.json_schema + assert json_schema is not None + overrides = {"json": json_schema.json_schema} + else: + assert isinstance( + response_format, + ( + LegacyStructuralTagResponseFormat, + StructuralTagResponseFormat, + ), + ) + overrides = { + "structural_tag": json.dumps(response_format.model_dump(by_alias=True)) + } + + if structured_outputs is None: + return StructuredOutputsParams(**overrides) + + return replace(structured_outputs, **overrides) + + def validate_structural_tag_response_format( response_format: AnyStructuralTagResponseFormat | dict[str, Any], ) -> None: @@ -180,8 +216,6 @@ def validate_structural_tag_response_format( Engine-side validation reports malformed structural tags as generation failures. OpenAI request parsing should classify them as bad requests. """ - import json - from pydantic import TypeAdapter, ValidationError if isinstance(response_format, dict): diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index d4708a5fb3ee..3f6857dcc323 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -354,6 +354,31 @@ def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: "top_k": 0, } + def extract_structured_outputs(self) -> StructuredOutputsParams | None: + """Normalize request constraints into ``StructuredOutputsParams``.""" + if self.text is None or self.text.format is None: + return self.structured_outputs + + if self.structured_outputs is not None: + raise VLLMValidationError( + "Cannot specify both structured_outputs and text.format", + parameter="structured_outputs", + ) + + response_format = self.text.format + if response_format.type == "json_object": + return StructuredOutputsParams(json_object=True) + if ( + response_format.type == "json_schema" + and response_format.schema_ is not None + ): + return StructuredOutputsParams( + json=response_format.schema_ # type: ignore[call-arg] + # --follow-imports skip hides the class definition but also hides + # multiple third party conflicts, so best of both evils + ) + return None + def to_sampling_params( self, default_max_tokens: int, @@ -387,27 +412,6 @@ def to_sampling_params( if (frequency_penalty := self.frequency_penalty) is None: frequency_penalty = default_sampling_params.get("frequency_penalty", 0.0) - # Structured output - structured_outputs = self.structured_outputs - - # Also check text.format for OpenAI-style json_schema - if self.text is not None and self.text.format is not None: - if structured_outputs is not None: - raise VLLMValidationError( - "Cannot specify both structured_outputs and text.format", - parameter="structured_outputs", - ) - response_format = self.text.format - if ( - response_format.type == "json_schema" - and response_format.schema_ is not None - ): - structured_outputs = StructuredOutputsParams( - json=response_format.schema_ # type: ignore[call-arg] - # --follow-imports skip hides the class definition but also hides - # multiple third party conflicts, so best of both evils - ) - stop = self.stop if self.stop else [] if isinstance(stop, str): stop = [stop] @@ -433,7 +437,7 @@ def to_sampling_params( output_kind=( RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY ), - structured_outputs=structured_outputs, + structured_outputs=self.extract_structured_outputs(), logit_bias=self.logit_bias, extra_args=extra_args, skip_clone=True, # Created fresh per request, safe to skip clone