From dcc8d62732c8259307c7ecaf69e6a3173a0f0600 Mon Sep 17 00:00:00 2001 From: Yifan Zong Date: Thu, 16 Jul 2026 20:39:47 -0400 Subject: [PATCH 1/3] Rebase Signed-off-by: Yifan Zong --- .../openai/responses/test_harmony.py | 56 +-- tests/parser/test_harmony.py | 268 ++++++++++++++ .../reasoning/test_gptoss_reasoning_parser.py | 349 +----------------- .../test_structural_tag_registry.py | 4 +- vllm/entrypoints/openai/responses/serving.py | 13 +- vllm/parser/harmony.py | 201 +++++++++- vllm/reasoning/gptoss_reasoning_parser.py | 132 +------ vllm/renderers/online_renderer.py | 9 + vllm/tool_parsers/gptoss_tool_parser.py | 2 + vllm/tool_parsers/structural_tag_registry.py | 9 +- 10 files changed, 533 insertions(+), 510 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_harmony.py b/tests/entrypoints/openai/responses/test_harmony.py index 2c70b06d8129..574f33a8f22b 100644 --- a/tests/entrypoints/openai/responses/test_harmony.py +++ b/tests/entrypoints/openai/responses/test_harmony.py @@ -13,7 +13,7 @@ import pytest import pytest_asyncio import requests -from openai import InternalServerError, NotFoundError, OpenAI +from openai import NotFoundError, OpenAI from openai_harmony import Message from tests.utils import RemoteOpenAIServer @@ -368,8 +368,12 @@ async def test_streaming_types( @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) async def test_function_calling_with_streaming_types( - pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str + pairs_of_event_types: dict[str, str], + client: OpenAI, + model_name: str, + tool_choice: str, ): """Streaming event nesting for function-calling responses.""" @@ -382,6 +386,7 @@ def _has_function_events(evts: list) -> bool: validate_events=_has_function_events, input=[{"role": "user", "content": "What's the weather like in Paris today?"}], tools=[GET_WEATHER_SCHEMA], + tool_choice=tool_choice, temperature=0.0, ) @@ -558,7 +563,8 @@ async def test_reasoning_item(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_function_calling(client: OpenAI, model_name: str): +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +async def test_function_calling(client: OpenAI, model_name: str, tool_choice: str): tools = [GET_WEATHER_SCHEMA] response = await retry_for_tool_call( @@ -567,8 +573,9 @@ async def test_function_calling(client: OpenAI, model_name: str): expected_tool_type="function_call", input="What's the weather like in Paris today?", tools=tools, + tool_choice=tool_choice, temperature=0.0, - extra_body={"request_id": "test_function_calling_non_resp"}, + extra_body={"request_id": f"test_function_calling_non_resp_{tool_choice}"}, ) assert response.status == "completed" assert has_output_type(response, "function_call"), ( @@ -610,7 +617,10 @@ async def test_function_calling(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_function_calling_multi_turn(client: OpenAI, model_name: str): +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +async def test_function_calling_multi_turn( + client: OpenAI, model_name: str, tool_choice: str +): """Multi-tool, multi-turn function calling with retry at API level.""" tools = [ { @@ -635,6 +645,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str): expected_tool_type="function_call", input="Help me plan a trip to a random place. And tell me the weather there.", tools=tools, + tool_choice=tool_choice, temperature=0.0, ) assert response.status == "completed" @@ -659,6 +670,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str): } ], tools=tools, + tool_choice=tool_choice, previous_response_id=response.id, temperature=0.0, ) @@ -695,20 +707,6 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str): ) -@pytest.mark.asyncio -@pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_function_calling_required(client: OpenAI, model_name: str): - tools = [GET_WEATHER_SCHEMA] - - with pytest.raises(InternalServerError): - await client.responses.create( - model=model_name, - input="What's the weather like in Paris today?", - tools=tools, - tool_choice="required", - ) - - @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_system_message_with_tools(client: OpenAI, model_name: str): @@ -726,7 +724,10 @@ async def test_system_message_with_tools(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_function_calling_full_history(client: OpenAI, model_name: str): +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +async def test_function_calling_full_history( + client: OpenAI, model_name: str, tool_choice: str +): tools = [GET_WEATHER_SCHEMA] input_messages = [ @@ -739,6 +740,7 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str): expected_tool_type="function_call", input=input_messages, tools=tools, + tool_choice=tool_choice, temperature=0.0, ) assert response.status == "completed" @@ -772,7 +774,10 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_function_calling_with_stream(client: OpenAI, model_name: str): +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) +async def test_function_calling_with_stream( + client: OpenAI, model_name: str, tool_choice: str +): """Function calling via streaming, with retry for non-determinism.""" tools = [GET_WEATHER_SCHEMA] input_list = [ @@ -792,6 +797,7 @@ def _has_function_call(evts: list) -> bool: validate_events=_has_function_call, input=input_list, tools=tools, + tool_choice=tool_choice, temperature=0.0, ) @@ -853,8 +859,9 @@ def _has_function_call(evts: list) -> bool: @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) async def test_function_calling_no_code_interpreter_events( - client: OpenAI, model_name: str + client: OpenAI, model_name: str, tool_choice: str ): """Verify that function calls don't trigger code_interpreter events. @@ -880,6 +887,7 @@ def _has_function_call(evts: list) -> bool: validate_events=_has_function_call, input=input_list, tools=tools, + tool_choice=tool_choice, temperature=0.0, ) @@ -1048,8 +1056,9 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.parametrize("tool_choice", ["auto", "required"]) async def test_function_call_with_previous_input_messages( - client: OpenAI, model_name: str + client: OpenAI, model_name: str, tool_choice: str ): """Multi-turn function calling using previous_input_messages.""" tools = [ @@ -1074,6 +1083,7 @@ async def test_function_call_with_previous_input_messages( expected_tool_type="function_call", input="What is the horoscope for Aquarius today?", tools=tools, + tool_choice=tool_choice, temperature=0.0, extra_body={"enable_response_messages": True}, max_output_tokens=1000, diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 3e9cff64aa13..6ea832e3fbed 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -3,6 +3,7 @@ import json from collections.abc import Sequence +from typing import Any, Literal import pytest from openai_harmony import ( @@ -13,13 +14,17 @@ ) from transformers import AutoTokenizer +from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.engine.protocol import FunctionCall from vllm.entrypoints.openai.parser.harmony_utils import ( get_encoding, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.harmony import HarmonyParser from vllm.parser.parser_manager import ParserManager +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend REASONING_MODEL_NAME = "openai/gpt-oss-20b" @@ -857,3 +862,266 @@ def test_multi_boundary(self, harmony_parser): ("analysis", "One"), ("final", "Two"), ] + + +class TestAdjustRequest: + REQUEST_TEXT = "Hello" + TOOL_TYPE = "function" + TOOL_1_NAME = "get_user_location" + TOOL_2_NAME = "get_weather" + TOOLS = [ + { + "name": TOOL_1_NAME, + "parameters": { + "type": "object", + "properties": {}, + "required": [], + }, + }, + { + "name": TOOL_2_NAME, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + ] + OUTPUT_SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + } + + ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant" + COMMENTARY = ( + "<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant" + ) + TOOL_CALL_1 = ( + ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>" + "{}<|call|>" + ) + TOOL_CALL_2 = ( + ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>" + '{"city": "Tokyo"}<|call|>' + ) + FINAL_JSON_SCHEMA = ( + ANALYSIS + + '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>' + ) + FINAL_JSON_OBJECT = ( + ANALYSIS + + '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>' + ) + FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>Final only<|end|>" + + @pytest.fixture(scope="class") + def xgrammar_backend(self, gpt_oss_tokenizer): + return XgrammarBackend( + VllmConfig( + device_config=DeviceConfig(device="cpu"), + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar"), + ), + tokenizer=gpt_oss_tokenizer, + vocab_size=len(gpt_oss_tokenizer.get_vocab()), + ) + + @staticmethod + def _build_request( + request_kind: Literal["chat", "responses"], + tool_choice: str = "none", + strict_tools: bool = False, + response_format_type: str | None = None, + ) -> ChatCompletionRequest | ResponsesRequest: + data: dict[str, Any] = { + "model": REASONING_MODEL_NAME, + } + if request_kind == "chat": + data["messages"] = [ + { + "role": "user", + "content": TestAdjustRequest.REQUEST_TEXT, + } + ] + else: + data["input"] = TestAdjustRequest.REQUEST_TEXT + + if request_kind == "chat": + data["tools"] = [ + { + "type": TestAdjustRequest.TOOL_TYPE, + "function": {"strict": strict_tools, **tool_def}, + } + for tool_def in TestAdjustRequest.TOOLS + ] + data["tool_choice"] = ( + { + "type": TestAdjustRequest.TOOL_TYPE, + "function": {"name": TestAdjustRequest.TOOL_2_NAME}, + } + if tool_choice == "named" + else tool_choice + ) + else: + data["tools"] = [ + { + "type": TestAdjustRequest.TOOL_TYPE, + "strict": strict_tools, + **tool_def, + } + for tool_def in TestAdjustRequest.TOOLS + ] + data["tool_choice"] = ( + { + "type": TestAdjustRequest.TOOL_TYPE, + "name": TestAdjustRequest.TOOL_2_NAME, + } + if tool_choice == "named" + else tool_choice + ) + + if response_format_type == "json_schema": + schema_format = { + "name": "answer_format", + "schema": TestAdjustRequest.OUTPUT_SCHEMA, + "strict": True, + } + if request_kind == "chat": + data["response_format"] = { + "type": "json_schema", + "json_schema": schema_format, + } + else: + data["text"] = { + "format": { + "type": "json_schema", + **schema_format, + } + } + elif response_format_type == "json_object": + if request_kind == "chat": + data["response_format"] = {"type": "json_object"} + else: + data["text"] = {"format": {"type": "json_object"}} + + if request_kind == "chat": + return ChatCompletionRequest.model_validate(data) + return ResponsesRequest.model_validate(data) + + @classmethod + def _assert_structured_outputs_admission( + cls, + xgrammar_backend: XgrammarBackend, + adjusted_request: ChatCompletionRequest | ResponsesRequest, + **expected_admission: bool, + ) -> None: + structured_outputs = adjusted_request.structured_outputs + assert structured_outputs is not None + assert structured_outputs.structural_tag is not None + assert structured_outputs.all_non_structural_tag_constraints_none() + + grammar = xgrammar_backend.compile_grammar( + StructuredOutputOptions.STRUCTURAL_TAG, + structured_outputs.structural_tag, + ) + + for sample_name, should_admit in expected_admission.items(): + token_ids = xgrammar_backend.tokenizer.encode( + getattr(cls, sample_name), + add_special_tokens=False, + ) + admitted = grammar.validate_tokens(token_ids) == token_ids + assert admitted is should_admit, ( + f"Expected structured_outputs admission for {sample_name} " + f"to be {should_admit}, got {admitted}." + ) + + @pytest.mark.parametrize("request_kind", ["chat", "responses"]) + def test_structural_tag_auto_strict( + self, harmony_parser, xgrammar_backend, request_kind + ): + request = self._build_request( + request_kind, tool_choice="auto", strict_tools=True + ) + adjusted_request = harmony_parser.adjust_request(request) + assert adjusted_request.structured_outputs is not None + self._assert_structured_outputs_admission( + xgrammar_backend, + adjusted_request, + COMMENTARY=True, + TOOL_CALL_1=True, + TOOL_CALL_2=True, + FINAL_JSON_SCHEMA=True, + FINAL_JSON_OBJECT=True, + FINAL_TEXT_ONLY=True, + ) + + @pytest.mark.parametrize("request_kind", ["chat", "responses"]) + def test_structural_tag_required( + self, harmony_parser, xgrammar_backend, request_kind + ): + request = self._build_request(request_kind, tool_choice="required") + adjusted_request = harmony_parser.adjust_request(request) + assert adjusted_request.structured_outputs is not None + self._assert_structured_outputs_admission( + xgrammar_backend, + adjusted_request, + COMMENTARY=True, + TOOL_CALL_1=True, + TOOL_CALL_2=True, + FINAL_JSON_SCHEMA=False, + FINAL_JSON_OBJECT=False, + FINAL_TEXT_ONLY=False, + ) + + @pytest.mark.parametrize("request_kind", ["chat", "responses"]) + def test_structural_tag_named(self, harmony_parser, xgrammar_backend, request_kind): + request = self._build_request(request_kind, tool_choice="named") + adjusted_request = harmony_parser.adjust_request(request) + assert adjusted_request.structured_outputs is not None + self._assert_structured_outputs_admission( + xgrammar_backend, + adjusted_request, + COMMENTARY=True, + TOOL_CALL_1=False, + TOOL_CALL_2=True, + FINAL_JSON_SCHEMA=False, + FINAL_JSON_OBJECT=False, + FINAL_TEXT_ONLY=False, + ) + + @pytest.mark.parametrize("request_kind", ["chat", "responses"]) + def test_structural_tag_json_schema( + self, harmony_parser, xgrammar_backend, request_kind + ): + request = self._build_request(request_kind, response_format_type="json_schema") + adjusted_request = harmony_parser.adjust_request(request) + assert adjusted_request.structured_outputs is not None + self._assert_structured_outputs_admission( + xgrammar_backend, + adjusted_request, + COMMENTARY=False, + TOOL_CALL_1=False, + TOOL_CALL_2=False, + FINAL_JSON_SCHEMA=True, + FINAL_JSON_OBJECT=False, + FINAL_TEXT_ONLY=False, + ) + + @pytest.mark.parametrize("request_kind", ["chat", "responses"]) + def test_structural_tag_json_object( + self, harmony_parser, xgrammar_backend, request_kind + ): + request = self._build_request(request_kind, response_format_type="json_object") + adjusted_request = harmony_parser.adjust_request(request) + assert adjusted_request.structured_outputs is not None + self._assert_structured_outputs_admission( + xgrammar_backend, + adjusted_request, + COMMENTARY=False, + TOOL_CALL_1=False, + TOOL_CALL_2=False, + FINAL_JSON_SCHEMA=True, + FINAL_JSON_OBJECT=True, + FINAL_TEXT_ONLY=False, + ) diff --git a/tests/reasoning/test_gptoss_reasoning_parser.py b/tests/reasoning/test_gptoss_reasoning_parser.py index a6f815b6ae5c..e615bc7d73a5 100644 --- a/tests/reasoning/test_gptoss_reasoning_parser.py +++ b/tests/reasoning/test_gptoss_reasoning_parser.py @@ -1,351 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json from unittest.mock import Mock -import pytest -from transformers import AutoTokenizer +from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser -from vllm.entrypoints.mcp.tool_server import ToolServer -from vllm.reasoning import ReasoningParser -from vllm.reasoning.gptoss_reasoning_parser import ( - GptOssReasoningParser, - from_builtin_tool_to_tag, - no_func_reasoning_tag, -) -REASONING_MODEL_NAME = "openai/gpt-oss-120b" - - -@pytest.fixture(scope="module") -def gpt_oss_tokenizer(): - return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) - - -USER_MESSAGE_START = "<|start|>user<|message|>" -REASONING_SECTION_START = "<|end|><|start|>assistant<|channel|>analysis<|message|>" -END = "<|end|>" -ASSISTANT_START = "<|start|>assistant" -ASSISTANT_CONTENT_START_PREFIX = END + ASSISTANT_START + "<|channel|>final" -ASSISTANT_CONTENT_START_SUFFIX = "<|message|>" -ASSISTANT_CONTENT_START = ( - ASSISTANT_CONTENT_START_PREFIX + ASSISTANT_CONTENT_START_SUFFIX -) - -BASIC_CONTENT = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START - + "This is the rest", - "is_reasoning_end": True, -} - -BASIC_REASONING_ONLY = { - "output": REASONING_SECTION_START + "This is reasoning" + "<|end|>", - "is_reasoning_end": False, -} -BASIC_NO_REASONING_NO_ASSISTANT = { - "output": USER_MESSAGE_START + "This is a user message", - "is_reasoning_end": False, -} - -# Edge-case where the model omits the assistant tag entirely. -BASIC_NO_REASONING_ASSISTANT = { - "output": USER_MESSAGE_START + "This is a user message<|end|><|channel|>final", - "is_reasoning_end": True, -} - -COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_PREFIX, - "is_reasoning_end": False, -} - -COMPLEX_CONTENT_SUFFIX_ONLY = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_SUFFIX, - "is_reasoning_end": False, -} - -COMPLEX_CONTENT_1_NO_SUFFIX = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_PREFIX - + "<|constrain|> JSON ", - "is_reasoning_end": False, -} - -COMPLEX_CONTENT_1 = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_PREFIX - + "<|constrain|> JSON " - + ASSISTANT_CONTENT_START_SUFFIX, - "is_reasoning_end": True, -} - -COMPLEX_CONTENT_1_WITH_CONTENT = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_PREFIX - + "<|constrain|> JSON " - + ASSISTANT_CONTENT_START_SUFFIX - + "This is the rest", - "is_reasoning_end": True, -} - -COMPLEX_CONTENT_2 = { - "output": REASONING_SECTION_START - + "This is reasoning" - + ASSISTANT_CONTENT_START_PREFIX - + "<|constrain|>ReplyAction " - + ASSISTANT_CONTENT_START_SUFFIX - + "This is the rest", - "is_reasoning_end": True, -} - -MULTI_TURN_CONTENT = { - "output": USER_MESSAGE_START - + "1st turn user message" - + REASONING_SECTION_START - + "1st turn reasoning" - + ASSISTANT_CONTENT_START - + "1st turn response" - + END - + USER_MESSAGE_START - + "2nd turn user message" - + END - + ASSISTANT_START, - "is_reasoning_end": False, -} -TEST_CASES = [ - BASIC_CONTENT, - BASIC_REASONING_ONLY, - COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY, - COMPLEX_CONTENT_SUFFIX_ONLY, - COMPLEX_CONTENT_1_NO_SUFFIX, - COMPLEX_CONTENT_1, - COMPLEX_CONTENT_1_WITH_CONTENT, - COMPLEX_CONTENT_2, - MULTI_TURN_CONTENT, -] - - -@pytest.mark.parametrize( - "output, is_reasoning_end", - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], -) -def test_gptoss_is_reasoning_end( - output, - is_reasoning_end, - gpt_oss_tokenizer, -): - output = gpt_oss_tokenizer.tokenize(output) - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) - - # Test is_reasoning_end - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(output) - actual_is_reasoning_end = parser.is_reasoning_end(output_ids) - assert is_reasoning_end == actual_is_reasoning_end - - -class TestGptOssStructuralTags: - """Test cases for GptOssReasoningParser structural tag functionality.""" - - @pytest.fixture - def mock_tokenizer(self): - """Create a mock tokenizer for testing.""" - tokenizer = Mock() - tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5]) - tokenizer.get_vocab = Mock(return_value={"<|end|>": 6}) - return tokenizer - - @pytest.fixture - def reasoning_parser(self, mock_tokenizer): - """Create a GptOssReasoningParser instance.""" - return GptOssReasoningParser(mock_tokenizer) - - def test_prepare_structured_tag_no_tool_server(self, reasoning_parser): - """Test prepare_structured_tag with no tool server.""" - result = reasoning_parser.prepare_structured_tag(None, None) - expected = json.dumps(no_func_reasoning_tag) - - assert result == expected - - # Verify the structure is correct - parsed = json.loads(result) - assert parsed["type"] == "structural_tag" - assert parsed["format"]["type"] == "triggered_tags" - assert len(parsed["format"]["tags"]) == 1 - assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>" - assert parsed["format"]["triggers"] == ["<|channel|>analysis"] - - def test_prepare_structured_tag_with_original_tag(self, reasoning_parser): - """Test prepare_structured_tag when original_tag is provided.""" - original_tag = '{"custom": "tag"}' - result = reasoning_parser.prepare_structured_tag(original_tag, None) - - # Should return the original tag unchanged - assert result == original_tag - - def test_from_builtin_tool_to_tag(self): - """Test from_builtin_tool_to_tag function.""" - tags = from_builtin_tool_to_tag("python") - - assert len(tags) == 2 - assert tags[0]["begin"] == "<|channel|>commentary to=python" - assert tags[0]["content"]["type"] == "any_text" - assert tags[0]["end"] == "<|end|>" - - assert tags[1]["begin"] == "<|channel|>analysis to=python" - assert tags[1]["content"]["type"] == "any_text" - assert tags[1]["end"] == "<|end|>" - - @pytest.mark.parametrize( - "tools", - [ - [], - ["browser"], - ["python"], - ["container"], - ["browser", "python"], - ["browser", "container"], - ["python", "container"], - ["browser", "python", "container"], - ], - ) - def test_json_validity_comprehensive(self, reasoning_parser, tools): - """Test JSON validity across all possible tool combinations.""" - tool_server = Mock(spec=ToolServer) - tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools) - - result = reasoning_parser.prepare_structured_tag(None, tool_server) - parsed_result = json.loads(result) - - assert parsed_result["type"] == "structural_tag" - assert "format" in parsed_result - assert "tags" in parsed_result["format"] - assert "triggers" in parsed_result["format"] - - # Tag count should be: 1 (analysis) + 2 * len(tools) - expected_tag_count = 1 + (2 * len(tools)) - assert len(parsed_result["format"]["tags"]) == expected_tag_count - - # Verify triggers are correctly configured - expected_triggers = ["<|channel|>analysis"] - if tools: - expected_triggers.append("<|channel|>commentary to=") - assert set(parsed_result["format"]["triggers"]) == set(expected_triggers) - - def test_no_cross_request_state_pollution(self, reasoning_parser): - """Test that sequential calls with different tool servers produce - independent results, guarding against shared mutable state - (e.g. missing deepcopy in tag_with_builtin_funcs).""" - tool_server_1 = Mock(spec=ToolServer) - tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python") - - tool_server_2 = Mock(spec=ToolServer) - tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser") - - result_1 = reasoning_parser.prepare_structured_tag(None, tool_server_1) - result_2 = reasoning_parser.prepare_structured_tag(None, tool_server_2) - - tags_1 = [tag["begin"] for tag in json.loads(result_1)["format"]["tags"]] - tags_2 = [tag["begin"] for tag in json.loads(result_2)["format"]["tags"]] - - assert "<|channel|>commentary to=python" in tags_1 - assert "<|channel|>commentary to=browser" not in tags_1 - - assert "<|channel|>commentary to=browser" in tags_2 - assert "<|channel|>commentary to=python" not in tags_2 - - def test_tag_format_consistency(self, reasoning_parser): - """Test that all generated tags follow consistent format, - catching malformed tags from from_builtin_tool_to_tag.""" - tool_server = Mock(spec=ToolServer) - tool_server.has_tool = Mock( - side_effect=lambda tool: tool in ["python", "browser"] - ) - - result = reasoning_parser.prepare_structured_tag(None, tool_server) - parsed_result = json.loads(result) - - for tag in parsed_result["format"]["tags"]: - assert "begin" in tag - assert "content" in tag - assert "end" in tag - assert tag["content"]["type"] == "any_text" - assert tag["end"] == "<|end|>" - assert tag["begin"].startswith("<|channel|>") - - -@pytest.mark.parametrize( - "output, is_reasoning_end", - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], -) -def test_gptoss_is_reasoning_end_streaming( - output, - is_reasoning_end, - gpt_oss_tokenizer, -): - """Streaming override must agree with is_reasoning_end for all cases.""" - tokens = gpt_oss_tokenizer.tokenize(output) - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) - delta_ids = output_ids[-1:] if output_ids else [] - actual = parser.is_reasoning_end_streaming(output_ids, delta_ids) - assert is_reasoning_end == actual - - -@pytest.mark.parametrize( - "output, is_reasoning_end", - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], -) -def test_gptoss_is_reasoning_end_streaming_long_prefix( - output, - is_reasoning_end, - gpt_oss_tokenizer, -): - """Windowing must produce correct results even with a long prefix.""" - tokens = gpt_oss_tokenizer.tokenize(output) - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) - # Prepend 10k dummy reasoning tokens to simulate a long generation - long_prefix = [1] * 10_000 - padded_ids = long_prefix + list(output_ids) - delta_ids = output_ids[-1:] if output_ids else [] - actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids) - assert is_reasoning_end == actual - - -@pytest.mark.parametrize( - "output, is_reasoning_end", - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], -) -def test_gptoss_is_reasoning_end_streaming_large_delta( - output, - is_reasoning_end, - gpt_oss_tokenizer, -): - """Simulate speculative decoding where the entire test sequence arrives - as a single large delta appended after a long prefix. The window must - expand to cover delta_ids so the end pattern is never missed.""" - tokens = gpt_oss_tokenizer.tokenize(output) - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) - long_prefix = [1] * 10_000 - padded_ids = long_prefix + list(output_ids) - # delta_ids = the entire test sequence (as if accepted in one spec step) - delta_ids = list(output_ids) - actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids) - assert is_reasoning_end == actual - - -def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer): - """Verify the method is callable with the expected signature.""" - parser = GptOssReasoningParser(gpt_oss_tokenizer) - result = parser.is_reasoning_end_streaming([], []) - assert result is False +def test_gptoss_reasoning_ended_is_true(): + parser = GptOssReasoningParser(Mock()) + assert parser.is_reasoning_end([]) is True + assert parser.is_reasoning_end_streaming([], []) is True diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 58672cdb5de0..b84dfad0923c 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -33,7 +33,7 @@ SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS, - _get_function_parameters, + get_function_parameters, get_model_structural_tag, ) @@ -530,7 +530,7 @@ def test_get_function_parameters_relaxes_function_strict_false(): strict=False, ) - assert _get_function_parameters(function) is True + assert get_function_parameters(function) is True def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]: diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 598d9fe7a885..344ce0a209c1 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -749,11 +749,14 @@ def _make_request_with_harmony( request: ResponsesRequest, prev_response: ResponsesResponse | None, ): - if request.tool_choice not in ("auto", "none"): - raise NotImplementedError( - "Only 'auto' or 'none' tool_choice is supported " - "in response API with Harmony" - ) + if self.parser is not None: + # HarmonyParser doesn't need chat_template_kwargs + # TODO: Unify adjust_request() call with non-harmony branch + self.parser( + self.renderer.get_tokenizer(), + request.tools, + model_config=self.model_config, + ).adjust_request(request=request) arrival_time = time.time() messages = self._construct_input_messages_with_harmony(request, prev_response) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 1442246f139c..841f7051f433 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -10,9 +10,23 @@ from typing import TYPE_CHECKING, NamedTuple from openai_harmony import HarmonyError, Message, Role +from xgrammar import StructuralTag +from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam +from xgrammar.structural_tag import ( + AnyTextFormat, + ConstStringFormat, + Format, + JSONSchemaFormat, + OptionalFormat, + OrFormat, + SequenceFormat, + TagFormat, +) from vllm.entrypoints.chat_utils import make_tool_call_id -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) from vllm.entrypoints.openai.engine.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -28,7 +42,13 @@ from vllm.logger import init_logger from vllm.parser.abstract_parser import DelegatingParser from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser +from vllm.sampling_params import StructuredOutputsParams from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser +from vllm.tool_parsers.structural_tag_registry import ( + SimplifiedToolChoice, + get_function_parameters, + register_vllm_structural_tag, +) if TYPE_CHECKING: from openai_harmony import Message, StreamableParser @@ -346,6 +366,13 @@ def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult: reasoning_token_count=reasoning_token_count, ) + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + request = super().adjust_request(request) + request = _adjust_output_format(request) + return request + @staticmethod def _normalize_recipient(recipient: str | None) -> str | None: """Remove constrained formats misparsed into recipients by older Harmony.""" @@ -356,3 +383,175 @@ def _normalize_recipient(recipient: str | None) -> str | None: if constrain_index == -1: return recipient return recipient[:constrain_index].rstrip() or None + + +# Harmomy can parse either <|end|>, <|call|>, <|endoftext|>, or <|return|> +# <|return|> is represented as `""` since it's an xgrammar stop token +_END_TAG = ["<|end|>", "<|call|>", "<|endoftext|>", ""] +_FINAL_CONSTRAIN_BEGINS = [ + # "<|channel|>final json<|message|>", # disabled to trigger _normalize_recipient + "<|channel|>final <|constrain|>json<|message|>", +] +_TOOL_CALL_CHANNELS = [ + "<|channel|>commentary", + "<|channel|>analysis", + "<|channel|>final", +] +_FUNCTION_CALL_BEGINS = [ + "to=functions.{name} {channel} json<|message|>", + "to=functions.{name} {channel} <|constrain|>json<|message|>", + "{channel} to=functions.{name} json<|message|>", + "{channel} to=functions.{name} <|constrain|>json<|message|>", +] +_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"}) +_ANY_CONTENT = AnyTextFormat() + + +def _assemble_tag( + allow_analysis: bool, allow_commentary: bool, content: Format +) -> StructuralTag: + tags = [] + if allow_analysis: + analysis_tag = OptionalFormat( + content=SequenceFormat( + elements=[ + TagFormat( + begin="<|channel|>analysis<|message|>", + content=_ANY_CONTENT, + end="<|end|>", + ), + ConstStringFormat(value="<|start|>assistant"), + ] + ) + ) + tags.append(analysis_tag) + + if allow_commentary: + commentary_tag = OptionalFormat( + content=SequenceFormat( + elements=[ + TagFormat( + begin="<|channel|>commentary<|message|>", + content=_ANY_CONTENT, + end="<|end|>", + ), + ConstStringFormat(value="<|start|>assistant"), + ] + ) + ) + tags.append(commentary_tag) + + tags.append(content) + + return StructuralTag(format=SequenceFormat(elements=tags)) + + +@register_vllm_structural_tag("harmony") +def get_harmony_structural_tag( + tools: list[FunctionToolParam], + builtin_tools: list[BuiltinToolParam], + tool_choice: SimplifiedToolChoice, + reasoning: bool, +) -> StructuralTag: + # reasoning always enabled for Harmony + del reasoning + + if builtin_tools: + # Fallback for built-in tools + tags = [ + TagFormat( + begin="to=", + content=AnyTextFormat(excludes=["<|start|>"]), + end=_END_TAG, + ) + ] + tags.extend( + TagFormat( + begin=channel + " to=", + content=AnyTextFormat(excludes=["<|start|>", "<|channel|>"]), + end=_END_TAG, + ) + for channel in _TOOL_CALL_CHANNELS + ) + else: + tags = [ + TagFormat( + begin=pattern.format(name=tool.function.name, channel=channel), + content=JSONSchemaFormat( + json_schema=get_function_parameters(tool.function) + ), + end=_END_TAG, + ) + for tool in tools + for pattern in _FUNCTION_CALL_BEGINS + for channel in _TOOL_CALL_CHANNELS + ] + + if tool_choice == "auto": + tags.extend( + TagFormat(begin=begin, content=_JSON_CONTENT, end=_END_TAG) + for begin in _FINAL_CONSTRAIN_BEGINS + ) + tags.append( + TagFormat( + begin="<|channel|>final<|message|>", + content=_ANY_CONTENT, + end=_END_TAG, + ) + ) + + return _assemble_tag( + allow_analysis=True, allow_commentary=True, content=OrFormat(elements=tags) + ) + + +def _adjust_output_format( + request: ChatCompletionRequest | ResponsesRequest, +) -> ChatCompletionRequest | ResponsesRequest: + if isinstance(request, ResponsesRequest) and request.text is not None: + response_format = request.text.format + elif isinstance(request, ChatCompletionRequest): + response_format = request.response_format + else: + return request + + if response_format is None or response_format.type in ( + "text", + "structural_tag", + ): + return request + + if response_format.type == "json_object": + final_content = _JSON_CONTENT + elif response_format.type == "json_schema": + # Chat Completions nests the schema; Responses exposes `schema_`. + schema_wrapper = getattr(response_format, "json_schema", None) + if schema_wrapper is not None: + schema = getattr(schema_wrapper, "json_schema", None) + else: + schema = getattr(response_format, "schema_", None) + if schema is None: + return request + final_content = JSONSchemaFormat(json_schema=schema) + else: + return request + + structural_tag = _assemble_tag( + allow_analysis=True, + allow_commentary=False, + content=OrFormat( + elements=[ + TagFormat(begin=begin, content=final_content, end=_END_TAG) + for begin in _FINAL_CONSTRAIN_BEGINS + ] + ), + ) + + request.structured_outputs = StructuredOutputsParams( + structural_tag=json.dumps(structural_tag.model_dump()), + ) + if isinstance(request, ResponsesRequest): + request.text = None + else: + request.response_format = None + return request diff --git a/vllm/reasoning/gptoss_reasoning_parser.py b/vllm/reasoning/gptoss_reasoning_parser.py index d7bdca829126..b846768f4f35 100644 --- a/vllm/reasoning/gptoss_reasoning_parser.py +++ b/vllm/reasoning/gptoss_reasoning_parser.py @@ -1,65 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json from collections.abc import Iterable, Sequence from typing import TYPE_CHECKING from transformers import PreTrainedTokenizerBase -from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.engine.protocol import DeltaMessage -from vllm.logger import init_logger from vllm.reasoning import ReasoningParser if TYPE_CHECKING: from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -logger = init_logger(__name__) - -no_func_reasoning_tag = { - "type": "structural_tag", - "format": { - "type": "triggered_tags", - "tags": [ - { - "begin": "<|channel|>analysis<|message|>", - "content": {"type": "any_text"}, - "end": "<|end|>", - } - ], - "triggers": ["<|channel|>analysis"], - "stop_after_first": False, - }, -} - - -def from_builtin_tool_to_tag(tool: str) -> list[dict]: - tag = [ - { - "begin": f"<|channel|>commentary to={tool}", - "content": {"type": "any_text"}, - "end": "<|end|>", - }, - { - "begin": f"<|channel|>analysis to={tool}", - "content": {"type": "any_text"}, - "end": "<|end|>", - }, - ] - return tag - - -def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict: - import copy - - new_tag = copy.deepcopy(no_func_reasoning_tag) - new_tag["format"]["triggers"].append("<|channel|>commentary to=") - - for tool in builtin_tool_list: - new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool)) - return new_tag - class GptOssReasoningParser(ReasoningParser): """ @@ -71,64 +23,14 @@ class GptOssReasoningParser(ReasoningParser): def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) - # The model can output some special tokens between "final" and "<|message|>" - # So we need to look for both sequences to determine the end of reasoning. - self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode( - "<|channel|>final" - ) - self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>") - # We also need to check for the <|end|> token to avoid false positives from - # previous messages in multi-turn conversations. - self.eom_token_id = self.vocab["<|end|>"] - self.reasoning_max_num_between_tokens = 20 def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - end_token_ids_prefix = self.reasoning_end_token_ids_prefix - end_token_ids_suffix = self.reasoning_end_token_ids_suffix - assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty" - assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty" - # Check if the end sequence is present in the input_ids. - # We search from the end of input_ids to find the last match. - for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1): - if input_ids[i] == self.eom_token_id: - # We looped backwards far enough to find the end of a previous message, - # which means we have searched the entirety of the current message - # and can exit early without searching further back into prior - # messages of the conversation. - return False - if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix: - # We have found the prefix, now we look for the suffix after the prefix. - suffix_start = i + len(end_token_ids_prefix) - for j in range( - suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1 - ): - if j - suffix_start >= self.reasoning_max_num_between_tokens: - break - if ( - input_ids[j : j + len(end_token_ids_suffix)] - == end_token_ids_suffix - ): - return True - return False + return True def is_reasoning_end_streaming( self, input_ids: Sequence[int], delta_ids: Iterable[int] ) -> bool: - # The pattern window covers the end-of-reasoning marker itself. - # We add len(delta_ids) so that under speculative decoding (where - # a single step can accept many tokens) the entire accepted chunk - # is always inside the scan region. - delta_ids = tuple(delta_ids) - pattern_len = ( - len(self.reasoning_end_token_ids_prefix) - + self.reasoning_max_num_between_tokens - + len(self.reasoning_end_token_ids_suffix) - ) - window = pattern_len + len(delta_ids) - n = len(input_ids) - if n <= window: - return self.is_reasoning_end(input_ids) - return self.is_reasoning_end(input_ids[n - window :]) + return True def extract_content_ids(self, input_ids: list[int]) -> list[int]: raise NotImplementedError( @@ -159,33 +61,3 @@ def extract_reasoning( "GptOssReasoningParser only provides boundary detection. " "Use HarmonyParser for output parsing." ) - - # This function prepares the structural tag to format reasoning output - def prepare_structured_tag( - self, original_tag: str | None, tool_server: ToolServer | None - ) -> str | None: - if original_tag is None: - if tool_server is None: - return json.dumps(no_func_reasoning_tag) - else: - builtin_tool_list: list[str] = [] - if tool_server.has_tool("browser"): - builtin_tool_list.append("browser") - if tool_server.has_tool("python"): - builtin_tool_list.append("python") - if tool_server.has_tool("container"): - builtin_tool_list.append("container") - - if len(builtin_tool_list) > 0: - logger.info("Builtin_tool_list: %s", builtin_tool_list) - func_tag = json.dumps( - tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list) - ) - else: - logger.info("Builtin_tool_list is empty") - func_tag = json.dumps(no_func_reasoning_tag) - - return func_tag - else: - # There is potential risk for appending the tag to the original tag - return original_tag diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 995527654fe8..4d3b9911355a 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -201,6 +201,15 @@ async def render_chat( ) else: # For GPT-OSS. + if self.parser is not None: + # HarmonyParser doesn't need chat_template_kwargs + # TODO: Unify adjust_request() call with non-harmony branch + self.parser( + self.renderer.get_tokenizer(), + request.tools, + model_config=self.model_config, + ).adjust_request(request=request) + should_include_tools = tool_dicts is not None conversation, engine_inputs = self._make_request_with_harmony( request, should_include_tools diff --git a/vllm/tool_parsers/gptoss_tool_parser.py b/vllm/tool_parsers/gptoss_tool_parser.py index 6857e6bbe728..7321c2f049cb 100644 --- a/vllm/tool_parsers/gptoss_tool_parser.py +++ b/vllm/tool_parsers/gptoss_tool_parser.py @@ -22,6 +22,8 @@ class GptOssToolParser(ToolParser): capability declaration via HarmonyParser.tool_parser_cls. """ + structural_tag_model = "harmony" + def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None): super().__init__(tokenizer, tools) diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 3423fd304bd8..1f9a651ee46d 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -65,7 +65,6 @@ "qwen_3_5", "qwen_3_coder", "qwen_3", - "harmony", "deepseek_v3_2", "glm_4_7", "deepseek_v4", @@ -209,7 +208,7 @@ def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedTool return tool_ref -def _get_function_parameters(function) -> dict[str, Any] | bool: +def get_function_parameters(function) -> dict[str, Any] | bool: if getattr(function, "strict", None) is False: return True return function.parameters if function.parameters is not None else True @@ -230,7 +229,7 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]: TagFormat( begin=begin + tool.function.name + arguments_field_prefix, content=JSONSchemaFormat( - json_schema=_get_function_parameters(tool.function) + json_schema=get_function_parameters(tool.function) ), end=end, ) @@ -279,7 +278,7 @@ def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]: TagFormat( begin=f'\n', content=JSONSchemaFormat( - json_schema=_get_function_parameters(tool.function), + json_schema=get_function_parameters(tool.function), style="minimax_xml", ), end="\n", @@ -537,7 +536,7 @@ def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any: def _k3_call_tag(tool: FunctionToolParam) -> TagFormat: """One ``call`` tag: ``<|open|>call tool="N" index=""<|sep|> args``.""" function = tool.function - parameters = _get_function_parameters(function) + parameters = get_function_parameters(function) begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="' return TagFormat( begin=begin, From 8ec4081d670e1661be1d86d2a792a0364a9ca07e Mon Sep 17 00:00:00 2001 From: Yifan Zong Date: Tue, 21 Jul 2026 23:07:22 -0400 Subject: [PATCH 2/3] Wrap structured_outputs Signed-off-by: Yifan Zong --- tests/parser/test_harmony.py | 257 +++++++++++++++++++++++------------ vllm/parser/harmony.py | 114 ++++++++++------ 2 files changed, 243 insertions(+), 128 deletions(-) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 6ea832e3fbed..61b6f701cb0f 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -23,6 +23,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.harmony import HarmonyParser from vllm.parser.parser_manager import ParserManager +from vllm.sampling_params import StructuredOutputsParams from vllm.v1.structured_output.backend_types import StructuredOutputOptions from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend @@ -913,7 +914,23 @@ class TestAdjustRequest: ANALYSIS + '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>' ) - FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>Final only<|end|>" + FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>" + FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>" + FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>" + FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>" + FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>" + ADMISSION_SAMPLES = ( + "COMMENTARY", + "TOOL_CALL_1", + "TOOL_CALL_2", + "FINAL_JSON_SCHEMA", + "FINAL_JSON_OBJECT", + "FINAL_TEXT_ONLY", + "FINAL_REGEX", + "FINAL_CHOICE", + "FINAL_GRAMMAR", + "FINAL_STRUCTURAL_TAG", + ) @pytest.fixture(scope="class") def xgrammar_backend(self, gpt_oss_tokenizer): @@ -932,6 +949,7 @@ def _build_request( tool_choice: str = "none", strict_tools: bool = False, response_format_type: str | None = None, + structured_outputs: StructuredOutputsParams | None = None, ) -> ChatCompletionRequest | ResponsesRequest: data: dict[str, Any] = { "model": REASONING_MODEL_NAME, @@ -1004,16 +1022,33 @@ def _build_request( else: data["text"] = {"format": {"type": "json_object"}} + if structured_outputs is not None: + data["structured_outputs"] = structured_outputs + if request_kind == "chat": return ChatCompletionRequest.model_validate(data) return ResponsesRequest.model_validate(data) + @staticmethod + def _assert_format_cleared( + adjusted_request: ChatCompletionRequest | ResponsesRequest, + ) -> None: + if isinstance(adjusted_request, ResponsesRequest): + assert adjusted_request.text is None or adjusted_request.text.format is None + else: + assert adjusted_request.response_format is None + + structured_outputs = adjusted_request.structured_outputs + assert structured_outputs is not None + assert structured_outputs.structural_tag is not None + assert structured_outputs.all_non_structural_tag_constraints_none() + @classmethod def _assert_structured_outputs_admission( cls, xgrammar_backend: XgrammarBackend, adjusted_request: ChatCompletionRequest | ResponsesRequest, - **expected_admission: bool, + expected_admission: Sequence[str], ) -> None: structured_outputs = adjusted_request.structured_outputs assert structured_outputs is not None @@ -1024,104 +1059,156 @@ def _assert_structured_outputs_admission( StructuredOutputOptions.STRUCTURAL_TAG, structured_outputs.structural_tag, ) + expected_admission_set = set(expected_admission) - for sample_name, should_admit in expected_admission.items(): + for sample_name in cls.ADMISSION_SAMPLES: token_ids = xgrammar_backend.tokenizer.encode( getattr(cls, sample_name), add_special_tokens=False, ) admitted = grammar.validate_tokens(token_ids) == token_ids + should_admit = sample_name in expected_admission_set assert admitted is should_admit, ( f"Expected structured_outputs admission for {sample_name} " f"to be {should_admit}, got {admitted}." ) @pytest.mark.parametrize("request_kind", ["chat", "responses"]) - def test_structural_tag_auto_strict( - self, harmony_parser, xgrammar_backend, request_kind - ): - request = self._build_request( - request_kind, tool_choice="auto", strict_tools=True - ) - adjusted_request = harmony_parser.adjust_request(request) - assert adjusted_request.structured_outputs is not None - self._assert_structured_outputs_admission( - xgrammar_backend, - adjusted_request, - COMMENTARY=True, - TOOL_CALL_1=True, - TOOL_CALL_2=True, - FINAL_JSON_SCHEMA=True, - FINAL_JSON_OBJECT=True, - FINAL_TEXT_ONLY=True, - ) - - @pytest.mark.parametrize("request_kind", ["chat", "responses"]) - def test_structural_tag_required( - self, harmony_parser, xgrammar_backend, request_kind - ): - request = self._build_request(request_kind, tool_choice="required") - adjusted_request = harmony_parser.adjust_request(request) - assert adjusted_request.structured_outputs is not None - self._assert_structured_outputs_admission( - xgrammar_backend, - adjusted_request, - COMMENTARY=True, - TOOL_CALL_1=True, - TOOL_CALL_2=True, - FINAL_JSON_SCHEMA=False, - FINAL_JSON_OBJECT=False, - FINAL_TEXT_ONLY=False, - ) - - @pytest.mark.parametrize("request_kind", ["chat", "responses"]) - def test_structural_tag_named(self, harmony_parser, xgrammar_backend, request_kind): - request = self._build_request(request_kind, tool_choice="named") - adjusted_request = harmony_parser.adjust_request(request) - assert adjusted_request.structured_outputs is not None - self._assert_structured_outputs_admission( - xgrammar_backend, - adjusted_request, - COMMENTARY=True, - TOOL_CALL_1=False, - TOOL_CALL_2=True, - FINAL_JSON_SCHEMA=False, - FINAL_JSON_OBJECT=False, - FINAL_TEXT_ONLY=False, - ) - - @pytest.mark.parametrize("request_kind", ["chat", "responses"]) - def test_structural_tag_json_schema( - self, harmony_parser, xgrammar_backend, request_kind - ): - request = self._build_request(request_kind, response_format_type="json_schema") - adjusted_request = harmony_parser.adjust_request(request) - assert adjusted_request.structured_outputs is not None - self._assert_structured_outputs_admission( - xgrammar_backend, - adjusted_request, - COMMENTARY=False, - TOOL_CALL_1=False, - TOOL_CALL_2=False, - FINAL_JSON_SCHEMA=True, - FINAL_JSON_OBJECT=False, - FINAL_TEXT_ONLY=False, - ) - - @pytest.mark.parametrize("request_kind", ["chat", "responses"]) - def test_structural_tag_json_object( - self, harmony_parser, xgrammar_backend, request_kind + @pytest.mark.parametrize( + ("request_kwargs", "expected_admission"), + [ + ( + {"tool_choice": "auto", "strict_tools": True}, + [ + "COMMENTARY", + "TOOL_CALL_1", + "TOOL_CALL_2", + "FINAL_JSON_SCHEMA", + "FINAL_JSON_OBJECT", + "FINAL_TEXT_ONLY", + "FINAL_REGEX", + "FINAL_CHOICE", + "FINAL_GRAMMAR", + "FINAL_STRUCTURAL_TAG", + ], + ), + ( + {"tool_choice": "required"}, + ["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"], + ), + ( + {"tool_choice": "named"}, + ["COMMENTARY", "TOOL_CALL_2"], + ), + ( + {"response_format_type": "json_schema"}, + ["FINAL_JSON_SCHEMA"], + ), + ( + {"response_format_type": "json_object"}, + ["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"], + ), + ( + {"structured_outputs": StructuredOutputsParams(json=OUTPUT_SCHEMA)}, + ["FINAL_JSON_SCHEMA"], + ), + ( + {"structured_outputs": StructuredOutputsParams(json_object=True)}, + ["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"], + ), + ( + {"structured_outputs": StructuredOutputsParams(regex=r"regex")}, + ["FINAL_REGEX"], + ), + ( + { + "structured_outputs": StructuredOutputsParams( + choice=["choice1", "choice2"] + ) + }, + ["FINAL_CHOICE"], + ), + ( + { + "structured_outputs": StructuredOutputsParams( + grammar='root ::= "grammar"' + ) + }, + ["FINAL_GRAMMAR"], + ), + ( + { + "structured_outputs": StructuredOutputsParams( + structural_tag=json.dumps( + { + "type": "structural_tag", + "format": { + "type": "json_schema", + "json_schema": OUTPUT_SCHEMA, + }, + } + ) + ) + }, + ["FINAL_JSON_SCHEMA"], + ), + ( + { + "structured_outputs": StructuredOutputsParams( + structural_tag=json.dumps( + { + "type": "structural_tag", + "structures": [ + { + "begin": "", + "schema": {"type": "object"}, + "end": "", + } + ], + "triggers": [""], + } + ) + ) + }, + [ + # Legacy triggered tags allow free text until a trigger, so + # unconstrained final-channel payloads are also admitted. + "FINAL_TEXT_ONLY", + "FINAL_REGEX", + "FINAL_CHOICE", + "FINAL_GRAMMAR", + "FINAL_STRUCTURAL_TAG", + ], + ), + ], + ids=[ + "tool_auto_strict", + "tool_required", + "tool_named", + "response_format_json_schema", + "response_format_json_object", + "structured_outputs_json", + "structured_outputs_json_object", + "structured_outputs_regex", + "structured_outputs_choice", + "structured_outputs_grammar", + "structured_outputs_structural_tag_modern", + "structured_outputs_structural_tag_legacy", + ], + ) + def test_adjust_request( + self, + harmony_parser, + xgrammar_backend, + request_kind, + request_kwargs, + expected_admission, ): - request = self._build_request(request_kind, response_format_type="json_object") + request = self._build_request(request_kind, **request_kwargs) adjusted_request = harmony_parser.adjust_request(request) - assert adjusted_request.structured_outputs is not None + self._assert_format_cleared(adjusted_request) self._assert_structured_outputs_admission( xgrammar_backend, adjusted_request, - COMMENTARY=False, - TOOL_CALL_1=False, - TOOL_CALL_2=False, - FINAL_JSON_SCHEMA=True, - FINAL_JSON_OBJECT=True, - FINAL_TEXT_ONLY=False, + expected_admission, ) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 841f7051f433..a71c52513d59 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -5,7 +5,7 @@ import json from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from enum import Enum, auto from typing import TYPE_CHECKING, NamedTuple @@ -16,11 +16,14 @@ AnyTextFormat, ConstStringFormat, Format, + GrammarFormat, JSONSchemaFormat, OptionalFormat, OrFormat, + RegexFormat, SequenceFormat, TagFormat, + TriggeredTagsFormat, ) from vllm.entrypoints.chat_utils import make_tool_call_id @@ -369,9 +372,8 @@ def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult: def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) request = _adjust_output_format(request) - return request + return super().adjust_request(request) @staticmethod def _normalize_recipient(recipient: str | None) -> str | None: @@ -385,13 +387,12 @@ def _normalize_recipient(recipient: str | None) -> str | None: return recipient[:constrain_index].rstrip() or None -# Harmomy can parse either <|end|>, <|call|>, <|endoftext|>, or <|return|> -# <|return|> is represented as `""` since it's an xgrammar stop token -_END_TAG = ["<|end|>", "<|call|>", "<|endoftext|>", ""] -_FINAL_CONSTRAIN_BEGINS = [ - # "<|channel|>final json<|message|>", # disabled to trigger _normalize_recipient - "<|channel|>final <|constrain|>json<|message|>", -] +# Harmomy's stop tokens are <|end|>, <|call|>, <|return|>, <|endoftext|> +# <|return|> is represented as "" since it's the default stop token, which xgrammar +# disallows under constraints, leading to bad or infinite generation. +# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded +_END_TAG = ["<|end|>", "<|call|>", ""] +_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>" _TOOL_CALL_CHANNELS = [ "<|channel|>commentary", "<|channel|>analysis", @@ -488,13 +489,16 @@ def get_harmony_structural_tag( ] if tool_choice == "auto": - tags.extend( - TagFormat(begin=begin, content=_JSON_CONTENT, end=_END_TAG) - for begin in _FINAL_CONSTRAIN_BEGINS + tags.append( + TagFormat( + begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"), + content=_ANY_CONTENT, + end=_END_TAG, + ) ) tags.append( TagFormat( - begin="<|channel|>final<|message|>", + begin=_FINAL_BEGIN.format(constrain=""), content=_ANY_CONTENT, end=_END_TAG, ) @@ -505,49 +509,73 @@ def get_harmony_structural_tag( ) +def _params_to_final_content(params: StructuredOutputsParams) -> Format | None: + """Map StructuredOutputsParams in a XGrammar Format.""" + if params.json_object: + return _JSON_CONTENT + if params.json is not None: + schema = params.json + if isinstance(schema, str): + schema = json.loads(schema) + return JSONSchemaFormat(json_schema=schema) + if params.regex is not None: + return RegexFormat(pattern=params.regex) + if params.choice is not None: + return OrFormat( + elements=[ConstStringFormat(value=choice) for choice in params.choice] + ) + if params.grammar is not None: + return GrammarFormat(grammar=params.grammar) + if params.structural_tag is not None: + s_tag = json.loads(params.structural_tag) + if "structures" in s_tag: + # LegacyStructuralTagResponseFormat + return TriggeredTagsFormat( + triggers=s_tag["triggers"], + tags=[ + TagFormat( + begin=structure["begin"], + content=JSONSchemaFormat(json_schema=structure["schema"]), + end=structure["end"], + ) + for structure in s_tag["structures"] + ], + ) + # StructuralTagResponseFormat + return StructuralTag.model_validate(s_tag).format + return None + + def _adjust_output_format( request: ChatCompletionRequest | ResponsesRequest, ) -> ChatCompletionRequest | ResponsesRequest: - if isinstance(request, ResponsesRequest) and request.text is not None: - response_format = request.text.format - elif isinstance(request, ChatCompletionRequest): - response_format = request.response_format - else: + """Canonicalize request constraints into a reasoning-aware StructuralTag.""" + params = request.extract_structured_outputs() + if params is None: return request - if response_format is None or response_format.type in ( - "text", - "structural_tag", - ): + final_content = _params_to_final_content(params) + if final_content is None: return request - if response_format.type == "json_object": - final_content = _JSON_CONTENT - elif response_format.type == "json_schema": - # Chat Completions nests the schema; Responses exposes `schema_`. - schema_wrapper = getattr(response_format, "json_schema", None) - if schema_wrapper is not None: - schema = getattr(schema_wrapper, "json_schema", None) - else: - schema = getattr(response_format, "schema_", None) - if schema is None: - return request - final_content = JSONSchemaFormat(json_schema=schema) + if isinstance(final_content, JSONSchemaFormat): + begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json") else: - return request + begin = _FINAL_BEGIN.format(constrain="") structural_tag = _assemble_tag( allow_analysis=True, allow_commentary=False, - content=OrFormat( - elements=[ - TagFormat(begin=begin, content=final_content, end=_END_TAG) - for begin in _FINAL_CONSTRAIN_BEGINS - ] - ), + content=TagFormat(begin=begin, content=final_content, end=_END_TAG), ) - request.structured_outputs = StructuredOutputsParams( + request.structured_outputs = replace( + params, + json=None, + regex=None, + choice=None, + grammar=None, + json_object=None, structural_tag=json.dumps(structural_tag.model_dump()), ) if isinstance(request, ResponsesRequest): From c4181af7044c703387d418868478e520561a5950 Mon Sep 17 00:00:00 2001 From: Yifan Zong Date: Fri, 31 Jul 2026 21:36:31 -0400 Subject: [PATCH 3/3] Use _is_grammar_accept_string for tests Signed-off-by: Yifan Zong --- tests/parser/test_harmony.py | 30 ++++++------------------------ vllm/parser/harmony.py | 3 ++- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 61b6f701cb0f..028bbed012dc 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -13,8 +13,9 @@ Role, ) from transformers import AutoTokenizer +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string -from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.engine.protocol import FunctionCall from vllm.entrypoints.openai.parser.harmony_utils import ( @@ -24,8 +25,6 @@ from vllm.parser.harmony import HarmonyParser from vllm.parser.parser_manager import ParserManager from vllm.sampling_params import StructuredOutputsParams -from vllm.v1.structured_output.backend_types import StructuredOutputOptions -from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend REASONING_MODEL_NAME = "openai/gpt-oss-20b" @@ -932,17 +931,6 @@ class TestAdjustRequest: "FINAL_STRUCTURAL_TAG", ) - @pytest.fixture(scope="class") - def xgrammar_backend(self, gpt_oss_tokenizer): - return XgrammarBackend( - VllmConfig( - device_config=DeviceConfig(device="cpu"), - structured_outputs_config=StructuredOutputsConfig(backend="xgrammar"), - ), - tokenizer=gpt_oss_tokenizer, - vocab_size=len(gpt_oss_tokenizer.get_vocab()), - ) - @staticmethod def _build_request( request_kind: Literal["chat", "responses"], @@ -1046,7 +1034,6 @@ def _assert_format_cleared( @classmethod def _assert_structured_outputs_admission( cls, - xgrammar_backend: XgrammarBackend, adjusted_request: ChatCompletionRequest | ResponsesRequest, expected_admission: Sequence[str], ) -> None: @@ -1055,18 +1042,15 @@ def _assert_structured_outputs_admission( assert structured_outputs.structural_tag is not None assert structured_outputs.all_non_structural_tag_constraints_none() - grammar = xgrammar_backend.compile_grammar( - StructuredOutputOptions.STRUCTURAL_TAG, - structured_outputs.structural_tag, - ) + grammar = Grammar.from_structural_tag(structured_outputs.structural_tag) expected_admission_set = set(expected_admission) for sample_name in cls.ADMISSION_SAMPLES: - token_ids = xgrammar_backend.tokenizer.encode( + admitted = _is_grammar_accept_string( + grammar, getattr(cls, sample_name), - add_special_tokens=False, + require_termination=False, ) - admitted = grammar.validate_tokens(token_ids) == token_ids should_admit = sample_name in expected_admission_set assert admitted is should_admit, ( f"Expected structured_outputs admission for {sample_name} " @@ -1199,7 +1183,6 @@ def _assert_structured_outputs_admission( def test_adjust_request( self, harmony_parser, - xgrammar_backend, request_kind, request_kwargs, expected_admission, @@ -1208,7 +1191,6 @@ def test_adjust_request( adjusted_request = harmony_parser.adjust_request(request) self._assert_format_cleared(adjusted_request) self._assert_structured_outputs_admission( - xgrammar_backend, adjusted_request, expected_admission, ) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index a71c52513d59..3cc493b65923 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -387,10 +387,11 @@ def _normalize_recipient(recipient: str | None) -> str | None: return recipient[:constrain_index].rstrip() or None -# Harmomy's stop tokens are <|end|>, <|call|>, <|return|>, <|endoftext|> +# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|> # <|return|> is represented as "" since it's the default stop token, which xgrammar # disallows under constraints, leading to bad or infinite generation. # StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded +# TODO: Remove <|call|> once #50595 lands. _END_TAG = ["<|end|>", "<|call|>", ""] _FINAL_BEGIN = "<|channel|>final{constrain}<|message|>" _TOOL_CALL_CHANNELS = [