From 6836b3c7709b3f939e1c275c9ed04da5fd7bc0b2 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Mon, 10 Aug 2026 15:53:55 -0700 Subject: [PATCH 01/15] Add ToolCallConfig and header-inspection mode for Harmony tool calling --- .../generative/toolcalling/tool_call_config.h | 66 +++++++ .../tool_call_stream_accumulator.h | 178 +++++++++++++++++- 2 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h new file mode 100644 index 000000000..c08e847f8 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +namespace fl { + +/// Data-driven configuration for tool-call parsing. +/// All format-specific knowledge is captured as data properties — no model-specific +/// code branches. A single accumulator interprets the config at runtime. +struct ToolCallConfig { + enum class Mode { + kSimple, // ChatML: bot → JSON body (has name+args) → eot + kHeaderInspection // Harmony: bot → header → message_token → args JSON → end_token + }; + + Mode mode = Mode::kSimple; + + // --- Header-inspection mode properties --- + + /// Token that separates the header from the message body (e.g., "<|message|>") + std::string message_token; + + /// Regex to match the header and extract the function name. + /// Capture group 1 = function name (e.g., "to=functions\\.(.+)" captures "get_weather") + std::string header_regex; + + /// Token between header routing and channel info (e.g., "<|channel|>") + /// Used to trim the header before regex matching. + std::string channel_token; + + /// All tokens that terminate a tool-call block. + /// For Harmony: both <|end|> (intermediate) and <|call|> (final/EOS) terminate. + /// For Simple mode: this is just {eot_marker} (populated from context). + std::vector end_tokens; + + // --- Factory methods --- + + /// Default config for ChatML-style models (Phi, Qwen). + /// Markers come from ToolCallContext; this just sets mode = kSimple. + static ToolCallConfig Simple() { + return ToolCallConfig{Mode::kSimple}; + } + + /// Config for GPT-OSS (Harmony) models. + static ToolCallConfig Harmony() { + ToolCallConfig cfg; + cfg.mode = Mode::kHeaderInspection; + cfg.message_token = "<|message|>"; + cfg.header_regex = R"(to=functions\.(.+))"; + cfg.channel_token = "<|channel|>"; + cfg.end_tokens = {"<|end|>", "<|call|>"}; + return cfg; + } + + /// Infer config from model type string (from genai_config.json "model.type"). + static ToolCallConfig FromModelType(const std::string& model_type) { + if (model_type == "gptoss") return Harmony(); + return Simple(); + } +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h index d8233ab15..b0b6a4c35 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h @@ -2,9 +2,11 @@ // Licensed under the MIT License. #pragma once +#include "inferencing/generative/toolcalling/tool_call_config.h" #include "inferencing/generative/toolcalling/tool_call_utils.h" #include +#include #include #include #include @@ -48,6 +50,13 @@ class ToolCallStreamAccumulator { ToolCallStreamAccumulator(std::string start_marker, std::string end_marker) : start_marker_(std::move(start_marker)), end_marker_(std::move(end_marker)) {} + /// Construct with a ToolCallConfig for header-inspection (Harmony) or simple (ChatML) mode. + /// In header-inspection mode, start_marker is the bot marker and end_tokens come from config. + ToolCallStreamAccumulator(std::string start_marker, std::string end_marker, ToolCallConfig config) + : start_marker_(std::move(start_marker)), + end_marker_(std::move(end_marker)), + config_(std::move(config)) {} + /// Feed a chunk into the accumulator. Returns visible text and any tool calls completed by this chunk. Output Push(const std::string& chunk) { Output out; @@ -63,7 +72,12 @@ class ToolCallStreamAccumulator { } buffer_ += chunk; - Drain(out, /*flushing=*/false); + + if (config_.mode == ToolCallConfig::Mode::kHeaderInspection) { + DrainHeaderInspection(out, /*flushing=*/false); + } else { + Drain(out, /*flushing=*/false); + } return out; } @@ -77,7 +91,11 @@ class ToolCallStreamAccumulator { return out; } - Drain(out, /*flushing=*/true); + if (config_.mode == ToolCallConfig::Mode::kHeaderInspection) { + DrainHeaderInspection(out, /*flushing=*/true); + } else { + Drain(out, /*flushing=*/true); + } return out; } @@ -180,6 +198,162 @@ class ToolCallStreamAccumulator { std::string buffer_; // pending bytes from Push() that haven't yet been routed std::string tool_call_buffer_; // accumulated bytes of the in-progress tool-call block (incl. start marker) bool inside_tool_call_ = false; + + ToolCallConfig config_; // default = Simple mode + + // --- Header-inspection mode state --- + enum class HState { Idle, InHeader, InBody }; + HState hstate_ = HState::Idle; + std::string header_buffer_; // header text between bot marker and message_token + std::string body_buffer_; // body text between message_token and end_token + std::string extracted_name_; // function name extracted from header regex + + /// Header-inspection state machine for Harmony-style models. + /// States: Idle → InHeader → InBody → (emit) → Idle + void DrainHeaderInspection(Output& out, bool flushing) { + while (true) { + switch (hstate_) { + case HState::Idle: { + // Look for the start marker (bot token) + size_t found = buffer_.find(start_marker_); + if (found != std::string::npos) { + // Emit everything before the marker as visible text + if (found > 0) { + out.visible_text.append(buffer_, 0, found); + } + buffer_.erase(0, found + start_marker_.size()); + hstate_ = HState::InHeader; + header_buffer_.clear(); + continue; + } + + if (flushing) { + out.visible_text.append(buffer_); + buffer_.clear(); + return; + } + + // Hold back suffix that could grow into start_marker + size_t hold = LongestSuffixThatIsPrefixOf(buffer_, start_marker_); + size_t safe = buffer_.size() - hold; + if (safe > 0) { + out.visible_text.append(buffer_, 0, safe); + buffer_.erase(0, safe); + } + return; + } + + case HState::InHeader: { + // Accumulate until we see the message_token + size_t msg_pos = buffer_.find(config_.message_token); + if (msg_pos != std::string::npos) { + header_buffer_.append(buffer_, 0, msg_pos); + buffer_.erase(0, msg_pos + config_.message_token.size()); + + // Trim header at channel_token if present (e.g., "assistant to=functions.foo<|channel|>default") + std::string header_to_match = header_buffer_; + if (!config_.channel_token.empty()) { + size_t ch_pos = header_to_match.find(config_.channel_token); + if (ch_pos != std::string::npos) { + header_to_match = header_to_match.substr(0, ch_pos); + } + } + + // Try to extract function name from header + std::smatch match; + std::regex re(config_.header_regex); + if (std::regex_search(header_to_match, match, re) && match.size() > 1) { + // This is a tool call — extract name, transition to InBody + extracted_name_ = match[1].str(); + body_buffer_.clear(); + hstate_ = HState::InBody; + } else { + // Not a tool call — this is regular assistant text. + // Emit the header content as visible text and look for end token to return to Idle. + out.visible_text.append(header_buffer_); + out.visible_text.append(config_.message_token); + header_buffer_.clear(); + hstate_ = HState::Idle; + } + continue; + } + + if (flushing) { + // Unterminated header — emit as visible text + out.visible_text.append(start_marker_); + out.visible_text.append(header_buffer_); + out.visible_text.append(buffer_); + header_buffer_.clear(); + buffer_.clear(); + hstate_ = HState::Idle; + return; + } + + // Buffer everything — header not complete yet + header_buffer_.append(buffer_); + buffer_.clear(); + return; + } + + case HState::InBody: { + // Accumulate until we see any end token + size_t best_pos = std::string::npos; + size_t best_len = 0; + for (const auto& end_tok : config_.end_tokens) { + size_t pos = buffer_.find(end_tok); + if (pos != std::string::npos && (best_pos == std::string::npos || pos < best_pos)) { + best_pos = pos; + best_len = end_tok.size(); + } + } + + if (best_pos != std::string::npos) { + // Found end token — finalize the tool call + body_buffer_.append(buffer_, 0, best_pos); + buffer_.erase(0, best_pos + best_len); + + // Emit as a parsed tool call + ParsedToolCall call; + call.name = extracted_name_; + call.arguments = body_buffer_; + call.id = GenerateToolCallId(); + out.ready_calls.push_back(std::move(call)); + + body_buffer_.clear(); + extracted_name_.clear(); + hstate_ = HState::Idle; + continue; + } + + if (flushing) { + // Unterminated body — emit everything as visible text (not a valid tool call) + out.visible_text.append(start_marker_); + out.visible_text.append(header_buffer_); + out.visible_text.append(config_.message_token); + out.visible_text.append(body_buffer_); + out.visible_text.append(buffer_); + header_buffer_.clear(); + body_buffer_.clear(); + buffer_.clear(); + hstate_ = HState::Idle; + return; + } + + // Buffer body content, hold back suffix that could be an end token prefix + size_t hold = 0; + for (const auto& end_tok : config_.end_tokens) { + hold = std::max(hold, LongestSuffixThatIsPrefixOf(buffer_, end_tok)); + } + size_t safe = buffer_.size() - hold; + if (safe > 0) { + body_buffer_.append(buffer_, 0, safe); + buffer_.erase(0, safe); + } + return; + } + } + } + } }; } // namespace fl From 2cdc513636f177ce7df2ae8d22268dcfba41baa6 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Mon, 10 Aug 2026 16:23:14 -0700 Subject: [PATCH 02/15] Add ToolCallConfig, header-inspection mode, and Harmony unit tests --- .../generative/chat/chat_session.cc | 10 +- .../tool_call_stream_accumulator_test.cc | 107 ++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index eb76ef94d..055a51954 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -524,7 +524,10 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // tool-call markers configured, both marker strings are empty and the accumulator degrades to passthrough. // REASONING segments bypass the accumulator entirely — tool-call-shaped text inside ... is the // model's scratchpad and is not a real tool call. - ToolCallStreamAccumulator tool_accumulator(cached_tool_ctx_.tool_call_start, cached_tool_ctx_.tool_call_end); + auto tool_call_config = ToolCallConfig::FromModelType( + model_.GetGenAIConfig().model ? model_.GetGenAIConfig().model->type : ""); + ToolCallStreamAccumulator tool_accumulator(cached_tool_ctx_.tool_call_start, cached_tool_ctx_.tool_call_end, + std::move(tool_call_config)); // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream // deltas and the final response (OpenAI Chat Completions contract). Populated even when there is no streaming @@ -719,7 +722,10 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // qwen tokenizer happens to emit `` as a single special token. Tokenizers that split the marker across // multiple tokens (or chat templates that produce marker-shaped text gradually) would silently fail. The shared // accumulator buffers across tokens and is verified by unit tests. - ToolCallStreamAccumulator tool_accumulator(tool_ctx.tool_call_start, tool_ctx.tool_call_end); + auto tool_call_config = ToolCallConfig::FromModelType( + model_.GetGenAIConfig().model ? model_.GetGenAIConfig().model->type : ""); + ToolCallStreamAccumulator tool_accumulator(tool_ctx.tool_call_start, tool_ctx.tool_call_end, + std::move(tool_call_config)); // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream // deltas and the final ChatCompletionResponse (OpenAI Chat Completions contract). diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc index 7c607723f..2b3a2a0fb 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc @@ -236,3 +236,110 @@ TEST(ToolCallStreamAccumulatorTest, FalseStartPrefixReleasesAfterDisambiguation) EXPECT_EQ(out2.visible_text, ""); EXPECT_TRUE(out2.ready_calls.empty()); } + +// ======================================================================== +// Header-inspection mode (Harmony / GPT-OSS protocol) +// ======================================================================== + +TEST(ToolCallStreamAccumulatorTest, HarmonyBasicToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + auto out = acc.Push("<|start|>assistant to=functions.get_weather<|channel|>default<|message|>{\"city\": \"Seattle\"}<|call|>"); + EXPECT_EQ(out.visible_text, ""); + ASSERT_EQ(out.ready_calls.size(), 1u); + EXPECT_EQ(out.ready_calls[0].name, "get_weather"); + EXPECT_EQ(out.ready_calls[0].arguments, "{\"city\": \"Seattle\"}"); + EXPECT_FALSE(out.ready_calls[0].id.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyToolCallSplitAcrossChunks) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + std::vector chunks = { + "<|start|>", + "assistant to=functions.multiply", + "<|channel|>default", + "<|message|>", + "{\"a\": 7, \"b\": 6}", + "<|call|>"}; + + auto outs = RunChunks(acc, chunks); + std::string visible = CollectVisible(outs); + EXPECT_EQ(visible, ""); + + // Collect all ready_calls across all outputs + std::vector all_calls; + for (const auto& o : outs) { + for (const auto& c : o.ready_calls) { + all_calls.push_back(c); + } + } + ASSERT_EQ(all_calls.size(), 1u); + EXPECT_EQ(all_calls[0].name, "multiply"); + EXPECT_EQ(all_calls[0].arguments, "{\"a\": 7, \"b\": 6}"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyRegularTextNotToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + // Regular assistant text: header doesn't match "to=functions.X" pattern + auto out = acc.Push("<|start|>assistant<|channel|>default<|message|>Hello, how can I help?<|end|>"); + // The header doesn't match → emitted as visible text (header + message_token + body until we see end) + // Note: <|end|> is an end_token in config, so body terminates there + // But since header didn't match, it went back to Idle and this is all visible + EXPECT_FALSE(out.visible_text.empty()); + EXPECT_TRUE(out.ready_calls.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyParallelToolCalls) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + // Two tool calls: first ends with <|end|> (intermediate), second with <|call|> (final) + std::string input = + "<|start|>assistant to=functions.func1<|channel|>default<|message|>{\"x\": 1}<|end|>" + "<|start|>assistant to=functions.func2<|channel|>default<|message|>{\"x\": 2}<|call|>"; + + auto out = acc.Push(input); + EXPECT_EQ(out.visible_text, ""); + ASSERT_EQ(out.ready_calls.size(), 2u); + EXPECT_EQ(out.ready_calls[0].name, "func1"); + EXPECT_EQ(out.ready_calls[0].arguments, "{\"x\": 1}"); + EXPECT_EQ(out.ready_calls[1].name, "func2"); + EXPECT_EQ(out.ready_calls[1].arguments, "{\"x\": 2}"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyTextBeforeToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + auto out = acc.Push("Some preamble text<|start|>assistant to=functions.foo<|channel|>default<|message|>{}<|call|>"); + EXPECT_EQ(out.visible_text, "Some preamble text"); + ASSERT_EQ(out.ready_calls.size(), 1u); + EXPECT_EQ(out.ready_calls[0].name, "foo"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyFlushUnterminatedHeader) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + acc.Push("<|start|>assistant to=functions.bar"); + auto flush_out = acc.Flush(); + // Unterminated → surface as visible text + EXPECT_FALSE(flush_out.visible_text.empty()); + EXPECT_TRUE(flush_out.ready_calls.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyFlushUnterminatedBody) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + acc.Push("<|start|>assistant to=functions.bar<|channel|>default<|message|>{\"partial\": true"); + auto flush_out = acc.Flush(); + // Unterminated body → surface everything as visible text + EXPECT_FALSE(flush_out.visible_text.empty()); + EXPECT_TRUE(flush_out.ready_calls.empty()); +} From 7566cd3e21054ea0bbec014ae4be50bd277c4028 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 11 Aug 2026 08:38:09 -0700 Subject: [PATCH 03/15] Fix GCC -Werror=missing-field-initializers in ToolCallConfig::Simple() --- .../src/inferencing/generative/toolcalling/tool_call_config.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h index c08e847f8..bbd0fdf36 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h @@ -42,7 +42,9 @@ struct ToolCallConfig { /// Default config for ChatML-style models (Phi, Qwen). /// Markers come from ToolCallContext; this just sets mode = kSimple. static ToolCallConfig Simple() { - return ToolCallConfig{Mode::kSimple}; + ToolCallConfig cfg; + cfg.mode = Mode::kSimple; + return cfg; } /// Config for GPT-OSS (Harmony) models. From 0d0da5245de3103a93a9f7c1c03281ebe9725ccc Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Thu, 5 Nov 2026 23:20:57 +0000 Subject: [PATCH 04/15] Share byte-preserving tool batch admission and structural JSON grammar Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a563a06-04d5-47cf-9b20-af502d60b1eb --- .../generative/toolcalling/tool_call_plan.cc | 96 +++++++++++++------ .../generative/toolcalling/tool_call_plan.h | 9 +- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.cc index c703aed73..c9f5d1572 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.cc @@ -241,6 +241,37 @@ void CheckReferences(Json& document, const std::string& context) { WalkSchema(document, "", check); } +std::string JsonGrammar() { + // Whitespace is explicit at structural boundaries, never ignored inside string, escape or number terminals. + // Escaped surrogates must be paired; all other BMP escapes and raw Unicode scalar values are accepted. + return R"lark( +tool_json_value: tool_json_object | tool_json_array | TOOL_JSON_STRING | TOOL_JSON_NUMBER | "true" | "false" | "null" +tool_json_object: "{" tool_json_ws (tool_json_member (tool_json_comma tool_json_member)*)? tool_json_ws "}" +tool_json_member: TOOL_JSON_STRING tool_json_colon tool_json_value +tool_json_array: "[" tool_json_ws (tool_json_value (tool_json_comma tool_json_value)*)? tool_json_ws "]" +tool_json_comma: tool_json_ws "," tool_json_ws +tool_json_colon: tool_json_ws ":" tool_json_ws +tool_json_ws: TOOL_JSON_WS? +TOOL_JSON_WS: /[ \t\r\n]+/ +TOOL_JSON_STRING: /)lark" + R"regex("(?:[^"\\\x00-\x1f]|\\(?:["\\\/bfnrt]|u(?:)regex" + R"regex([0-9a-cA-Ce-fE-F][0-9a-fA-F]{3}|[dD][0-7][0-9a-fA-F]{2}|)regex" + R"regex([dD][89aAbB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2})))*")regex" + R"lark(/ +TOOL_JSON_NUMBER: /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/ +)lark"; +} + +void ValidateRootRule(std::string_view root_rule) { + const auto is_initial = [](char c) { return (c >= 'a' && c <= 'z') || c == '_'; }; + const auto is_remaining = [&](char c) { return is_initial(c) || (c >= '0' && c <= '9'); }; + if (root_rule.empty() || !is_initial(root_rule.front()) || root_rule.starts_with("tool_json_") || + !std::ranges::all_of(root_rule, is_remaining)) { + Fail(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "grammar root rule", + "expected a lowercase Lark identifier outside the reserved tool_json_ prefix"); + } +} + std::string BuildGrammar(const std::vector& names, bool parallel) { std::string grammar = parallel ? "start: tool_json_ws \"[\" tool_json_ws tool_json_call " @@ -252,31 +283,13 @@ std::string BuildGrammar(const std::vector& names, bool parallel) { if (i != 0) { grammar += " | "; } - - // The inner string is JSON data; the outer string quotes those exact bytes as a Lark literal. grammar += Json(Json(names[i]).dump()).dump(); } - - // Whitespace is explicit at structural boundaries, never ignored inside string, escape or number terminals. - // Escaped surrogates must be paired; all other BMP escapes and raw Unicode scalar values are accepted. grammar += R"lark( tool_json_call: "{" tool_json_ws "\"name\"" tool_json_colon tool_json_name tool_json_comma )lark" R"lark("\"parameters\"" tool_json_colon tool_json_object tool_json_ws "}" -tool_json_value: tool_json_object | tool_json_array | TOOL_JSON_STRING | TOOL_JSON_NUMBER | "true" | "false" | "null" -tool_json_object: "{" tool_json_ws (tool_json_member (tool_json_comma tool_json_member)*)? tool_json_ws "}" -tool_json_member: TOOL_JSON_STRING tool_json_colon tool_json_value -tool_json_array: "[" tool_json_ws (tool_json_value (tool_json_comma tool_json_value)*)? tool_json_ws "]" -tool_json_comma: tool_json_ws "," tool_json_ws -tool_json_colon: tool_json_ws ":" tool_json_ws -tool_json_ws: TOOL_JSON_WS? -TOOL_JSON_WS: /[ \t\r\n]+/ -TOOL_JSON_STRING: /)lark" - R"regex("(?:[^"\\\x00-\x1f]|\\(?:["\\\/bfnrt]|u(?:)regex" - R"regex([0-9a-cA-Ce-fE-F][0-9a-fA-F]{3}|[dD][0-7][0-9a-fA-F]{2}|)regex" - R"regex([dD][89aAbB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2})))*")regex" - R"lark(/ -TOOL_JSON_NUMBER: /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/ )lark"; + grammar += JsonGrammar(); return grammar; } @@ -391,17 +404,25 @@ std::shared_ptr ToolCallPlan::Create( } std::string ToolCallPlan::Grammar(std::string_view root_rule) const { - const auto is_initial = [](char c) { return (c >= 'a' && c <= 'z') || c == '_'; }; - const auto is_remaining = [&](char c) { return is_initial(c) || (c >= '0' && c <= '9'); }; - if (root_rule.empty() || !is_initial(root_rule.front()) || root_rule.starts_with("tool_json_") || - !std::ranges::all_of(root_rule, is_remaining)) { - Fail(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "grammar root rule", - "expected a lowercase Lark identifier outside the reserved tool_json_ prefix"); - } - + ValidateRootRule(root_rule); return std::string(root_rule) + impl_->grammar.substr(std::string_view("start").size()); } +std::string ToolCallPlan::ObjectGrammar(std::string_view root_rule) const { + ValidateRootRule(root_rule); + return std::string(root_rule) + ": tool_json_ws tool_json_object tool_json_ws\n" + JsonGrammar(); +} + +std::vector ToolCallPlan::AllowedNames() const { + std::vector names; + for (const auto& entry : impl_->validators) { + if (!impl_->forced_name || entry.first == *impl_->forced_name) { + names.push_back(entry.first); + } + } + return names; +} + std::vector ToolCallPlan::ParseCalls(std::string_view canonical_json) const { const auto batch = ParseJson(canonical_json, kMaxBatchBytes, FOUNDRY_LOCAL_ERROR_INTERNAL, "tool call batch"); if (!batch.is_array() || batch.empty() || (!impl_->parallel && batch.size() > 1)) { @@ -415,7 +436,6 @@ std::vector ToolCallPlan::ParseCalls(std::string_view canonical_ const auto envelope = batch[i].get(); ParsedToolCall call{"", envelope.name, envelope.parameters.dump()}; call.argument_source = call.arguments; - ValidateCall(call); calls.push_back(std::move(call)); } catch (const Exception&) { throw; @@ -424,6 +444,7 @@ std::vector ToolCallPlan::ParseCalls(std::string_view canonical_ } } + ValidateBatch(calls); for (auto& call : calls) { call.id = GenerateToolCallId(); } @@ -431,6 +452,25 @@ std::vector ToolCallPlan::ParseCalls(std::string_view canonical_ return calls; } +void ToolCallPlan::ValidateBatch(const std::vector& calls) const { + if (calls.empty() || (!impl_->parallel && calls.size() > 1)) { + Fail(FOUNDRY_LOCAL_ERROR_INTERNAL, "tool call batch", "expected non-empty batch within parallel call limit"); + } + size_t bytes = 0; + for (const auto& call : calls) { + const auto& source = call.argument_source.empty() ? call.arguments : call.argument_source; + if (call.name.size() > kMaxBatchBytes - bytes) { + Fail(FOUNDRY_LOCAL_ERROR_INTERNAL, "tool call batch", "batch exceeds byte limit"); + } + bytes += call.name.size(); + if (source.size() > kMaxBatchBytes - bytes) { + Fail(FOUNDRY_LOCAL_ERROR_INTERNAL, "tool call batch", "batch exceeds byte limit"); + } + bytes += source.size(); + ValidateCall(call); + } +} + void ToolCallPlan::ValidateCall(const ParsedToolCall& call) const { const auto context = "tool '" + call.name + "'"; const auto& source = call.argument_source.empty() ? call.arguments : call.argument_source; diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.h index cc158cf7b..5ebf38b0e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_plan.h @@ -14,14 +14,14 @@ namespace fl { -/// Immutable, offline authority for nonempty canonical [{"name": ..., "parameters": {...}}] tool-call batches. +/// Immutable, offline authority for nonempty tool-call batches. /// Supports Draft 7 and 2020-12 (default) with local JSON Pointer references. Resource IDs, anchors, /// recursive references, custom vocabularies, reserved controls and unsupported content features are rejected. /// Unknown non-reserved keywords and format are annotations; original schemas are preserved without format assertions. /// Draft 7 tuple items/dependencies are not supported. No defaults, coercion or repair are performed. /// Omitted function.parameters declares a parameterless tool accepting only {}; supplied schemas are never defaulted. /// Grammar is a structural mask, NOT a semantic schema mask. It constrains JSON transport, envelopes, names and count. -/// ParseCalls/ValidateCall enforce the original schemas before ANY call publication, without schema projection. +/// ParseCalls/ValidateBatch enforce the original schemas before ANY call publication, without schema projection. class ToolCallPlan final { public: // Four MiB accommodates source-code/custom-tool arguments; the batch cap bounds aggregate parsing memory. @@ -42,7 +42,12 @@ class ToolCallPlan final { /// Internal rules use tool_json_/TOOL_JSON_ prefixes for embedding. /// root_rule must be a lowercase Lark rule identifier outside the reserved tool_json_ prefix. std::string Grammar(std::string_view root_rule = "start") const; + /// Structural grammar for a single argument object, with the same JSON syntax and root-rule contract. + std::string ObjectGrammar(std::string_view root_rule = "start") const; + std::vector AllowedNames() const; std::vector ParseCalls(std::string_view canonical_json) const; + /// Validate an entire batch without changing IDs, argument bytes, or custom-tool argument sources. + void ValidateBatch(const std::vector& calls) const; void ValidateCall(const ParsedToolCall& call) const; private: From bec08f12e1f40eece817217198feb089f3c32770 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Sat, 7 Nov 2026 00:39:10 +0000 Subject: [PATCH 05/15] Add qualified native Harmony chat and tool calling Build on the original Harmony PR protocol-selection idea while keeping regular tool calling unchanged. Qualify native token/template contracts, decode channels and handoffs, protect transcript data, and reuse shared admission and recovery. Generation is sequential; coding-model qualification remains blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a563a06-04d5-47cf-9b20-af502d60b1eb --- .../cpp-tool-calling.instructions.md | 6 +- sdk_v2/cpp/CMakeLists.txt | 13 +- sdk_v2/cpp/README.md | 39 ++- .../contracts/chat_completions_converter.cc | 3 + sdk_v2/cpp/src/hosting/item_codec.cc | 2 +- .../generative/chat/chat_generator.h | 1 + .../generative/chat/chat_session.cc | 214 +++++++++++++--- .../generative/chat/chat_template.cc | 178 ++++++++++++- .../generative/chat/chat_template.h | 9 + .../generative/chat/chat_transcript.cc | 2 +- .../generative/chat/chat_transcript.h | 4 +- .../generative/chat/harmony_decoder.cc | 237 ++++++++++++++++++ .../generative/chat/harmony_decoder.h | 86 +++++++ .../generative/chat/harmony_protocol.h | 30 +++ .../generative/chat/onnx_chat_engine.cc | 3 + .../generative/chat/onnx_chat_engine.h | 1 + .../chat/onnx_engine_chat_stream.cc | 1 + .../generative/chat/prepared_chat_prompt.cc | 38 ++- .../generative/chat/prepared_chat_prompt.h | 2 + .../generative/chat/search_options.cc | 6 +- .../generative/genai_model_instance.cc | 92 ++++++- .../generative/genai_model_instance.h | 3 + .../openresponses/response_converter.cc | 1 + .../generative/toolcalling/grammar.cc | 70 ++++++ .../toolcalling/tool_call_context.h | 25 +- sdk_v2/cpp/src/inferencing/session/types.h | 2 + 26 files changed, 1011 insertions(+), 57 deletions(-) create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h diff --git a/.github/instructions/cpp-tool-calling.instructions.md b/.github/instructions/cpp-tool-calling.instructions.md index 5e43cbc33..0bbce03a9 100644 --- a/.github/instructions/cpp-tool-calling.instructions.md +++ b/.github/instructions/cpp-tool-calling.instructions.md @@ -13,9 +13,13 @@ applyTo: "sdk_v2/cpp/src/inferencing/generative/{chat,toolcalling}/**" Permit JSON whitespace at structural boundaries explicitly; never use global whitespace ignoring inside strings. - Reasoning and executable tool boundaries use published token IDs. Added tokens may have `special:false`; quoted marker spellings are data. Preserve payload ownership once a canonical or explicit raw envelope has opened. -- Automatic structured output must explicitly choose `MESSAGE\n` plus ordinary text or an executable tool boundary. +- Canonical automatic structured output must explicitly choose `MESSAGE\n` plus ordinary text or an executable tool boundary. Strip text framing incrementally and project it into assistant history. Use whole-turn guidance for automatic turns, including nonreasoning Engine models; region-only masks cannot enforce this initial mode selection. +- Qualified native Harmony uses assistant channel/recipient headers instead of canonical text framing or arrays. + Decode its structure by token identity, seed the prepared assistant-header continuation explicitly, and authorize + a call only at the actual sampled handoff. Reuse the same original-schema plan and publication/recovery sinks. + Terminal-token completion metadata is not a retained token and must never be added to the Engine sequence ledger. - Encode transcript/schema/argument data with the independent literal tokenizer. Only template-owned structure, runtime-owned protocol boundaries, and actual historical call boundaries use ordinary encoding. Encoding protocol boundary exemplars literally teaches the model to echo ordinary text instead of invoking tools. diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 8d795760e..7acb4c674 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -126,6 +126,15 @@ check_cxx_source_compiles( }" FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI ) +unset(FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN CACHE) +check_cxx_source_compiles( + "#include + int main() { + int32_t token = -1; + return OgaEngineEventGetTerminalTokenId(nullptr, &token) != nullptr; + }" + FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN +) unset(CMAKE_REQUIRED_INCLUDES) unset(CMAKE_REQUIRED_LIBRARIES) if(NOT FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI) @@ -262,6 +271,7 @@ set(FOUNDRY_LOCAL_SOURCES src/inferencing/session/session_manager.cc src/inferencing/session/tool_registry.cc src/inferencing/generative/chat/chat_session.cc + src/inferencing/generative/chat/harmony_decoder.cc src/inferencing/generative/chat/chat_template.cc src/inferencing/generative/chat/chat_transcript.cc src/inferencing/generative/chat/media_input.cc @@ -393,7 +403,8 @@ list(APPEND FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_PUBLIC_HEADERS}) function(foundry_local_configure_target TARGET LINK_SCOPE) target_compile_options(${TARGET} PRIVATE ${FOUNDRY_LOCAL_COMPILE_OPTIONS}) target_compile_definitions(${TARGET} ${LINK_SCOPE} - FOUNDRY_LOCAL_HAS_SHARED_HOST=$) + FOUNDRY_LOCAL_HAS_SHARED_HOST=$ + FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN=$) target_include_directories(${TARGET} PUBLIC diff --git a/sdk_v2/cpp/README.md b/sdk_v2/cpp/README.md index e45a77d2f..74f4a0177 100644 --- a/sdk_v2/cpp/README.md +++ b/sdk_v2/cpp/README.md @@ -313,7 +313,7 @@ The unchanged original parameter schema is validated with jsoncons before any call in the batch is published. Structural guidance alone does not guarantee schema compliance or that the model chose the user's intended argument value. -Automatic tool turns select their response mode explicitly: ordinary assistant +Canonical automatic tool turns select their response mode explicitly: ordinary assistant text starts with an internal `MESSAGE` line, while executable calls start with the model's dedicated tool boundary. The runtime removes the text framing line incrementally; callers still receive normal content and tool-call events. @@ -363,6 +363,43 @@ Recovery also shares implicit classic Generator output limits; it cannot acquire fresh default allowance or use the retained conversation's full context as an output budget. Engine text turns remain uncapped when the caller leaves their limit unset. +### Native Harmony + +GPT-OSS packages use their qualified native template and assistant channel/recipient +headers, not canonical `MESSAGE` framing or tool-call arrays. Model loading checks +the tokenizer's distinct control/literal encodings, EOS configuration, protected +argument/result slots, reasoning/commentary projection, and generation prefix. +The model must use the Engine backend. An incompatible template, tokenizer, or +runtime is rejected rather than falling back to string-based parsing. + +Native support requires matching GenAI headers and libraries exposing +`OgaEngineEventGetTerminalTokenId`, plus the matching CUDA add-on when applicable. +The CMake capability probe checks both compilation and linking. Existing pinned +packages without this API still build for canonical models but cannot load a +qualified GPT-OSS package. Sampled handoff and answer completion are metadata, +not extra retained sequence tokens. + +Each generated turn supports analysis, optional visible commentary, then a final +answer or one directed tool call. Calls are validated before publication; JSON +argument bytes are preserved. Custom tools with the admitted stock patch grammar +use native raw bodies; unframed text custom tools retain JSON input-object transport. +Both reuse the synthesized `{"input":"raw text"}` object for schema admission and +deliver unchanged text to callers. Recorded native body encodings survive tool +definition changes during history replay. Tool results are correlated by call ID +and rendered as named, protected JSON values. Source text and literal marker spellings remain data. +Original schemas and custom input formats are supplied as bounded literal prompt +information; schema admission remains owned by the existing `ToolCallPlan`. + +GPT-OSS generation is sequential even when the client permits parallel calls. +Native reasoning effort supports `low`, `medium`, and `high`; model-owned reasoning +cannot be disabled with `enable_thinking` or altered with `preserve_thinking`. +Top-level API effort overrides the generic thinking-switch mapping for this +protocol. Explicit whole-response guidance is incompatible with native Harmony. +Rejected selected calls use the shared one-retry policy with a whole-message +checkpoint and a legal, literal developer correction. + +### Local coding qualification + Qualification requires both protocol fidelity and the **real Copilot CLI** coding workflow, not just an offline harness self-test. With the server already running: diff --git a/sdk_v2/cpp/src/contracts/chat_completions_converter.cc b/sdk_v2/cpp/src/contracts/chat_completions_converter.cc index 67ee23946..e3c9bd826 100644 --- a/sdk_v2/cpp/src/contracts/chat_completions_converter.cc +++ b/sdk_v2/cpp/src/contracts/chat_completions_converter.cc @@ -313,6 +313,9 @@ void MapRequestParameters(const ChatCompletionRequest& req, Request& session_req session_request.options["chat_template_kwargs"] = ResolveReasoningTemplateKwargs(req.chat_template_kwargs, req.reasoning_effort).dump(); } + if (req.reasoning_effort.has_value()) { + session_request.options["reasoning_effort"] = *req.reasoning_effort; + } // Extract metadata extensions (matching C# GetTopK/GetRandomSeed) if (req.metadata.has_value()) { diff --git a/sdk_v2/cpp/src/hosting/item_codec.cc b/sdk_v2/cpp/src/hosting/item_codec.cc index 091234241..c29703863 100644 --- a/sdk_v2/cpp/src/hosting/item_codec.cc +++ b/sdk_v2/cpp/src/hosting/item_codec.cc @@ -410,7 +410,7 @@ std::unique_ptr Decode(const Json& value, const ItemDecodingContext& conte ? std::nullopt : std::optional(ReadEnum(declared_kind, 0, static_cast(ToolKind::kCustom))); call->generated_encoding = ReadEnum(value.at("encoding"), 0, - static_cast(GeneratedCallEncoding::kRawEnvelope)); + static_cast(GeneratedCallEncoding::kNativeJson)); call->replayed_from_store = value.at("replayed").get(); call->replayed_arguments = value.at("replayed_arguments").get(); const auto& replayed_kind = value.at("replayed_kind"); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h index 07518087a..6769d1bbe 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h @@ -36,6 +36,7 @@ struct ChatTurnUsage { std::optional finish_reason; std::optional termination_cause; int cached_prompt_tokens = 0; + std::optional terminal_token_id = std::nullopt; }; class RetainedPromptMismatchError : public std::runtime_error { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index a795f2ef5..d584aaaab 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -12,6 +12,7 @@ #include "inferencing/generative/chat/onnx_chat_generator.h" #include "inferencing/generative/chat/onnx_engine_chat_stream.h" #include "inferencing/generative/chat/prepared_chat_prompt.h" +#include "inferencing/generative/chat/harmony_decoder.h" #include "inferencing/generative/chat/reasoning_stream_splitter.h" #include "inferencing/generative/chat/stop_strings.h" #include "inferencing/generative/genai_model_instance.h" @@ -100,7 +101,9 @@ class AssistantTextOutputFilter { public: AssistantTextOutputFilter(const ToolCallContext& context, const SearchOptions& options) : framer_(context), - stop_filter_(context.UsesAssistantTextFraming() ? options.stop_sequences : std::vector{}) {} + stop_filter_(context.UsesAssistantTextFraming() || context.harmony + ? options.stop_sequences + : std::vector{}) {} std::string Push(std::string_view text) { return stop_filter_.Push(framer_.Push(text)); @@ -170,7 +173,7 @@ TranscriptMessage MakeAssistantMessage(const std::vector& const auto& parsed = std::get(event); auto generated = MakeGeneratedToolCall( parsed.id, parsed.name, parsed.arguments, tool_ctx.KindOf(parsed.name), - parsed.raw_envelope ? GeneratedCallEncoding::kRawEnvelope : GeneratedCallEncoding::kStructured); + tool_ctx.CallEncoding(parsed.name, parsed.raw_envelope)); if (!generated.arguments_usable) { logger.Log(LogLevel::Warning, @@ -847,6 +850,83 @@ std::vector ParseStrictGuidedToolCalls(const std::string& text, return {}; } +GuidedToolRetryResult RunHarmonyToolRetry(const chat_internal::PreparedChatMessages& messages, + const SearchOptions& options, GenAIModelInstance& model, + ToolCallContext context, const Request& request, + const TextChatGeneratorFactory& factory, const std::string& validation_error, + std::optional prompt) { + if (!prompt) prompt = PrepareTextChatPrompt(messages, model, context); + if (prompt->harmony_continuation) { + const auto& continuation = *prompt->harmony_continuation; + prompt->token_ids.resize(prompt->token_ids.size() - continuation.header_tokens); + prompt->prompt.resize(prompt->prompt.size() - continuation.header_bytes); + } + auto& preprocessor = model.GetPreprocessor(); + const auto append = [&](const std::string& text, bool literal) { + const auto encoded = literal ? preprocessor.EncodeLiteral(text.c_str()) : preprocessor.Encode(text.c_str()); + prompt->token_ids.insert(prompt->token_ids.end(), encoded->SequenceData(0), + encoded->SequenceData(0) + encoded->SequenceCount(0)); + prompt->prompt += text; + }; + // The checkpoint ends before the provisional assistant message, not inside its generated header. + append("<|start|>developer<|message|>", false); + const auto diagnostic = nlohmann::json(validation_error).dump(-1, ' ', true); + append("The previous unexecuted tool call was rejected: " + diagnostic.substr(0, 4096) + + ". Emit one corrected tool call satisfying the original schemas and requested values. " + "Do not repeat earlier reasoning or commentary.", + true); + append("<|end|><|start|>assistant", false); + prompt->prompt_token_count = static_cast(prompt->token_ids.size()); + prompt->harmony_continuation = HarmonyContinuation{ + "assistant", context.harmony->assistant_prefix_tokens.size(), + std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + context.supports_reasoning = false; + HarmonyDecoder decoder(*context.harmony, context.tool_plan, *prompt, context.NativeRawToolNames()); + auto generator = + CreatePreparedTextChatGenerator(messages, std::move(*prompt), options, model, context, false, factory); + GuidedToolRetryResult result; + std::string error; + const auto capture = [&](HarmonyDecoder::Output output) { + if (output.malformed) error = std::move(output.error); + for (auto& event : output.events) { + if (auto* call = std::get_if(&event)) { + ToolCallStreamAccumulator::Output admitted; + admitted.events.emplace_back(std::move(*call)); + chat_session_internal::NormalizeToolOutputBatch(admitted, context); + result.events.emplace_back(std::move(std::get(admitted.events.front()))); + } else { + error = "Guided Harmony recovery emitted text instead of one corrected tool call"; + } + } + }; + while (!generator->IsDone() && !request.IsCancellationRequested()) { + generator->GenerateNextToken(); + const auto token = generator->CurrentTokenId(); + capture(decoder.Push(token, generator->Decode())); + } + result.canceled = request.IsCancellationRequested(); + if (result.canceled) generator->Cancel(); + const auto usage = generator->GetTurnUsage(); + if (!usage) ThrowInvalidGuidedToolRetry("Harmony recovery did not report Engine usage"); + result.prompt_tokens = usage->prompt_tokens; + result.total_tokens = usage->prompt_tokens + usage->generated_tokens; + result.cached_prompt_tokens = usage->cached_prompt_tokens; + result.finish_reason = usage->finish_reason; + result.termination_cause = usage->termination_cause; + result.canceled |= request.IsCancellationRequested(); + if (result.canceled) { + result.events.clear(); + return result; + } + const bool natural = usage->termination_cause == BackendTerminationCause::kNaturalEnd; + capture(decoder.Finish(usage->terminal_token_id, natural)); + if (!natural || !error.empty() || result.events.size() != 1 || + (options.max_output_tokens && usage->generated_tokens > *options.max_output_tokens)) { + ThrowInvalidGuidedToolRetry(error.empty() ? "Incomplete or over-budget Harmony retry" : error); + } + return result; +} + GuidedToolRetryResult RunGuidedToolRetry(const chat_internal::PreparedChatMessages& messages, const SearchOptions& options, GenAIModelInstance& model, ToolCallContext tool_ctx, const Request& request, @@ -861,6 +941,10 @@ GuidedToolRetryResult RunGuidedToolRetry(const chat_internal::PreparedChatMessag tool_ctx.fenced_tool_call_output = false; tool_ctx.guidance_disabled = false; } + if (tool_ctx.harmony) { + return RunHarmonyToolRetry(messages, options, model, std::move(tool_ctx), request, factory, + validation_error, std::move(retry_prompt)); + } if (retry_prompt) { tool_ctx.supports_reasoning = false; } @@ -1087,6 +1171,7 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, GenAIModelInstance& model, ILogger& logger) { ToolCallContext tool_ctx; + tool_ctx.harmony = model.GetHarmonyProtocol(); tool_ctx.tool_call_start = GetOptionOrEmpty(effective_options, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); tool_ctx.tool_call_end = GetOptionOrEmpty(effective_options, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); @@ -1097,9 +1182,26 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "parallel_tool_calls must be true or false"); } tool_ctx.template_kwargs_json = GetOptionOrEmpty(effective_options, "chat_template_kwargs"); - if (!tool_ctx.template_kwargs_json.empty()) { + const auto api_reasoning_effort = GetOptionOrEmpty(effective_options, "reasoning_effort"); + if (!tool_ctx.template_kwargs_json.empty() || (tool_ctx.harmony && !api_reasoning_effort.empty())) { tool_ctx.template_kwargs_json = NormalizeChatTemplateKwargs(tool_ctx.template_kwargs_json); - const auto kwargs = nlohmann::json::parse(tool_ctx.template_kwargs_json); + auto kwargs = nlohmann::json::parse(tool_ctx.template_kwargs_json); + if (tool_ctx.harmony && !api_reasoning_effort.empty()) { + kwargs["reasoning_effort"] = api_reasoning_effort; + kwargs.erase("enable_thinking"); + } + if (tool_ctx.harmony && (kwargs.contains("enable_thinking") || kwargs.contains("preserve_thinking"))) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "native Harmony uses model-owned reasoning; enable_thinking and preserve_thinking are unsupported"); + } + if (tool_ctx.harmony) { + const auto effort = kwargs.find("reasoning_effort"); + if (effort != kwargs.end() && (!effort->is_string() || + (*effort != "low" && *effort != "medium" && *effort != "high"))) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "native Harmony supports low, medium, or high reasoning effort"); + } + tool_ctx.template_kwargs_json = kwargs.dump(); + } const bool uses_reasoning_controls = kwargs.contains("enable_thinking") || kwargs.contains("preserve_thinking") || kwargs.contains("reasoning_effort"); @@ -1228,10 +1330,11 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, ApplyToolChoiceToContext(tool_choice, tool_ctx); if (tool_ctx.tool_output) { - if (!tool_ctx.HasToolCallTokens() && !tool_ctx.ActiveRawEnvelope()) { + if (!tool_ctx.harmony && !tool_ctx.HasToolCallTokens() && !tool_ctx.ActiveRawEnvelope()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "structured tool calling requires model-published opening and closing tool markers"); } + if (tool_ctx.harmony) tool_ctx.parallel_tool_calls = false; tool_ctx.tool_plan = ToolCallPlan::Create(tool_ctx.tools_json, tool_ctx.parallel_tool_calls, tool_ctx.forced_tool); tool_ctx.eos_token_ids = model.GetPreprocessor().GetEosTokenIds(); @@ -1247,6 +1350,20 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, } } + if (tool_ctx.harmony) { + if (tool_ctx.HasAnyExplicitGuidance()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Explicit whole-response guidance is incompatible with native Harmony"); + } + tool_ctx.eos_token_ids = model.GetPreprocessor().GetEosTokenIds(); + tool_ctx.supports_reasoning = true; + tool_ctx.tool_call_start.clear(); + tool_ctx.tool_call_end.clear(); + tool_ctx.tool_call_start_token_id.reset(); + tool_ctx.tool_call_end_token_id.reset(); + tool_ctx.reasoning_start.clear(); + tool_ctx.reasoning_end.clear(); + } chat_session_internal::ApplyRawEnvelopeGuidance(tool_ctx, logger); return tool_ctx; @@ -1577,12 +1694,11 @@ void ChatSession::ProcessGeneratedOutput(std::vector event flush_segments(); auto& call = std::get(event); const auto kind = tool_ctx.KindOf(call.name); + const auto encoding = tool_ctx.CallEncoding(call.name, call.raw_envelope); response.items.push_back(std::make_unique(std::move(call.id), std::move(call.name), std::move(call.arguments), /*replayed_from_store=*/false, kind, std::nullopt, - call.raw_envelope - ? GeneratedCallEncoding::kRawEnvelope - : GeneratedCallEncoding::kStructured)); + encoding)); has_tool_calls = true; } flush_segments(); @@ -1632,6 +1748,11 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) auto& prepared_messages = *prepared->messages; ToolCallRecoveryCheckpoint recovery_checkpoint(prepared->prompt, turn_tool_ctx.tool_call_start_token_id, turn_tool_ctx.tool_plan != nullptr && !media_turn); + std::optional harmony_decoder; + if (turn_tool_ctx.harmony) { + harmony_decoder.emplace(*turn_tool_ctx.harmony, turn_tool_ctx.tool_plan, prepared->prompt, + turn_tool_ctx.NativeRawToolNames()); + } int prompt_tokens = 0; // Empty until this turn's input is appended to an existing generator. A rebuilt generator bakes the input into its @@ -1739,7 +1860,9 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int output_tokens = 0; StopStringFilter stop_filter(effective_options.stop_sequences); auto* active_stop_filter = - effective_options.stop_sequences.empty() || cached_tool_ctx_.UsesAssistantTextFraming() ? nullptr : &stop_filter; + effective_options.stop_sequences.empty() || cached_tool_ctx_.UsesAssistantTextFraming() || harmony_decoder + ? nullptr + : &stop_filter; bool stop_sequence_matched = false; bool host_output_limit_reached = false; std::vector generated_events; @@ -1830,7 +1953,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) streaming_callback->PushItem(std::make_unique( call.id, call.name, call.arguments, /*replayed_from_store=*/false, cached_tool_ctx_.KindOf(call.name), std::nullopt, - call.raw_envelope ? GeneratedCallEncoding::kRawEnvelope : GeneratedCallEncoding::kStructured)); + cached_tool_ctx_.CallEncoding(call.name, call.raw_envelope))); } generated_events.push_back(std::move(call)); } @@ -1890,13 +2013,16 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) recovery_checkpoint.Observe(token_id, token, splitter.InsideReasoning()); ++output_tokens; - if (chat_session_internal::PushDecodedFragment( - token, token_id, active_stop_filter, splitter, emit_segments, - cached_tool_ctx_.tool_plan && inside_payload) || - stop_sequence_matched) { + if (harmony_decoder) { + DeliverHarmonyOutput(harmony_decoder->Push(token_id, token), emit_segments, emit_tool_output); + } else if (chat_session_internal::PushDecodedFragment( + token, token_id, active_stop_filter, splitter, emit_segments, + cached_tool_ctx_.tool_plan && inside_payload) || + stop_sequence_matched) { stop_sequence_matched = true; break; } + if (stop_sequence_matched) break; if (host_max_output.has_value() && chat_session_internal::DidHostOutputLimitTruncate( @@ -1932,19 +2058,26 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int cached_prompt_tokens = 0; std::optional backend_finish_reason; std::optional backend_termination; + std::optional terminal_token; if (const auto turn_usage = cached_generator_->GetTurnUsage()) { prompt_tokens = turn_usage->prompt_tokens; total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; cached_prompt_tokens = turn_usage->cached_prompt_tokens; backend_finish_reason = turn_usage->finish_reason; backend_termination = turn_usage->termination_cause; + terminal_token = turn_usage->terminal_token_id; } const bool natural_tool_output_end = !turn_guard.TurnEnded() && chat_session_internal::IsNaturalToolOutputEnd( request.IsCancellationRequested(), stop_sequence_matched, host_output_limit_reached, backend_termination); - flush_accumulator(natural_tool_output_end); + if (harmony_decoder) { + DeliverHarmonyOutput( + harmony_decoder->Finish(terminal_token, natural_tool_output_end), emit_segments, emit_tool_output); + } else { + flush_accumulator(natural_tool_output_end); + } emit_visible_text(text_framer.Flush()); if (natural_tool_output_end && cached_tool_ctx_.tool_plan && !cached_tool_ctx_.text_output && !cached_tool_ctx_.ActiveRawEnvelope() && !malformed_tool_output_seen && @@ -1952,7 +2085,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) malformed_tool_output_seen = true; malformed_tool_error = "The model did not emit the required tool call"; } - int accepted_reasoning_tokens = splitter.ReasoningTokenCount(); + int accepted_reasoning_tokens = + harmony_decoder ? harmony_decoder->ReasoningTokenCount() : splitter.ReasoningTokenCount(); if (streaming_callback) { streaming_callback->DrainPending(); @@ -1969,7 +2103,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) bool recovered_tool_output = false; ToolCallContext completed_tool_ctx; if (malformed_tool_output_seen) { - const bool preserve_prefix = recovery_checkpoint.HasBoundary(); + const bool preserve_prefix = + harmony_decoder ? harmony_decoder->HasRecoveryCheckpoint() : recovery_checkpoint.HasBoundary(); const bool tool_call_published = HasGeneratedToolCalls(generated_events); const int attempted_tokens = std::max(output_tokens, total_tokens - prompt_tokens); const bool recovery_eligible = IsGuidedToolRetryEligible( @@ -1999,7 +2134,10 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) prepared_messages, retry_options, Model(), std::move(retry_tool_ctx), request, text_generator_factory_, malformed_tool_error, - preserve_prefix ? std::optional(recovery_checkpoint.TakePrompt()) : std::nullopt); + preserve_prefix ? std::optional( + harmony_decoder ? harmony_decoder->TakeRecoveryPrompt() + : recovery_checkpoint.TakePrompt()) + : std::nullopt); if (!preserve_prefix) { generated_events.clear(); @@ -2117,6 +2255,9 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons const auto& options = prepared.options; ToolCallRecoveryCheckpoint recovery_checkpoint(prepared.prompt, tool_ctx.tool_call_start_token_id, tool_ctx.tool_plan != nullptr); + std::optional harmony_decoder; + if (tool_ctx.harmony) + harmony_decoder.emplace(*tool_ctx.harmony, tool_ctx.tool_plan, prepared.prompt, tool_ctx.NativeRawToolNames()); const auto& prepared_messages = *prepared.messages; std::unique_ptr generator; if (prepared.prompt.HasMedia()) { @@ -2287,7 +2428,7 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons // Generate token-by-token. StopStringFilter stop_filter(options.stop_sequences); auto* active_stop_filter = - options.stop_sequences.empty() || tool_ctx.UsesAssistantTextFraming() ? nullptr : &stop_filter; + options.stop_sequences.empty() || tool_ctx.UsesAssistantTextFraming() || harmony_decoder ? nullptr : &stop_filter; while (!generator->IsDone() && !original_request.IsCancellationRequested() && !turn_guard.TurnEnded()) { generator->GenerateNextToken(); const auto token_id = generator->CurrentTokenId(); @@ -2296,13 +2437,16 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons splitter.InsideReasoning(), inside_payload); recovery_checkpoint.Observe(token_id, token, splitter.InsideReasoning()); - if (chat_session_internal::PushDecodedFragment( - token, token_id, active_stop_filter, splitter, process_segments, - tool_ctx.tool_plan && inside_payload) || - stop_sequence_matched) { + if (harmony_decoder) { + DeliverHarmonyOutput(harmony_decoder->Push(token_id, token), process_segments, process_tool_output); + } else if (chat_session_internal::PushDecodedFragment( + token, token_id, active_stop_filter, splitter, process_segments, + tool_ctx.tool_plan && inside_payload) || + stop_sequence_matched) { stop_sequence_matched = true; break; } + if (stop_sequence_matched) break; } // Drain any buffered partial-marker bytes at end-of-stream. Reasoning splitter first so any final DEFAULT bytes @@ -2329,28 +2473,36 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons int cached_prompt_tokens = 0; std::optional backend_finish_reason; std::optional backend_termination; + std::optional terminal_token; if (const auto turn_usage = generator->GetTurnUsage()) { prompt_tokens = turn_usage->prompt_tokens; total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; cached_prompt_tokens = turn_usage->cached_prompt_tokens; backend_finish_reason = turn_usage->finish_reason; backend_termination = turn_usage->termination_cause; + terminal_token = turn_usage->terminal_token_id; } const bool natural_tool_output_end = !turn_guard.TurnEnded() && chat_session_internal::IsNaturalToolOutputEnd( original_request.IsCancellationRequested(), stop_sequence_matched, /*host_output_limit_reached=*/false, backend_termination); - process_tool_output(chat_session_internal::FlushToolOutput( - active_raw_detector, tool_accumulator, natural_tool_output_end, - tool_ctx.fenced_tool_call_output)); + if (harmony_decoder) { + DeliverHarmonyOutput( + harmony_decoder->Finish(terminal_token, natural_tool_output_end), process_segments, process_tool_output); + } else { + process_tool_output( + chat_session_internal::FlushToolOutput(active_raw_detector, tool_accumulator, natural_tool_output_end, + tool_ctx.fenced_tool_call_output)); + } emit_visible_text(text_framer.Flush()); if (natural_tool_output_end && tool_ctx.tool_plan && !tool_ctx.text_output && !tool_ctx.ActiveRawEnvelope() && !malformed_tool_output_seen && !HasGeneratedToolCalls(generated_events)) { malformed_tool_output_seen = true; malformed_tool_error = "The model did not emit the required tool call"; } - int accepted_reasoning_tokens = splitter.ReasoningTokenCount(); + int accepted_reasoning_tokens = + harmony_decoder ? harmony_decoder->ReasoningTokenCount() : splitter.ReasoningTokenCount(); if (streaming_callback) { streaming_callback->DrainPending(); @@ -2365,7 +2517,8 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons text_framer.ValidateBoundary(); if (malformed_tool_output_seen) { - const bool preserve_prefix = recovery_checkpoint.HasBoundary(); + const bool preserve_prefix = + harmony_decoder ? harmony_decoder->HasRecoveryCheckpoint() : recovery_checkpoint.HasBoundary(); const bool tool_call_published = HasGeneratedToolCalls(generated_events); const auto attempted_tokens = total_tokens - prompt_tokens; const auto turn_output_budget = prepared.host_max_output_tokens @@ -2395,7 +2548,10 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons prepared_messages, retry_options, Model(), tool_ctx, original_request, text_generator_factory_, malformed_tool_error, - preserve_prefix ? std::optional(recovery_checkpoint.TakePrompt()) : std::nullopt); + preserve_prefix ? std::optional( + harmony_decoder ? harmony_decoder->TakeRecoveryPrompt() + : recovery_checkpoint.TakePrompt()) + : std::nullopt); if (!preserve_prefix) { generated_events.clear(); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index dacc15634..fe3970c87 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/prepared_chat_prompt.h" #include "exception.h" #include "inferencing/generative/genai_model_instance.h" #include "inferencing/generative/toolcalling/tool_call_context.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" #include "items/message_item.h" #include "items/text_item.h" #include "utils.h" @@ -296,8 +298,10 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& } if (const auto* raw = tool_ctx.ActiveRawEnvelope()) { const auto raw_instructions = - "\nThe custom tool \"" + raw->tool_name + "\" also accepts a raw envelope instead of a JSON call. " - "To call this tool in raw form, start with \"" + raw->start_marker + + "\nThe custom tool \"" + raw->tool_name + + "\" also accepts a raw envelope instead of a JSON call. " + "To call this tool in raw form, start with \"" + + raw->start_marker + "\" on its own line, then the exact tool payload, and finish with \"" + raw->end_marker + "\" on its own line. Include both marker lines in the raw input. " "Do not wrap the raw envelope in tool boundary tokens, JSON, or Markdown. " @@ -325,14 +329,175 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& return projected.dump(); } +HarmonyPromptProjection BuildHarmonyPromptProjection( + const std::vector& messages, const ToolCallContext& context, + const std::function& protect_literal) { + using Json = nlohmann::ordered_json; + auto projected = Json::array(); + std::unordered_map call_names; + for (const auto& message : messages) { + Json entry{{"role", Utils::RoleToString(message.role)}, {"content", protect_literal(message.VisibleText())}}; + if (message.role == FOUNDRY_LOCAL_ROLE_ASSISTANT) { + entry["thinking"] = protect_literal(message.ReasoningText()); + auto calls = Json::array(); + for (const auto* call : message.ToolCalls()) { + if (!call_names.emplace(call->call_id, call->name).second) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Duplicate Harmony history call ID"); + } + const auto raw_body = call->kind == ToolKind::kCustom && + (call->generated_encoding == GeneratedCallEncoding::kNativeRaw || + (call->generated_encoding == GeneratedCallEncoding::kStructured && + context.UsesNativeRawBody(call->name))); + const auto arguments = raw_body || (call->kind == ToolKind::kFunction && !call->arguments.empty()) + ? call->arguments + : call->normalized_arguments.dump(); + calls.push_back( + {{"id", call->call_id}, + {"type", "function"}, + {"function", + {{"name", protect_literal(call->name)}, + {"arguments", Json::object()}, + {"arguments_json", protect_literal(arguments)}, + {"native_freeform", raw_body}}}}); + } + + if (!calls.empty()) entry["tool_calls"] = std::move(calls); + } else if (message.role == FOUNDRY_LOCAL_ROLE_TOOL) { + const auto found = call_names.find(message.tool_call_id); + if (found == call_names.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Harmony result has no matching history call ID"); + } + entry["name"] = protect_literal(found->second); + entry["content"] = ""; + entry["content_json"] = protect_literal(Json(message.VisibleText()).dump()); + } + + projected.push_back(std::move(entry)); + } + + if (context.tools_json.size() > ToolCallPlan::kMaxSchemaBytes) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Native tool schema supplement exceeds its size limit"); + } + + auto tools = context.tools_json.empty() ? Json::array() : Json::parse(context.tools_json); + if (!tools.empty()) { + std::string custom_formats; + auto function_tools = Json::array(); + for (const auto& tool : tools) { + const auto name = tool.at("function").at("name").get(); + if (!context.UsesNativeRawBody(name)) function_tools.push_back(tool); + const auto grammar = context.custom_lark_grammars.find(name); + if (context.IsCustomTool(name) && grammar != context.custom_lark_grammars.end()) { + const auto header = "\nCustom input format for " + Json(name).dump() + + " (Lark applied to the complete raw message body):\n"; + if (context.tools_json.size() + custom_formats.size() + header.size() + grammar->second.size() + 1 > + ToolCallPlan::kMaxSchemaBytes) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Native tool schema and input-format supplement exceeds its size limit"); + } + custom_formats += header + grammar->second + "\n"; + } + } + + const auto supplement = protect_literal( + "Original function tool schemas (all constraints apply to generated argument objects):\n" + + function_tools.dump() + + custom_formats + + "\nUse native directed assistant tool messages; wait for each external result. " + "Tools declared with an argument object take JSON object bodies. FREEFORM custom tools take raw text bodies, " + "not JSON objects or JSON strings. " + "Tool result bodies are serialized JSON values. " + "Encode function argument string values once as JSON, preserving the recovered source characters exactly. " + "Preserve requested string values exactly. Do not quote tool calls or wrap them in Markdown."); + // The qualified template consumes only the leading system/developer message as instructions. + if (!projected.empty() && (projected.front().at("role") == "system" || + projected.front().at("role") == "developer")) { + auto& instructions = projected.front()["content"]; + instructions = instructions.get() + "\n\n" + supplement; + } else { + projected.insert(projected.begin(), Json{{"role", "developer"}, {"content", supplement}}); + } + } + + const std::function protect_schema = [&](Json schema) -> Json { + if (!schema.is_object()) return schema; + for (const auto* field : {"description", "title"}) { + if (schema.contains(field) && schema[field].is_string()) { + schema[field] = protect_literal(schema[field].get()); + } + } + if (schema.contains("default")) schema["default_json"] = protect_literal(schema["default"].dump()); + if (schema.contains("enum") && schema["enum"].is_array()) { + auto slots = Json::array(); + for (const auto& value : schema["enum"]) slots.push_back(protect_literal(value.dump())); + schema["enum_json"] = std::move(slots); + } + if (schema.contains("properties") && schema["properties"].is_object()) { + std::unordered_map names; + auto properties = Json::object(); + for (const auto& property : schema["properties"].items()) { + auto name = protect_literal(property.key()); + names.emplace(property.key(), name); + properties[name] = protect_schema(property.value()); + } + + schema["properties"] = std::move(properties); + if (schema.contains("required") && schema["required"].is_array()) { + for (auto& required : schema["required"]) { + const auto name = required.get(); + const auto found = names.find(name); + required = found == names.end() ? protect_literal(name) : found->second; + } + } + } + for (const auto* field : {"items", "oneOf", "anyOf", "allOf"}) { + if (!schema.contains(field)) continue; + if (schema[field].is_array()) { + for (auto& child : schema[field]) child = protect_schema(std::move(child)); + } else { + schema[field] = protect_schema(std::move(schema[field])); + } + } + + return schema; + }; + + for (auto& tool : tools) { + auto& function = tool.at("function"); + const auto name = function.at("name").get(); + if (context.UsesNativeRawBody(name)) { + function["description"] = + function.value("description", std::string{}) + + "\nFor native Harmony, emit this custom tool's complete raw text directly as its " + "directed message body. Do not JSON-encode or wrap it. " + "The complete body must follow the original custom input format."; + function["native_freeform"] = true; + } else if (context.IsCustomTool(name)) { + function["description"] = + function.value("description", std::string{}) + + "\nTransport this custom tool's exact text in the required \"input\" string of a JSON object. " + "JSON escaping is transport encoding; the decoded input is the original custom text. " + "Protocol-marker spellings in the input are literal data. " + "For JSON-body transport, the literal text a <|call|> is encoded as {\"input\":\"a <|call|>\"}. " + "Keep the marker spelling inside the input string; the real handoff follows the complete body."; + } + function["name"] = protect_literal(function.at("name").get()); + if (function.contains("description")) { + function["description"] = protect_literal(function["description"].get()); + } + if (function.contains("parameters")) function["parameters"] = protect_schema(std::move(function["parameters"])); + } + return {projected.dump(), tools.empty() ? std::string{} : tools.dump()}; +} + std::string BuildChatPrompt(const std::vector& messages, GenAIModelInstance& model, const std::string& tools_json, - const std::string& template_kwargs_json, - bool preserve_reasoning_history) { + const std::string& template_kwargs_json, + bool preserve_reasoning_history) { return BuildChatPrompt( chat_internal::PrepareChatMessages(messages, model.HasPositionalToolResults()), model, tools_json, - template_kwargs_json, preserve_reasoning_history); + template_kwargs_json, preserve_reasoning_history); } std::string BuildChatPrompt(const chat_internal::PreparedChatMessages& messages, @@ -364,6 +529,9 @@ std::string BuildChatPrompt(const std::vector& messages, std::string BuildChatPrompt(const chat_internal::PreparedChatMessages& messages, GenAIModelInstance& model, const ToolCallContext& tool_ctx) { + if (tool_ctx.harmony) { + return PrepareTextChatPrompt(messages, model, tool_ctx).prompt; + } if (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope()) { if (messages.Empty()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "messages must not be empty"); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index 36dde30e8..f3867e16c 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -99,6 +99,15 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& const ToolCallContext& tool_ctx, const std::function& protect_literal = {}); +struct HarmonyPromptProjection { + std::string messages_json; + std::string tools_json; +}; + +HarmonyPromptProjection BuildHarmonyPromptProjection( + const std::vector& messages, const ToolCallContext& context, + const std::function& protect_literal); + /// Build a chat prompt string from the transcript messages of a conversation. /// Uses the tokenizer's built-in chat template (via GenAIModelInstance::ApplyChatTemplate). /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc index 15d9bb9eb..8156f4d6a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc @@ -58,7 +58,7 @@ std::optional ParseToolCallArguments(const std::string& /// /// kFunction defers to ParseToolCallArguments and can fail. kCustom never fails: the payload is free-form text by /// definition, so it is wrapped as `{"input": }` — the exact shape the synthesized custom-tool schema -/// asks the model for, and therefore the shape a replayed call must render back to. Wrapping unconditionally keeps +/// uses for admission. Native raw-body history bypasses this object during rendering. Wrapping unconditionally keeps /// the rule unambiguous: a payload that happens to look like `{"input": "..."}` is still just text. static std::optional NormalizeToolCallArguments(ToolKind kind, const std::string& arguments) { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h index dbce3dac8..4dd74b919 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h @@ -30,8 +30,8 @@ struct TranscriptToolCall { std::string arguments; /// Object form used for template projection. Always a JSON object, so projecting a committed transcript can never /// fail. For kFunction this is the parsed `arguments`, or an empty object when model output failed to produce one. - /// For kCustom it is the payload rewrapped as `{"input": arguments}` — the exact shape the model emitted and the - /// only shape the chat template and the tool-call grammar understand. + /// For kCustom it is the payload rewrapped as `{"input": arguments}` for schema admission and JSON transport. + /// A native raw call instead replays `arguments` using its recorded encoding. nlohmann::ordered_json normalized_arguments = nlohmann::ordered_json::object(); /// Which of the session's tool kinds produced this call. Recorded so the authoritative transcript reports the call /// as what it was, and so a replayed call normalizes the same way the generated one did. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc new file mode 100644 index 000000000..2a58c788c --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "inferencing/generative/chat/harmony_decoder.h" + +#include "exception.h" +#include "inferencing/generative/toolcalling/tool_call_stream_accumulator.h" + +#include +#include +#include +#include + +namespace fl { +namespace { + +bool HasRecipientField(std::string_view header) { + size_t position = 0; + while ((position = header.find("to=", position)) != std::string_view::npos) { + if (position == 0 || std::string_view(" \t\r\n\x01\x02").find(header[position - 1]) != std::string_view::npos) { + return true; + } + position += 3; + } + return false; +} + +} // namespace + +HarmonyDecoder::HarmonyDecoder(HarmonyProtocol protocol, std::shared_ptr plan, + const PreparedChatPrompt& prompt, + std::unordered_set raw_tool_names) + : protocol_(std::move(protocol)), plan_(std::move(plan)), raw_tool_names_(std::move(raw_tool_names)) { + if (!prompt.harmony_continuation) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Harmony decoding requires an explicit prepared continuation"); + } + const auto& continuation = *prompt.harmony_continuation; + if (continuation.header_tokens > prompt.token_ids.size() || continuation.header_bytes > prompt.prompt.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Invalid prepared Harmony header extent"); + } + header_ = continuation.header; + checkpoint_.prompt = prompt.prompt; + checkpoint_.token_ids = prompt.token_ids; + message_token_offset_ = prompt.token_ids.size() - continuation.header_tokens; + message_byte_offset_ = prompt.prompt.size() - continuation.header_bytes; +} + +HarmonyDecoder::Output HarmonyDecoder::Fail(std::string error) { + failed_ = true; + arguments_.clear(); + return {{}, true, std::move(error)}; +} + +void HarmonyDecoder::CaptureCheckpoint() { + if (!checkpoint_captured_ && plan_) { + checkpoint_.token_ids.resize(message_token_offset_); + checkpoint_.prompt.resize(message_byte_offset_); + checkpoint_captured_ = true; + } +} + +void HarmonyDecoder::RecordToken(int32_t token, const std::string& text) { + if (checkpoint_captured_ || !plan_) { + return; + } + if (token == protocol_.start) { + message_token_offset_ = checkpoint_.token_ids.size(); + message_byte_offset_ = checkpoint_.prompt.size(); + } + checkpoint_.token_ids.push_back(token); + if (token == protocol_.start) + checkpoint_.prompt += "<|start|>"; + else if (token == protocol_.channel) + checkpoint_.prompt += "<|channel|>"; + else if (token == protocol_.message) + checkpoint_.prompt += "<|message|>"; + else if (token == protocol_.end) + checkpoint_.prompt += "<|end|>"; + else if (token == protocol_.format) + checkpoint_.prompt += "<|constrain|>"; + else + checkpoint_.prompt += text; +} + +void HarmonyDecoder::ParseHeader() { + if (HasRecipientField(header_)) { + CaptureCheckpoint(); + } + if (std::count(header_.begin(), header_.end(), '\x01') != 1 || + std::count(header_.begin(), header_.end(), '\x02') > 1) { + throw std::runtime_error("Harmony header requires exactly one channel"); + } + const auto channel_position = header_.find('\x01'); + const auto format_position = header_.find('\x02'); + if (format_position != std::string::npos && format_position < channel_position) { + throw std::runtime_error("Harmony format metadata must follow the channel"); + } + std::istringstream prefix(header_.substr(0, channel_position)); + std::string field; + if (!(prefix >> field) || field != "assistant") { + throw std::runtime_error("Harmony output participant must be assistant"); + } + channel_.clear(); + recipient_.clear(); + const auto read_recipient = [&](const std::string& value) { + if (!recipient_.empty() || !value.starts_with("to=functions.") || value.size() == 13) { + throw std::runtime_error("Invalid Harmony tool recipient"); + } + recipient_ = value.substr(13); + }; + while (prefix >> field) read_recipient(field); + std::istringstream channel(header_.substr(channel_position + 1, + format_position == std::string::npos + ? std::string::npos + : format_position - channel_position - 1)); + if (!(channel >> channel_) || + (channel_ != "analysis" && channel_ != "commentary" && channel_ != "final")) { + throw std::runtime_error("Unsupported Harmony channel"); + } + bool has_format = false; + while (channel >> field) { + if (field == "json" && !recipient_.empty() && format_position == std::string::npos) { + has_format = true; + if (channel >> field) throw std::runtime_error("Unexpected field after Harmony format metadata"); + } else { + read_recipient(field); + } + } + if (format_position != std::string::npos) { + has_format = true; + std::istringstream format(header_.substr(format_position + 1)); + if (!(format >> field) || field != "json" || (format >> field)) { + throw std::runtime_error("Unsupported Harmony format metadata"); + } + } + if ((!recipient_.empty() && channel_ != "commentary") || + (has_format && recipient_.empty()) || (channel_ == "analysis" && visible_started_)) { + throw std::runtime_error("Unsupported Harmony assistant-channel transition"); + } + if (has_format && raw_tool_names_.contains(recipient_)) { + throw std::runtime_error("Native custom tools require a raw body without JSON metadata"); + } + visible_started_ |= channel_ != "analysis"; +} + +HarmonyDecoder::Output HarmonyDecoder::Push(std::optional token, const std::string& text) { + if (failed_) return {}; + if (!token) return text.empty() ? Output{} : Fail("Harmony decoding requires token identity"); + if (phase_ == Phase::kDone) { + return token == ending_ ? Output{} : Fail("Output after Harmony turn completion"); + } + RecordToken(*token, text); + try { + if (*token == protocol_.start) { + if (phase_ != Phase::kBetween) return Fail("Harmony message started before the previous message ended"); + header_.clear(); + phase_ = Phase::kHeader; + } else if (*token == protocol_.channel || *token == protocol_.format) { + if (phase_ != Phase::kHeader) return Fail("Harmony header control in a message body"); + header_ += *token == protocol_.channel ? '\x01' : '\x02'; + } else if (*token == protocol_.message) { + if (phase_ != Phase::kHeader) return Fail("Unexpected Harmony message-body boundary"); + ParseHeader(); + phase_ = Phase::kBody; + } else if (*token == protocol_.end) { + if (phase_ != Phase::kBody || !recipient_.empty() || channel_ == "final") { + return Fail("Harmony message completion cannot authorize a tool call or final answer"); + } + phase_ = Phase::kBetween; + } else if (*token == protocol_.handoff) { + if (phase_ != Phase::kBody || recipient_.empty() || !plan_) { + return Fail("Harmony handoff requires a selected declared tool"); + } + ParsedToolCall call; + call.name = recipient_; + call.arguments = raw_tool_names_.contains(recipient_) + ? nlohmann::json{{"input", arguments_}}.dump() + : arguments_; + call.argument_source = call.arguments; + plan_->ValidateBatch({call}); + call.id = GenerateToolCallId(); + phase_ = Phase::kDone; + ending_ = token; + return {{std::move(call)}, false, {}}; + } else if (*token == protocol_.answer) { + if (phase_ != Phase::kBody || channel_ != "final" || !recipient_.empty()) { + return Fail("Harmony answer completion requires a final message"); + } + phase_ = Phase::kDone; + ending_ = token; + } else if (phase_ == Phase::kHeader) { + if (HasRecipientField(header_ + text.substr(0, 1025))) { + CaptureCheckpoint(); + } + if (text.find_first_of("\x01\x02") != std::string::npos) { + return Fail("Ordinary header bytes cannot manufacture Harmony metadata controls"); + } + if (header_.size() + text.size() > 1024) return Fail("Harmony header exceeds its size limit"); + header_ += text; + } else if (phase_ == Phase::kBody && !recipient_.empty()) { + if (arguments_.size() + text.size() > ToolCallPlan::kMaxArgumentBytes) { + return Fail("Harmony argument body exceeds its size limit"); + } + arguments_ += text; + } else if (phase_ == Phase::kBody) { + const auto type = channel_ == "analysis" ? FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING + : FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT; + reasoning_tokens_ += channel_ == "analysis"; + if (!text.empty()) return {{ReasoningStreamSplitter::Segment{text, type, token}}, false, {}}; + } else if (text.find_first_not_of(" \t\r\n") != std::string::npos) { + return Fail("Unexpected bytes between Harmony messages"); + } + } catch (const std::exception& error) { + return Fail(error.what()); + } + return {}; +} + +HarmonyDecoder::Output HarmonyDecoder::Finish(std::optional terminal_token, bool natural_end) { + if (failed_) return {}; + if (natural_end && terminal_token && + (*terminal_token == protocol_.handoff || *terminal_token == protocol_.answer)) { + return Push(terminal_token, {}); + } + if (phase_ == Phase::kDone) return {}; + if (!recipient_.empty() || natural_end) return Fail("Incomplete or interrupted Harmony turn"); + return {}; +} + +PreparedChatPrompt HarmonyDecoder::TakeRecoveryPrompt() { + if (!checkpoint_captured_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Harmony recovery has no selected-call checkpoint"); + } + checkpoint_.prompt_token_count = static_cast(checkpoint_.token_ids.size()); + return std::move(checkpoint_); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h new file mode 100644 index 000000000..949d65d5f --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "inferencing/generative/chat/harmony_protocol.h" +#include "inferencing/generative/chat/prepared_chat_prompt.h" +#include "inferencing/generative/chat/reasoning_stream_splitter.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" +#include "inferencing/generative/toolcalling/tool_call_stream_accumulator.h" + +#include +#include +#include +#include +#include +#include + +namespace fl { + +/// Decodes native message structure above payload admission. No session, retry policy, or publication ownership. +class HarmonyDecoder { + public: + using Event = std::variant; + struct Output { + std::vector events; + bool malformed = false; + std::string error; + }; + + HarmonyDecoder(HarmonyProtocol protocol, std::shared_ptr plan, + const PreparedChatPrompt& prompt, + std::unordered_set raw_tool_names = {}); + Output Push(std::optional token, const std::string& text); + Output Finish(std::optional terminal_token, bool natural_end); + bool HasRecoveryCheckpoint() const noexcept { return checkpoint_captured_; } + PreparedChatPrompt TakeRecoveryPrompt(); + int ReasoningTokenCount() const noexcept { return reasoning_tokens_; } + + private: + enum class Phase { kHeader, + kBody, + kBetween, + kDone }; + void ParseHeader(); + void RecordToken(int32_t token, const std::string& text); + void CaptureCheckpoint(); + Output Fail(std::string error); + + const HarmonyProtocol protocol_; + const std::shared_ptr plan_; + const std::unordered_set raw_tool_names_; + Phase phase_ = Phase::kHeader; + std::string header_; + std::string channel_; + std::string recipient_; + std::string arguments_; + bool visible_started_ = false; + bool failed_ = false; + bool checkpoint_captured_ = false; + size_t message_token_offset_ = 0; + size_t message_byte_offset_ = 0; + PreparedChatPrompt checkpoint_; + std::optional ending_; + int reasoning_tokens_ = 0; +}; + +template +void DeliverHarmonyOutput(HarmonyDecoder::Output output, SegmentSink&& segments, ToolSink&& tools) { + for (auto& event : output.events) { + if (auto* segment = std::get_if(&event)) { + segments(std::vector{std::move(*segment)}); + } else { + ToolCallStreamAccumulator::Output admitted; + admitted.events.emplace_back(std::move(std::get(event))); + tools(std::move(admitted)); + } + } + if (output.malformed) { + ToolCallStreamAccumulator::Output rejected; + rejected.malformed = true; + rejected.error = std::move(output.error); + tools(std::move(rejected)); + } +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h new file mode 100644 index 000000000..6158590a1 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include +#include + +namespace fl { + +struct HarmonyProtocol { + static constexpr const char* kAssistantPrefix = "<|start|>assistant"; + int32_t start; + int32_t channel; + int32_t message; + int32_t end; + int32_t handoff; + int32_t answer; + int32_t format; + std::vector assistant_prefix_tokens; +}; + +struct HarmonyContinuation { + std::string header; + size_t header_tokens = 0; + size_t header_bytes = 0; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc index 98bd81b94..923a3836e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -447,6 +447,9 @@ void OnnxChatEngine::RouteEvents() { conversation->result.generated_tokens = usage.GeneratedTokens(); conversation->result.cached_prompt_tokens = usage.CachedPromptTokens(); conversation->result.finish_reason = event->FinishReason(); +#if FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN + conversation->result.terminal_token_id = event->TerminalTokenId(); +#endif conversation->turn_finished = true; } if ((flags & OgaEngineEventFlag_Failed) != 0) { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h index f58582caa..0a40c4137 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h @@ -45,6 +45,7 @@ class OnnxChatEngine { uint64_t generated_tokens = 0; uint64_t cached_prompt_tokens = 0; uint32_t finish_reason = 0; + std::optional terminal_token_id = std::nullopt; }; class Conversation { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc index cc9c78bdf..09d999b8b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc @@ -62,6 +62,7 @@ ChatTurnUsage BuildTurnUsage(int prompt_tokens, const OnnxChatEngine::TurnResult MapFinishReason(static_cast(result.finish_reason)), MapTerminationCause(result.finish_reason), static_cast(result.cached_prompt_tokens), + result.terminal_token_id, }; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc index a4d6aa41a..5bdb2d2e0 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc @@ -14,6 +14,7 @@ #include #include +#include namespace fl { @@ -30,7 +31,7 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag } PreparedChatPrompt prepared; - if (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope()) { + if (tool_ctx.harmony || (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope())) { const auto original = BuildChatMessagesJson(messages.Messages(), tool_ctx.supports_reasoning_history); std::string prefix; do { @@ -39,19 +40,25 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag tool_ctx.template_kwargs_json.find(prefix) != std::string::npos); std::unordered_map literals; - const auto projected = BuildCanonicalToolMessagesJson( - messages.Messages(), tool_ctx, [&](std::string text) { - if (text.empty()) { - return text; - } - auto marker = prefix + std::to_string(literals.size()) + "__"; - literals.emplace(marker, std::move(text)); - return marker; - }); + const auto protect = [&](std::string text) { + if (text.empty()) { + return text; + } + auto marker = prefix + std::to_string(literals.size()) + "__"; + literals.emplace(marker, std::move(text)); + return marker; + }; + const auto native = tool_ctx.harmony + ? BuildHarmonyPromptProjection(messages.Messages(), tool_ctx, protect) + : HarmonyPromptProjection{}; + const auto projected = tool_ctx.harmony ? native.messages_json + : BuildCanonicalToolMessagesJson(messages.Messages(), tool_ctx, protect); auto& preprocessor = model.GetPreprocessor(); const auto* kwargs = tool_ctx.template_kwargs_json.empty() ? nullptr : tool_ctx.template_kwargs_json.c_str(); const auto rendered = - preprocessor.ApplyChatTemplateWithOptions(projected.c_str(), nullptr, kwargs, true); + preprocessor.ApplyChatTemplateWithOptions(projected.c_str(), + native.tools_json.empty() ? nullptr : native.tools_json.c_str(), + kwargs, true); const auto append = [&](const std::string& text, bool literal) { if (text.empty()) { return; @@ -84,6 +91,15 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag } prepared.prompt_token_count = static_cast(prepared.token_ids.size()); + if (tool_ctx.harmony) { + const auto& prefix = tool_ctx.harmony->assistant_prefix_tokens; + if (!rendered.ends_with(HarmonyProtocol::kAssistantPrefix) || prepared.token_ids.size() < prefix.size() || + !std::equal(prefix.rbegin(), prefix.rend(), prepared.token_ids.rbegin())) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Harmony template produced an unqualified generation continuation"); + } + prepared.harmony_continuation = HarmonyContinuation{ + "assistant", prefix.size(), std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + } return prepared; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h index 8fccdb531..1cb104f10 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h @@ -3,6 +3,7 @@ #pragma once #include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include #include @@ -33,6 +34,7 @@ struct PreparedChatPrompt { std::vector token_ids; std::unique_ptr media_tensors; int64_t prompt_token_count = 0; + std::optional harmony_continuation; bool HasMedia() const noexcept { return media_tensors != nullptr; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc index 66ccc16fa..b8ae5822d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc @@ -155,7 +155,7 @@ std::optional ResolveTurnGuidanceOptions(const ToolCallCont const bool tool_call_only = tool_ctx.tool_output && !tool_ctx.text_output; if (!guidance_type.empty() && !guidance_data.empty() && - (user_specified_guidance || tool_call_only || tool_ctx.tool_plan)) { + (user_specified_guidance || tool_call_only || tool_ctx.tool_plan || tool_ctx.harmony)) { return TurnGuidanceOptions{ std::move(guidance_type), std::move(guidance_data), @@ -248,8 +248,8 @@ EngineTurnOptionsPlan BuildEngineTurnOptionsPlan(const SearchOptions& options, plan.seed = *options.seed; } - // Native stops see hidden MESSAGE framing too; framed turns instead match ordinary visible text on the host. - if (ShouldForwardStopSequencesToEngine(backend_kind) && !tool_ctx.UsesAssistantTextFraming()) { + // Backend stops also see hidden framing and reasoning; protocol-aware turns match only visible text on the host. + if (ShouldForwardStopSequencesToEngine(backend_kind) && !tool_ctx.UsesAssistantTextFraming() && !tool_ctx.harmony) { plan.stop_sequences = options.stop_sequences; } diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index b53cbc096..5bbca1738 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -10,8 +10,8 @@ #include #include - #include +#include namespace fl { namespace { @@ -122,6 +122,86 @@ std::vector FindDelimitedRegions(std::string_view text, return regions; } +std::optional QualifyHarmonyProtocol(std::string_view model_type, Preprocessor& preprocessor) { + if (model_type != "gptoss") return std::nullopt; +#if !FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN + (void)preprocessor; + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS Harmony requires an ORT GenAI runtime exposing sampled Engine terminal-token identity"); +#else + const auto resolve = [&](const char* spelling) { + const auto encoded = preprocessor.Encode(spelling); + const auto literal = preprocessor.EncodeLiteral(spelling); + if (encoded->SequenceCount(0) != 1 || encoded->SequenceData(0)[0] < 0 || + (literal->SequenceCount(0) == 1 && literal->SequenceData(0)[0] == encoded->SequenceData(0)[0])) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Harmony control must have distinct literal/control encoding: ", spelling); + } + return encoded->SequenceData(0)[0]; + }; + HarmonyProtocol protocol{ + resolve("<|start|>"), + resolve("<|channel|>"), + resolve("<|message|>"), + resolve("<|end|>"), + resolve("<|call|>"), + resolve("<|return|>"), + resolve("<|constrain|>"), + {}, + }; + const std::set controls{protocol.start, protocol.channel, protocol.message, protocol.end, + protocol.handoff, protocol.answer, protocol.format}; + const auto& eos = preprocessor.GetEosTokenIds(); + if (controls.size() != 7 || std::find(eos.begin(), eos.end(), protocol.handoff) == eos.end() || + std::find(eos.begin(), eos.end(), protocol.answer) == eos.end() || + std::find(eos.begin(), eos.end(), protocol.end) != eos.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid Harmony control-token or EOS configuration"); + } + constexpr const char* probe = R"([ + {"role":"user","content":"probe"}, + {"role":"assistant","content":"probe-visible","thinking":"probe-private","tool_calls":[ + {"function":{"name":"probe_function","arguments":{},"arguments_json":"__probe_arguments__"}} + ]}, + {"role":"tool","name":"probe_function","content":"","content_json":"__probe_result__"} + ])"; + const auto rendered = preprocessor.ApplyChatTemplate(probe, nullptr, true); + if (!ContainsInOrder(rendered, {"<|channel|>analysis<|message|>probe-private<|end|>", + "<|channel|>commentary<|message|>probe-visible<|end|>", + "<|start|>assistant to=functions.probe_function<|channel|>commentary json" + "<|message|>__probe_arguments__<|call|>", + "<|start|>functions.probe_function to=assistant<|channel|>commentary" + "<|message|>__probe_result__<|end|>"}) || + !rendered.ends_with(HarmonyProtocol::kAssistantPrefix)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "GPT-OSS requires the qualified protected Harmony transcript template"); + } + constexpr const char* custom_probe = R"([ + {"role":"user","content":"probe"}, + {"role":"assistant","content":"","tool_calls":[ + {"function":{"name":"probe_custom","native_freeform":true, + "arguments":{},"arguments_json":"__probe_raw_input__"}} + ]}, + {"role":"tool","name":"probe_custom","content":"","content_json":"__probe_custom_result__"} + ])"; + constexpr const char* custom_tools = R"([ + {"type":"function","function":{"name":"probe_custom","description":"probe", + "native_freeform":true,"parameters":{}}} + ])"; + const auto custom_rendered = preprocessor.ApplyChatTemplate(custom_probe, custom_tools, true); + if (!ContainsInOrder(custom_rendered, {"type probe_custom = (_: FREEFORM) => any;", + "<|start|>assistant to=functions.probe_custom<|channel|>commentary" + "<|message|>__probe_raw_input__<|call|>", + "<|start|>functions.probe_custom to=assistant<|channel|>commentary" + "<|message|>__probe_custom_result__<|end|>"}) || + !custom_rendered.ends_with(HarmonyProtocol::kAssistantPrefix)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS requires protected native custom-tool declarations and history"); + } + const auto prefix = preprocessor.Encode(HarmonyProtocol::kAssistantPrefix); + protocol.assistant_prefix_tokens.assign(prefix->SequenceData(0), prefix->SequenceData(0) + prefix->SequenceCount(0)); + return protocol; +#endif +} + ModelCapabilities ResolveModelCapabilities(std::string_view model_type, Preprocessor& preprocessor, std::string_view tool_call_start, std::string_view tool_call_end, @@ -311,6 +391,16 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, capabilities_ = ResolveModelCapabilities(model_type_, *preprocessor_, tag_info.bot_str, tag_info.eot_str, tag_info.bor_str, tag_info.eor_str); + capabilities_.harmony = QualifyHarmonyProtocol(model_type_, *preprocessor_); + if (capabilities_.harmony) { + if (genai_config_.GetChatBackendKind() != ChatBackendKind::kEngine) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Qualified Harmony generation requires the Engine backend"); + } + capabilities_.supports_tool_calling = true; + capabilities_.supports_reasoning_history = true; + capabilities_.positional_tool_results = true; + capabilities_.supports_reasoning_controls = true; + } if (IsMultiModal() && genai_config_.GetChatBackendKind() == ChatBackendKind::kEngine) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 65df74b3d..f83a20ef9 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -4,6 +4,7 @@ #include "inferencing/execution_provider.h" #include "inferencing/generative/genai_config.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include "inferencing/generative/preprocessor.h" #include "logger.h" @@ -33,6 +34,7 @@ struct ModelCapabilities { bool supports_reasoning_history = false; bool supports_preserve_thinking = false; bool supports_reasoning_controls = false; + std::optional harmony = std::nullopt; }; namespace model_capabilities_internal { @@ -73,6 +75,7 @@ class GenAIModelInstance { bool SupportsReasoningHistory() const { return capabilities_.supports_reasoning_history; } bool SupportsPreserveThinking() const { return capabilities_.supports_preserve_thinking; } bool SupportsReasoningControls() const { return capabilities_.supports_reasoning_controls; } + const std::optional& GetHarmonyProtocol() const { return capabilities_.harmony; } /// Cached tag token IDs and decoded strings for tool/reasoning detection. /// Populated once on first access using OGA tokenizer APIs. diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 33d9660b7..86f739cb8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -741,6 +741,7 @@ Request ToSessionRequest(const ResponseCreateParams& params, const ResponseChain if (params.reasoning.has_value() && params.reasoning->effort.has_value()) { request.options["chat_template_kwargs"] = ResolveReasoningTemplateKwargs(std::nullopt, params.reasoning->effort).dump(); + request.options["reasoning_effort"] = *params.reasoning->effort; } // Text format / grammar guidance → metadata parameters diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc index e74b2acf6..de3b43acf 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc @@ -3,6 +3,7 @@ #include "inferencing/generative/toolcalling/grammar.h" #include "exception.h" +#include "contracts/tool_definitions.h" #include "inferencing/generative/toolcalling/tool_call_plan.h" #include @@ -12,6 +13,74 @@ namespace fl { +namespace { + +std::string BuildHarmonyGrammar(const ToolCallContext& context) { + if (context.tool_output && !context.tool_plan) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Harmony tool guidance requires an original-schema plan"); + } + const auto& protocol = *context.harmony; + const auto token = [](int32_t id) { return "<[" + std::to_string(id) + "]>"; }; + auto excluded = context.eos_token_ids; + for (const auto id : {protocol.start, protocol.channel, protocol.message, protocol.end, + protocol.handoff, protocol.answer, protocol.format}) excluded.push_back(id); + std::sort(excluded.begin(), excluded.end()); + excluded.erase(std::unique(excluded.begin(), excluded.end()), excluded.end()); + std::ostringstream grammar; + grammar << "start: " << (context.supports_reasoning ? "reasoning* " : ""); + if (!context.tool_output) { + grammar << "commentary* final\n"; + } else { + grammar << (context.text_output ? "commentary* (final | tool)\n" : "tool\n"); + } + grammar << "reasoning: " << token(protocol.channel) << " \"analysis\" " << token(protocol.message) + << " plain_token* " << token(protocol.end) << " " << token(protocol.start) << " \"assistant\"\n"; + grammar << "commentary: " << token(protocol.channel) << " \"commentary\" " << token(protocol.message) + << " plain_token* " << token(protocol.end) << " " << token(protocol.start) << " \"assistant\"\n"; + grammar << "final: " << token(protocol.channel) << " \"final\" " << token(protocol.message) + << " plain_token* " << token(protocol.answer) << "\n"; + if (context.tool_output) { + const auto names = context.tool_plan->AllowedNames(); + grammar << "tool: "; + for (size_t i = 0; i < names.size(); ++i) grammar << (i ? " | " : "") << "tool_" << i; + grammar << "\n"; + for (size_t i = 0; i < names.size(); ++i) { + const auto custom = context.UsesNativeRawBody(names[i]); + grammar << "tool_" << i << ": header_" << i << (custom ? " " : " metadata ") + << token(protocol.message) << " "; + const auto format = context.custom_lark_grammars.find(names[i]); + if (custom && format != context.custom_lark_grammars.end()) { + if (names[i] != "apply_patch" || format->second != tools::kStockGhcpApplyPatchLarkGrammar) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Unsupported native custom-tool input grammar"); + } + grammar << "native_patch"; + } else { + grammar << "arguments"; + } + grammar << " " << token(protocol.handoff) << "\n"; + const auto recipient = EscapeLarkLiteral("functions." + names[i]); + grammar << "header_" << i << ": \" to=\" " << recipient << " " << token(protocol.channel) + << " \"commentary\" | " << token(protocol.channel) << " \"commentary to=\" " << recipient << "\n"; + } + grammar << "metadata: (\" json\" | " << token(protocol.format) << " \"json\" | \" \" " + << token(protocol.format) << " \"json\")?\n"; + if (std::any_of(names.begin(), names.end(), [&](const auto& name) { + return context.UsesNativeRawBody(name); + })) { + auto patch = std::string(tools::kStockGhcpApplyPatchLarkGrammar); + patch.replace(0, std::string_view("start").size(), "native_patch"); + grammar << patch << "\n"; + } + } + grammar << "plain_token: <[^"; + for (size_t i = 0; i < excluded.size(); ++i) grammar << (i ? "," : "") << excluded[i]; + grammar << "]>\n"; + if (context.tool_output) grammar << context.tool_plan->ObjectGrammar("arguments"); + return grammar.str(); +} + +} // namespace + std::string EscapeLarkLiteral(const std::string& text) { return nlohmann::json(text).dump(); } @@ -28,6 +97,7 @@ std::string BuildLarkGrammar(const ToolCallContext& ctx, bool prompt_opens_reaso if (!ctx.text_output && !ctx.tool_output) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "neither text output nor tool calling output is enabled"); } + if (ctx.harmony) return BuildHarmonyGrammar(ctx); if (!ctx.tool_output) { return {}; } diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h index b7cd173d1..2e2c10cfa 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h @@ -3,6 +3,7 @@ #pragma once #include "inferencing/session/types.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include namespace fl { @@ -25,6 +27,7 @@ struct ToolCallContext { /// Whether the model supports tool calling, using explicit catalog metadata when present /// and otherwise the loaded model's rendered-template probe. bool supports_tool_calling = false; + std::optional harmony; /// The special token that marks the start of a tool call in generated text. /// Empty if the model does not advertise tool call tokens. @@ -102,6 +105,7 @@ struct ToolCallContext { } const RawEnvelopeDescriptor* ActiveRawEnvelope() const { + if (harmony) return nullptr; if (!tool_output || !raw_envelope.has_value() || !IsCustomTool(raw_envelope->tool_name)) { return nullptr; } @@ -132,6 +136,25 @@ struct ToolCallContext { /// Whether the named tool takes a raw text payload rather than JSON arguments. bool IsCustomTool(const std::string& name) const { return KindOf(name) == ToolKind::kCustom; } + /// Unframed custom text keeps JSON transport; native raw bodies require an admitted input grammar. + bool UsesNativeRawBody(const std::string& name) const { + return harmony && IsCustomTool(name) && custom_lark_grammars.find(name) != custom_lark_grammars.end(); + } + + std::unordered_set NativeRawToolNames() const { + std::unordered_set names; + for (const auto& entry : custom_lark_grammars) { + if (UsesNativeRawBody(entry.first)) names.insert(entry.first); + } + return names; + } + + GeneratedCallEncoding CallEncoding(const std::string& name, bool raw_envelope) const { + if (raw_envelope) return GeneratedCallEncoding::kRawEnvelope; + if (!harmony) return GeneratedCallEncoding::kStructured; + return UsesNativeRawBody(name) ? GeneratedCallEncoding::kNativeRaw : GeneratedCallEncoding::kNativeJson; + } + /// User-specified guidance type from response_format (e.g., "lark_grammar", "json_schema"). /// Empty means no explicit guidance — the generator may still apply auto-generated tool guidance. std::string guidance_type; @@ -152,7 +175,7 @@ struct ToolCallContext { } bool UsesAssistantTextFraming() const { - return tool_plan && text_output && tool_output && !HasAnyExplicitGuidance() && !ActiveRawEnvelope(); + return !harmony && tool_plan && text_output && tool_output && !HasAnyExplicitGuidance() && !ActiveRawEnvelope(); } /// Whether any tools were provided in the request. diff --git a/sdk_v2/cpp/src/inferencing/session/types.h b/sdk_v2/cpp/src/inferencing/session/types.h index 6488ff30f..0d85f6191 100644 --- a/sdk_v2/cpp/src/inferencing/session/types.h +++ b/sdk_v2/cpp/src/inferencing/session/types.h @@ -28,6 +28,8 @@ enum class ToolKind { enum class GeneratedCallEncoding { kStructured, kRawEnvelope, + kNativeRaw, + kNativeJson, }; struct ForcedToolChoice { From bb6be93af32a937bc498a09a4fe2774943ce36dd Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Sat, 7 Nov 2026 00:50:09 +0000 Subject: [PATCH 06/15] Preserve tool body encoding through Responses replay and recovery Carry per-call encoding outside public JSON through response publication, storage, replay snapshots and compaction. Keep raw-envelope callers compatible using the same metadata map. Recovered streaming items use the completed turn context so their encoding matches final responses and history. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a563a06-04d5-47cf-9b20-af502d60b1eb --- .../generative/chat/chat_session.cc | 3 ++- .../generative/openresponses/response_chain.h | 12 ++++++--- .../openresponses/response_converter.cc | 23 +++++++++------- .../openresponses/response_store.cc | 20 +++++++++++--- .../generative/openresponses/response_store.h | 17 ++++++++---- sdk_v2/cpp/src/service/responses_handler.cc | 27 ++++++++++--------- 6 files changed, 65 insertions(+), 37 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index d584aaaab..d8a3e9cbb 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -2159,7 +2159,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) if (streaming_callback) { streaming_callback->PushItem(std::make_unique( call.id, call.name, call.arguments, /*replayed_from_store=*/false, - completed_tool_ctx.KindOf(call.name), std::nullopt, GeneratedCallEncoding::kStructured)); + completed_tool_ctx.KindOf(call.name), std::nullopt, + completed_tool_ctx.CallEncoding(call.name, call.raw_envelope))); } } diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h index 2859c7cb4..e8f9288c5 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h @@ -2,15 +2,19 @@ // Licensed under the MIT License. #pragma once +#include "inferencing/session/types.h" + #include #include #include -#include +#include #include namespace fl { +using ToolCallEncodings = std::unordered_map; + /// One hop of a reconstructed Responses chain: the input items that request was given, followed by the output items /// it produced. /// @@ -24,14 +28,14 @@ namespace fl { struct ResponseChainHop { ResponseChainHop() = default; ResponseChainHop(nlohmann::json input, nlohmann::json output, - std::unordered_set raw_call_ids = {}) + ToolCallEncodings encodings = {}) : input_items(std::move(input)), output_items(std::move(output)), - raw_envelope_call_ids(std::move(raw_call_ids)) {} + call_encodings(std::move(encodings)) {} nlohmann::json input_items = nlohmann::json::array(); nlohmann::json output_items = nlohmann::json::array(); - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// A complete chain, oldest hop first. diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 86f739cb8..8e20b3f00 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -251,6 +251,11 @@ std::unique_ptr MakeReasoningMessage(std::string reasoning) { return message; } +GeneratedCallEncoding ReplayedCallEncoding(const std::string& call_id, const ToolCallEncodings& encodings) { + const auto found = encodings.find(call_id); + return found == encodings.end() ? GeneratedCallEncoding::kStructured : found->second; +} + /// Rebuild a stored `function_call` item. /// /// A hop's stored items are replayed as the hop recorded them, whether the call came from the model's own output or @@ -258,7 +263,7 @@ std::unique_ptr MakeReasoningMessage(std::string reasoning) { /// validated strictly at the time, and a model-emitted call whose argument bytes were not a JSON object was already /// presented to the model as having none. Replay reproduces that rather than re-admitting bytes the strict /// caller-supplied path would now reject, which would fail a continuation of a conversation that already happened. -std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { +std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item, const ToolCallEncodings& encodings) { const auto call_id = RequiredReplayString(item, "call_id", "stored function_call"); const auto name = RequiredReplayString(item, "name", "stored function_call"); std::string arguments; @@ -271,7 +276,7 @@ std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { } auto call = std::make_unique(call_id, name, arguments, /*replayed_from_store=*/true, - ToolKind::kFunction); + ToolKind::kFunction, std::nullopt, ReplayedCallEncoding(call_id, encodings)); call->replayed_arguments = std::move(arguments); call->replayed_kind = ToolKind::kFunction; return call; @@ -283,15 +288,13 @@ std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { /// produced it in, and the same form the tool would receive. It is never parsed or re-encoded: doing so would change /// the bytes for a payload that is itself JSON, or reject one that is not. std::unique_ptr MakeReplayedCustomToolCall( - const nlohmann::json& item, const std::unordered_set& raw_envelope_call_ids) { + const nlohmann::json& item, const ToolCallEncodings& encodings) { const auto call_id = RequiredReplayString(item, "call_id", "stored custom_tool_call"); const auto name = RequiredReplayString(item, "name", "stored custom_tool_call"); auto input = RequiredReplayString(item, "input", "stored custom_tool_call", true); auto call = std::make_unique(call_id, name, input, /*replayed_from_store=*/true, ToolKind::kCustom, std::nullopt, - raw_envelope_call_ids.contains(call_id) - ? GeneratedCallEncoding::kRawEnvelope - : GeneratedCallEncoding::kStructured); + ReplayedCallEncoding(call_id, encodings)); call->replayed_arguments = std::move(input); call->replayed_kind = ToolKind::kCustom; return call; @@ -411,7 +414,7 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) /// text moved in front of the call. A hop that produced nothing replayable still emits the assistant boundary it /// committed live. static void AddHopOutputToRequest(Request& request, const nlohmann::json& output_items, - const std::unordered_set& raw_envelope_call_ids) { + const ToolCallEncodings& encodings) { bool emitted = false; if (output_items.is_array()) { @@ -423,13 +426,13 @@ static void AddHopOutputToRequest(Request& request, const nlohmann::json& output const std::string type = entry.value("type", ""); if (type == "function_call") { - request.AddOwnedItem(MakeReplayedToolCall(entry)); + request.AddOwnedItem(MakeReplayedToolCall(entry, encodings)); emitted = true; continue; } if (type == "custom_tool_call") { - request.AddOwnedItem(MakeReplayedCustomToolCall(entry, raw_envelope_call_ids)); + request.AddOwnedItem(MakeReplayedCustomToolCall(entry, encodings)); emitted = true; continue; } @@ -466,7 +469,7 @@ static void AddChainContextToRequest(Request& request, const ResponseChainContex request.BeginItemSegment(); AddJsonItemsToRequest(request, hop.input_items); request.BeginItemSegment(); - AddHopOutputToRequest(request, hop.output_items, hop.raw_envelope_call_ids); + AddHopOutputToRequest(request, hop.output_items, hop.call_encodings); } } diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc index cabd988de..06495e066 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc @@ -50,15 +50,27 @@ void ResponseStore::Store(const std::string& response_id, nlohmann::json response, nlohmann::json input_items, std::string model_id, - std::unordered_set raw_envelope_call_ids) { + ToolCallEncodings call_encodings) { std::lock_guard lock(mutex_); auto state = PrepareStoreLocked({response_id, std::move(model_id), std::move(response), std::move(input_items), - std::move(raw_envelope_call_ids)}); + std::move(call_encodings)}); DropEvictedArtifactsLocked(state); CommitStoreLocked(std::move(state)); } +void ResponseStore::Store(const std::string& response_id, + nlohmann::json response, + nlohmann::json input_items, + std::string model_id, + std::unordered_set raw_envelope_call_ids) { + ToolCallEncodings encodings; + for (const auto& id : raw_envelope_call_ids) { + encodings.emplace(id, GeneratedCallEncoding::kRawEnvelope); + } + Store(response_id, std::move(response), std::move(input_items), std::move(model_id), std::move(encodings)); +} + ResponseStore::MetadataState ResponseStore::PrepareStoreLocked( StoredResponse response, const std::shared_ptr& replay_snapshot) { const auto response_id = response.id; @@ -116,7 +128,7 @@ void ResponseStore::StoreLocked(StoredResponse response, MetadataState& state) { .model_id = std::move(response.model_id), .response = std::move(response.response), .input_items = std::move(response.input_items), - .raw_envelope_call_ids = std::move(response.raw_envelope_call_ids), + .call_encodings = std::move(response.call_encodings), .replay_prefix = std::move(replay_prefix), .insertion_sequence = state.next_insertion_sequence++}); state.index[response.id] = state.entries.begin(); @@ -597,7 +609,7 @@ ResponseChainHop ResponseStore::ToReplayHop(const Entry& entry) { if (output != entry.response.end() && output->is_array()) { replay.output_items = *output; } - replay.raw_envelope_call_ids = entry.raw_envelope_call_ids; + replay.call_encodings = entry.call_encodings; return replay; } diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h index 6d90ca1ca..ebaf2a400 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h @@ -120,12 +120,12 @@ class ResponseStore { struct StoredResponse { StoredResponse(std::string id_in, std::string model_id_in, nlohmann::json response_in, nlohmann::json input_items_in, - std::unordered_set raw_call_ids_in = {}) + ToolCallEncodings call_encodings_in = {}) : id(std::move(id_in)), model_id(std::move(model_id_in)), response(std::move(response_in)), input_items(std::move(input_items_in)), - raw_envelope_call_ids(std::move(raw_call_ids_in)) {} + call_encodings(std::move(call_encodings_in)) {} std::string id; /// Resolved model that produced it. A continuation must resolve to the same model; empty means unbound, which @@ -135,7 +135,7 @@ class ResponseStore { nlohmann::json response; /// This hop's own request items — exactly what the /input_items endpoint returns. nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// Outcome of ResponseStore::BeginResponse. @@ -191,7 +191,14 @@ class ResponseStore { nlohmann::json response, nlohmann::json input_items, std::string model_id = {}, - std::unordered_set raw_envelope_call_ids = {}); + ToolCallEncodings call_encodings = {}); + + /// Compatibility entry point for callers that record only legacy raw-envelope provenance. + void Store(const std::string& response_id, + nlohmann::json response, + nlohmann::json input_items, + std::string model_id, + std::unordered_set raw_envelope_call_ids); /// Retrieve a stored response by ID. Returns nullopt if not found. std::optional Get(const std::string& response_id); @@ -268,7 +275,7 @@ class ResponseStore { std::string model_id; nlohmann::json response; nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; std::shared_ptr replay_prefix; uint64_t insertion_sequence = 0; }; diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index fcc652bea..639359a66 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -22,7 +22,6 @@ #include #include #include -#include namespace fl { @@ -61,7 +60,7 @@ struct PublishRequest { std::string model_id; nlohmann::json response; nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// Publish a completed response: store its metadata and cache its session as one indivisible step. @@ -79,7 +78,7 @@ void PublishResponse(PublishRequest request) { const bool published = request.store.Commit( request.lease, ResponseStore::StoredResponse{request.response_id, request.model_id, std::move(request.response), - std::move(request.input_items), std::move(request.raw_envelope_call_ids)}, + std::move(request.input_items), std::move(request.call_encodings)}, &admission); if (!published) { @@ -426,11 +425,13 @@ std::shared_ptr ResponsesHandler::HandleNo session->ProcessRequest(session_request, session_response); auto [output, output_text] = ResponseConverter::FromSessionResponse(session_response); - std::unordered_set raw_envelope_call_ids; - for (const auto& item : output) { - if (const auto* custom = std::get_if(&item); - custom != nullptr && custom->generated_encoding == GeneratedCallEncoding::kRawEnvelope) { - raw_envelope_call_ids.insert(custom->call_id); + ToolCallEncodings call_encodings; + for (const auto& item : session_response.items) { + if (item->type == FOUNDRY_LOCAL_ITEM_TOOL_CALL) { + const auto& call = static_cast(*item); + if (call.generated_encoding != GeneratedCallEncoding::kStructured) { + call_encodings.emplace(call.call_id, call.generated_encoding); + } } } @@ -450,7 +451,7 @@ std::shared_ptr ResponsesHandler::HandleNo .model_id = turn.model_id, .response = response_json, .input_items = ResponseConverter::ToInputItems(req_json), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .call_encodings = std::move(call_encodings)}); } return JsonResponse(Status::CODE_200, response_json); @@ -531,7 +532,7 @@ std::shared_ptr ResponsesHandler::HandleSt // Items that have been *closed* (or, for the currently-open item at end-of-stream, finalized in place). // Used to construct the final `output[]` array for the response.completed event. std::vector closed_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; auto push_event = [&](const std::string& event_name, const StreamEvent& ev) { stream->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); @@ -669,8 +670,8 @@ std::shared_ptr ResponsesHandler::HandleSt } closed_items.push_back(std::move(output.completed_item)); - if (call.generated_encoding == GeneratedCallEncoding::kRawEnvelope) { - raw_envelope_call_ids.insert(call.call_id); + if (call.generated_encoding != GeneratedCallEncoding::kStructured) { + call_encodings.emplace(call.call_id, call.generated_encoding); } }; @@ -779,7 +780,7 @@ std::shared_ptr ResponsesHandler::HandleSt .model_id = turn.model_id, .response = std::move(response_json), .input_items = ResponseConverter::ToInputItems(req_copy), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .call_encodings = std::move(call_encodings)}); })) { stream->Finish(); return; From 21c6f0edebe491a878e11b708563834c7da65353 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Sat, 7 Nov 2026 00:52:45 +0000 Subject: [PATCH 07/15] Keep caller-supplied replay calls on the default encoding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1a563a06-04d5-47cf-9b20-af502d60b1eb --- .../inferencing/generative/openresponses/response_converter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 8e20b3f00..1234543e8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -360,7 +360,7 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) } if (type == "function_call") { - request.AddOwnedItem(MakeReplayedToolCall(entry)); + request.AddOwnedItem(MakeReplayedToolCall(entry, {})); continue; } From e81aca67140ab1fc361ad6661aa599a74e6a519c Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:36:58 +0000 Subject: [PATCH 08/15] Support legal Harmony tool history Qualify the GPT-OSS thinking-only tool-call shape and allow runtimes with terminal-token identity to use Harmony through the Generator backend. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/inferencing/generative/genai_model_instance.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 5bbca1738..4e30406e3 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -159,20 +159,20 @@ std::optional QualifyHarmonyProtocol(std::string_view model_typ } constexpr const char* probe = R"([ {"role":"user","content":"probe"}, - {"role":"assistant","content":"probe-visible","thinking":"probe-private","tool_calls":[ + {"role":"assistant","content":"","thinking":"probe-private","tool_calls":[ {"function":{"name":"probe_function","arguments":{},"arguments_json":"__probe_arguments__"}} ]}, {"role":"tool","name":"probe_function","content":"","content_json":"__probe_result__"} ])"; const auto rendered = preprocessor.ApplyChatTemplate(probe, nullptr, true); if (!ContainsInOrder(rendered, {"<|channel|>analysis<|message|>probe-private<|end|>", - "<|channel|>commentary<|message|>probe-visible<|end|>", "<|start|>assistant to=functions.probe_function<|channel|>commentary json" "<|message|>__probe_arguments__<|call|>", "<|start|>functions.probe_function to=assistant<|channel|>commentary" "<|message|>__probe_result__<|end|>"}) || !rendered.ends_with(HarmonyProtocol::kAssistantPrefix)) { - FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "GPT-OSS requires the qualified protected Harmony transcript template"); + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS requires the qualified protected Harmony transcript template"); } constexpr const char* custom_probe = R"([ {"role":"user","content":"probe"}, @@ -393,9 +393,6 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, tag_info.bor_str, tag_info.eor_str); capabilities_.harmony = QualifyHarmonyProtocol(model_type_, *preprocessor_); if (capabilities_.harmony) { - if (genai_config_.GetChatBackendKind() != ChatBackendKind::kEngine) { - FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Qualified Harmony generation requires the Engine backend"); - } capabilities_.supports_tool_calling = true; capabilities_.supports_reasoning_history = true; capabilities_.positional_tool_results = true; From f9d2b43e3fe6af15e89367d0d82476d02c958a42 Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:42:36 +0000 Subject: [PATCH 09/15] Cover Harmony terminal and replay contracts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/onnx_chat_generator.cc | 2 + .../generative/genai_model_instance.cc | 2 +- sdk_v2/cpp/test/CMakeLists.txt | 1 + .../internal_api/chat/harmony_decoder_test.cc | 213 ++++++++++++++++++ 4 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 22a51a777..3fc56eaf7 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -208,6 +208,8 @@ std::optional OnnxChatGenerator::GetTurnUsage() const { generated_tokens, termination.finish_reason, termination.cause, + 0, + eos_matched ? last_generated_token_ : std::nullopt, }; } diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 4e30406e3..f37af264b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -127,7 +127,7 @@ std::optional QualifyHarmonyProtocol(std::string_view model_typ #if !FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN (void)preprocessor; FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, - "GPT-OSS Harmony requires an ORT GenAI runtime exposing sampled Engine terminal-token identity"); + "GPT-OSS Harmony requires an ORT GenAI runtime exposing sampled terminal-token identity"); #else const auto resolve = [&](const char* spelling) { const auto encoded = preprocessor.Encode(spelling); diff --git a/sdk_v2/cpp/test/CMakeLists.txt b/sdk_v2/cpp/test/CMakeLists.txt index 012dbe138..587721db9 100644 --- a/sdk_v2/cpp/test/CMakeLists.txt +++ b/sdk_v2/cpp/test/CMakeLists.txt @@ -97,6 +97,7 @@ add_executable(foundry_local_tests internal_api/callback_handler_test.cc internal_api/catalog_cache_test.cc internal_api/chat/chat_generator_test.cc + internal_api/chat/harmony_decoder_test.cc internal_api/chat/reasoning_stream_splitter_test.cc internal_api/chat/chat_session_test.cc internal_api/chat/chat_template_test.cc diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc new file mode 100644 index 000000000..3dde8cc0d --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/chat_transcript.h" +#include "inferencing/generative/chat/harmony_decoder.h" +#include "inferencing/generative/toolcalling/tool_call_context.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace fl; + +namespace { + +const HarmonyProtocol kProtocol{ + 10, 11, 12, 13, 14, 15, 16, {10, 17}}; + +constexpr const char* kTools = R"([ + {"type":"function","function":{ + "name":"lookup", + "parameters":{ + "type":"object", + "properties":{"city":{"type":"string"}}, + "required":["city"] + } + }}, + {"type":"function","function":{ + "name":"patch", + "parameters":{ + "type":"object", + "properties":{"input":{"type":"string"}}, + "required":["input"] + } + }} +])"; + +PreparedChatPrompt MakePrompt() { + PreparedChatPrompt prompt; + prompt.prompt = "prefix<|start|>assistant"; + prompt.token_ids = {99, kProtocol.start, 17}; + prompt.harmony_continuation = HarmonyContinuation{ + "assistant", kProtocol.assistant_prefix_tokens.size(), + std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + return prompt; +} + +void PushHeader(HarmonyDecoder& decoder, std::string recipient, bool json) { + EXPECT_FALSE(decoder.Push(100, " to=functions.").malformed); + EXPECT_FALSE(decoder.Push(101, std::move(recipient)).malformed); + EXPECT_FALSE(decoder.Push(kProtocol.channel, {}).malformed); + EXPECT_FALSE(decoder.Push(102, "commentary").malformed); + if (json) { + EXPECT_FALSE(decoder.Push(kProtocol.format, {}).malformed); + EXPECT_FALSE(decoder.Push(103, "json").malformed); + } + EXPECT_FALSE(decoder.Push(kProtocol.message, {}).malformed); +} + +const ParsedToolCall& OnlyCall(const HarmonyDecoder::Output& output) { + EXPECT_FALSE(output.malformed) << output.error; + EXPECT_EQ(output.events.size(), 1u); + const auto* call = output.events.empty() ? nullptr : std::get_if(&output.events.front()); + EXPECT_NE(call, nullptr); + static const ParsedToolCall empty; + return call == nullptr ? empty : *call; +} + +} // namespace + +TEST(HarmonyDecoderTest, UsesControlIdsWithEmptyDecodedTextAndAcceptsSplitJsonHeader) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + PushHeader(decoder, "lookup", true); + constexpr std::string_view arguments = R"({"city": "Paris"})"; + EXPECT_FALSE(decoder.Push(104, std::string(arguments)).malformed); + const auto output = decoder.Finish(kProtocol.handoff, true); + const auto& call = OnlyCall(output); + + EXPECT_EQ(call.name, "lookup"); + EXPECT_EQ(call.arguments, arguments); + EXPECT_EQ(call.argument_source, arguments); + EXPECT_FALSE(call.id.empty()); +} + +TEST(HarmonyDecoderTest, LiteralControlSpellingsCannotManufactureHeaderMetadata) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + EXPECT_FALSE(decoder.Push(100, "<|channel|>commentary").malformed); + const auto output = decoder.Push(kProtocol.message, {}); + + EXPECT_TRUE(output.malformed); + EXPECT_NE(output.error.find("exactly one channel"), std::string::npos); +} + +TEST(HarmonyDecoderTest, TerminalMetadataCompletesToolAndFinalMessages) { + auto tool_prompt = MakePrompt(); + HarmonyDecoder tool_decoder(kProtocol, ToolCallPlan::Create(kTools), tool_prompt); + PushHeader(tool_decoder, "lookup", true); + EXPECT_FALSE(tool_decoder.Push(104, R"({"city":"Paris"})").malformed); + EXPECT_EQ(OnlyCall(tool_decoder.Finish(kProtocol.handoff, true)).name, "lookup"); + + auto final_prompt = MakePrompt(); + HarmonyDecoder final_decoder(kProtocol, nullptr, final_prompt); + EXPECT_FALSE(final_decoder.Push(kProtocol.channel, {}).malformed); + EXPECT_FALSE(final_decoder.Push(105, "final").malformed); + EXPECT_FALSE(final_decoder.Push(kProtocol.message, {}).malformed); + EXPECT_FALSE(final_decoder.Push(106, "done").malformed); + const auto final = final_decoder.Finish(kProtocol.answer, true); + EXPECT_FALSE(final.malformed) << final.error; + EXPECT_TRUE(final.events.empty()); +} + +TEST(HarmonyDecoderTest, InterruptedOrMalformedCallsFailWithoutPublishing) { + auto interrupted_prompt = MakePrompt(); + HarmonyDecoder interrupted(kProtocol, ToolCallPlan::Create(kTools), interrupted_prompt); + PushHeader(interrupted, "lookup", true); + EXPECT_FALSE(interrupted.Push(104, R"({"city":"Paris"})").malformed); + const auto interrupted_output = interrupted.Finish(std::nullopt, true); + EXPECT_TRUE(interrupted_output.malformed); + EXPECT_TRUE(interrupted_output.events.empty()); + + auto unauthorized_prompt = MakePrompt(); + HarmonyDecoder unauthorized(kProtocol, ToolCallPlan::Create(kTools), unauthorized_prompt); + PushHeader(unauthorized, "missing", true); + EXPECT_FALSE(unauthorized.Push(104, R"({"city":"Paris"})").malformed); + const auto unauthorized_output = unauthorized.Push(kProtocol.handoff, {}); + EXPECT_TRUE(unauthorized_output.malformed); + EXPECT_TRUE(unauthorized_output.events.empty()); +} + +TEST(HarmonyDecoderTest, SelectedRecipientCapturesARecoveryCheckpointBeforeItsHeader) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + EXPECT_FALSE(decoder.Push(100, " to=func").malformed); + EXPECT_TRUE(decoder.HasRecoveryCheckpoint()); + EXPECT_FALSE(decoder.Push(101, "tions.lookup").malformed); + ASSERT_TRUE(decoder.HasRecoveryCheckpoint()); + + const auto recovered = decoder.TakeRecoveryPrompt(); + EXPECT_EQ(recovered.prompt, "prefix"); + EXPECT_EQ(recovered.token_ids, (std::vector{99})); +} + +TEST(HarmonyDecoderTest, NativeRawBodyUsesInputEnvelopeWithoutLosingSourceBytes) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt, {"patch"}); + + PushHeader(decoder, "patch", false); + const std::string body = "*** Begin Patch\n+literal <|call|>\n*** End Patch"; + EXPECT_FALSE(decoder.Push(104, body).malformed); + const auto output = decoder.Push(kProtocol.handoff, {}); + const auto& call = OnlyCall(output); + const auto expected = nlohmann::json{{"input", body}}.dump(); + + EXPECT_EQ(call.name, "patch"); + EXPECT_EQ(call.arguments, expected); + EXPECT_EQ(call.argument_source, expected); +} + +TEST(HarmonyDecoderTest, PromptProjectionPreservesNativeJsonRawAndResultProvenance) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"lookup", ToolKind::kFunction}, {"patch", ToolKind::kCustom}}; + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + + TranscriptMessage raw_assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + raw_assistant.AppendToolCall( + MakeGeneratedToolCall("call_raw", "patch", "raw <|call|> text", + ToolKind::kCustom, GeneratedCallEncoding::kNativeRaw) + .call); + TranscriptMessage json_assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + const std::string json_arguments = R"({"path":"a\\b","text":"<|call|>"})"; + json_assistant.AppendToolCall( + MakeGeneratedToolCall("call_json", "lookup", json_arguments, + ToolKind::kFunction, GeneratedCallEncoding::kNativeJson) + .call); + const std::vector messages{ + std::move(raw_assistant), + TranscriptMessage::ToolResult("call_raw", "raw result"), + std::move(json_assistant), + TranscriptMessage::ToolResult("call_json", "json result"), + }; + + const auto projection = BuildHarmonyPromptProjection(messages, context, [](std::string value) { + return value; + }); + const auto projected = nlohmann::json::parse(projection.messages_json); + ASSERT_EQ(projected.size(), 5u); + EXPECT_EQ(projected[1]["tool_calls"][0]["function"]["arguments_json"], "raw <|call|> text"); + EXPECT_TRUE(projected[1]["tool_calls"][0]["function"]["native_freeform"]); + EXPECT_EQ(projected[2]["content_json"], nlohmann::json("raw result").dump()); + EXPECT_EQ(projected[3]["tool_calls"][0]["function"]["arguments_json"], json_arguments); + EXPECT_FALSE(projected[3]["tool_calls"][0]["function"]["native_freeform"]); + EXPECT_EQ(projected[4]["content_json"], nlohmann::json("json result").dump()); + + const auto projected_tools = nlohmann::json::parse(projection.tools_json); + EXPECT_FALSE(projected_tools[0]["function"].contains("native_freeform")); + EXPECT_TRUE(projected_tools[1]["function"]["native_freeform"].get()); +} From b25ac2269d9ec1c0bda71bf6e62b5258088dde10 Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:19:13 +0000 Subject: [PATCH 10/15] Cover direct tool batch admission and object grammar helpers Test exact aggregate limits, nonparallel and empty batches, unchanged supplied IDs and argument/source bytes, invalid final calls, object grammar roots, and forced-name filtering directly rather than only through ParseCalls. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../toolcalling/tool_call_plan_test.cc | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_plan_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_plan_test.cc index 89abc8c5f..af309c13b 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_plan_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_plan_test.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -503,5 +504,84 @@ TEST(ToolCallPlanTest, EntireBatchFailsIfLaterCallIsInvalid) { ExpectError([&] { plan->ParseCalls(batch); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "/n"); } +TEST(ToolCallPlanTest, PreparsedBatchPreservesIdsArgumentBytesAndCustomSource) { + const auto plan = Plan(Json::object()); + ParsedToolCall ordinary{"supplied-id", "edit", " \t{ \"n\" : 1 }\n"}; + ParsedToolCall custom{"custom-id", "edit", "line one\n\"quoted\"\\path"}; + custom.argument_source = R"( { "input" : "line one\n\"quoted\"\\path" } )"; + const std::vector calls{ordinary, custom}; + EXPECT_NO_THROW(plan->ValidateBatch(calls)); + EXPECT_EQ(calls[0].id, ordinary.id); + EXPECT_EQ(calls[0].arguments, ordinary.arguments); + EXPECT_TRUE(calls[0].argument_source.empty()); + EXPECT_EQ(calls[1].id, custom.id); + EXPECT_EQ(calls[1].arguments, custom.arguments); + EXPECT_EQ(calls[1].argument_source, custom.argument_source); +} + +TEST(ToolCallPlanTest, PreparsedBatchRejectsEmptyAndNonparallelCallsDirectly) { + const auto tools = Json::array({Tool("edit", Json::object())}).dump(); + const std::vector calls{{"a", "edit", "{}"}, {"b", "edit", "{}"}}; + for (const bool parallel : {false, true}) { + const auto plan = ToolCallPlan::Create(tools, parallel); + ExpectError([&] { plan->ValidateBatch({}); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "non-empty"); + EXPECT_NO_THROW(plan->ValidateBatch({calls[0]})); + if (parallel) { + EXPECT_NO_THROW(plan->ValidateBatch(calls)); + } else { + ExpectError([&] { plan->ValidateBatch(calls); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "parallel"); + } + } +} + +TEST(ToolCallPlanTest, PreparsedBatchChecksExactAggregateNameAndSourceByteLimits) { + const auto plan = Plan(Json::object()); + ParsedToolCall call{"existing", "edit", "{}"}; + call.argument_source = "{}" + std::string(ToolCallPlan::kMaxArgumentBytes - call.name.size() - 2, ' '); + std::vector calls(4, call); + ASSERT_EQ(calls.size() * (call.name.size() + call.argument_source.size()), ToolCallPlan::kMaxBatchBytes); + EXPECT_NO_THROW(plan->ValidateBatch(calls)); + calls.back().argument_source.push_back(' '); + ExpectError([&] { plan->ValidateBatch(calls); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "batch exceeds byte limit"); + calls.back().argument_source.pop_back(); + calls.push_back({"last", "edit", "{}"}); + ExpectError([&] { plan->ValidateBatch(calls); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "batch exceeds byte limit"); + call.argument_source = "{}" + std::string(ToolCallPlan::kMaxArgumentBytes - 1, ' '); + ExpectError([&] { plan->ValidateBatch({call}); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "byte limit"); +} + +TEST(ToolCallPlanTest, PreparsedBatchRejectsAnInvalidFinalCallWithoutChangingInputs) { + const auto plan = Plan({{"properties", {{"n", {{"type", "integer"}}}}}, {"required", {"n"}}}); + const std::vector calls{{"first", "edit", R"({ "n": 1 })"}, + {"last", "edit", R"({"n":"bad"})"}}; + ExpectError([&] { plan->ValidateBatch(calls); }, FOUNDRY_LOCAL_ERROR_INTERNAL, "/n"); + EXPECT_EQ(calls[0].id, "first"); + EXPECT_EQ(calls[0].arguments, R"({ "n": 1 })"); + EXPECT_EQ(calls[1].id, "last"); + EXPECT_EQ(calls[1].arguments, R"({"n":"bad"})"); +} + +TEST(ToolCallPlanTest, ObjectGrammarUsesOnlyAnObjectAndValidatesCallerRoots) { + const auto plan = Plan(Json::object()); + EXPECT_TRUE(plan->ObjectGrammar().starts_with("start: tool_json_ws tool_json_object tool_json_ws\n")); + EXPECT_TRUE(plan->ObjectGrammar("arguments").starts_with("arguments: tool_json_ws tool_json_object tool_json_ws\n")); + EXPECT_EQ(plan->ObjectGrammar("arguments").find("tool_json_call"), std::string::npos); + for (const std::string root : {"", "UPPER", "bad-name", "x\nstart", "tool_json_value"}) { + ExpectError([&] { plan->ObjectGrammar(root); }, FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "root rule"); + } +} + +TEST(ToolCallPlanTest, AllowedNamesAndPreparsedAdmissionApplyForcedNameFiltering) { + const auto tools = Json::array({Tool("first", Json::object()), Tool("second", Json::object())}).dump(); + auto names = ToolCallPlan::Create(tools)->AllowedNames(); + std::sort(names.begin(), names.end()); + EXPECT_EQ(names, (std::vector{"first", "second"})); + const auto plan = ToolCallPlan::Create(tools, true, ForcedToolChoice{"second"}); + EXPECT_EQ(plan->AllowedNames(), (std::vector{"second"})); + EXPECT_NO_THROW(plan->ValidateBatch({{"keep-id", "second", "{}"}})); + ExpectError([&] { plan->ValidateBatch({{"keep-id", "first", "{}"}}); }, + FOUNDRY_LOCAL_ERROR_INTERNAL, "first"); +} + } // namespace } // namespace fl From c714171dc81f3789bd9270c94b6aab53e26d0e17 Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:54:43 +0000 Subject: [PATCH 11/15] Reject Harmony instruction roles that would be silently omitted Require system or developer instructions to lead the transcript, with tool-enabled and tool-free projection coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/chat_template.cc | 5 +++++ .../generative/chat/chat_template.h | 1 + .../internal_api/chat/harmony_decoder_test.cc | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index fe3970c87..a7196a399 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -336,6 +336,11 @@ HarmonyPromptProjection BuildHarmonyPromptProjection( auto projected = Json::array(); std::unordered_map call_names; for (const auto& message : messages) { + if (!projected.empty() && + (message.role == FOUNDRY_LOCAL_ROLE_SYSTEM || message.role == FOUNDRY_LOCAL_ROLE_DEVELOPER)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Harmony requires system or developer instructions to be the first message"); + } Json entry{{"role", Utils::RoleToString(message.role)}, {"content", protect_literal(message.VisibleText())}}; if (message.role == FOUNDRY_LOCAL_ROLE_ASSISTANT) { entry["thinking"] = protect_literal(message.ReasoningText()); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index f3867e16c..723f5d9cf 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -104,6 +104,7 @@ struct HarmonyPromptProjection { std::string tools_json; }; +/// Reject non-leading system/developer messages that the qualified Harmony template would omit. HarmonyPromptProjection BuildHarmonyPromptProjection( const std::vector& messages, const ToolCallContext& context, const std::function& protect_literal); diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc index 3dde8cc0d..33bdecdb1 100644 --- a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "exception.h" #include "inferencing/generative/chat/chat_template.h" #include "inferencing/generative/chat/chat_transcript.h" #include "inferencing/generative/chat/harmony_decoder.h" @@ -211,3 +212,24 @@ TEST(HarmonyDecoderTest, PromptProjectionPreservesNativeJsonRawAndResultProvenan EXPECT_FALSE(projected_tools[0]["function"].contains("native_freeform")); EXPECT_TRUE(projected_tools[1]["function"]["native_freeform"].get()); } + +TEST(HarmonyDecoderTest, PromptProjectionRejectsNonLeadingInstructionRoles) { + for (const auto* tools : {"", kTools}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = tools; + for (const auto role : {FOUNDRY_LOCAL_ROLE_SYSTEM, FOUNDRY_LOCAL_ROLE_DEVELOPER}) { + const auto protect = [](std::string value) { return value; }; + EXPECT_NO_THROW(BuildHarmonyPromptProjection( + {{role, "Follow these instructions."}, {FOUNDRY_LOCAL_ROLE_USER, "Hello"}}, context, protect)); + try { + BuildHarmonyPromptProjection( + {{FOUNDRY_LOCAL_ROLE_USER, "Hello"}, {role, "Do not omit this instruction."}, {FOUNDRY_LOCAL_ROLE_USER, "Continue"}}, + context, protect); + FAIL() << "Expected non-leading instructions to be rejected"; + } catch (const fl::Exception& error) { + EXPECT_EQ(error.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } + } + } +} From 74c5e7f9327b1f30a1668c590572851a54b9ee20 Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:56:01 +0000 Subject: [PATCH 12/15] Reject Harmony retry calls that fail custom input normalization Do not read a cleared admission batch after malformed custom input. Keep the existing explicit retry error path and publish no call. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index d8a3e9cbb..154dd265a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -893,6 +893,10 @@ GuidedToolRetryResult RunHarmonyToolRetry(const chat_internal::PreparedChatMessa ToolCallStreamAccumulator::Output admitted; admitted.events.emplace_back(std::move(*call)); chat_session_internal::NormalizeToolOutputBatch(admitted, context); + if (admitted.malformed || admitted.events.empty()) { + error = "Guided Harmony recovery failed tool-call normalization"; + continue; + } result.events.emplace_back(std::move(std::get(admitted.events.front()))); } else { error = "Guided Harmony recovery emitted text instead of one corrected tool call"; From 67a8db7816f9a73e027e52cdd86d1d77fcdba8fb Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:23:17 +0000 Subject: [PATCH 13/15] Validate Harmony recipients before template rendering Reject declaration and history names containing header separators instead of generating an undecodable recipient. Keep ordinary valid tool selection unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/chat_template.cc | 8 +++++++ .../generative/chat/chat_template.h | 2 +- .../internal_api/chat/harmony_decoder_test.cc | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index a7196a399..aaa090f74 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -334,6 +334,12 @@ HarmonyPromptProjection BuildHarmonyPromptProjection( const std::function& protect_literal) { using Json = nlohmann::ordered_json; auto projected = Json::array(); + const auto validate_tool_name = [](std::string_view name) { + if (name.empty() || name.find_first_of(" \t\r\n\v\f\x01\x02") != std::string_view::npos) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Harmony tool names must not contain header separators"); + } + }; std::unordered_map call_names; for (const auto& message : messages) { if (!projected.empty() && @@ -346,6 +352,7 @@ HarmonyPromptProjection BuildHarmonyPromptProjection( entry["thinking"] = protect_literal(message.ReasoningText()); auto calls = Json::array(); for (const auto* call : message.ToolCalls()) { + validate_tool_name(call->name); if (!call_names.emplace(call->call_id, call->name).second) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Duplicate Harmony history call ID"); } @@ -390,6 +397,7 @@ HarmonyPromptProjection BuildHarmonyPromptProjection( auto function_tools = Json::array(); for (const auto& tool : tools) { const auto name = tool.at("function").at("name").get(); + validate_tool_name(name); if (!context.UsesNativeRawBody(name)) function_tools.push_back(tool); const auto grammar = context.custom_lark_grammars.find(name); if (context.IsCustomTool(name) && grammar != context.custom_lark_grammars.end()) { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index 723f5d9cf..80e0d2178 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -104,7 +104,7 @@ struct HarmonyPromptProjection { std::string tools_json; }; -/// Reject non-leading system/developer messages that the qualified Harmony template would omit. +/// Reject non-leading instructions and tool names containing Harmony header separators before rendering. HarmonyPromptProjection BuildHarmonyPromptProjection( const std::vector& messages, const ToolCallContext& context, const std::function& protect_literal); diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc index 33bdecdb1..26ec9a1f2 100644 --- a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -233,3 +233,26 @@ TEST(HarmonyDecoderTest, PromptProjectionRejectsNonLeadingInstructionRoles) { } } } + +TEST(HarmonyDecoderTest, PromptProjectionRejectsHeaderSeparatorsInToolAndHistoryNames) { + for (const char separator : std::string(" \t\r\n\v\f\x01\x02")) { + const std::string name = "lookup" + std::string(1, separator) + "city"; + for (const bool history : {false, true}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = history ? kTools : nlohmann::json::array({{{"type", "function"}, {"function", {{"name", name}, {"parameters", {{"type", "object"}}}}}}}).dump(); + std::vector messages{{FOUNDRY_LOCAL_ROLE_USER, "Hello"}}; + if (history) { + TranscriptMessage assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + assistant.AppendToolCall(MakeSuppliedToolCall("call_bad", name, "{}")); + messages.push_back(std::move(assistant)); + } + try { + BuildHarmonyPromptProjection(messages, context, [](std::string value) { return value; }); + FAIL() << "Expected an unrepresentable Harmony recipient to be rejected"; + } catch (const fl::Exception& error) { + EXPECT_EQ(error.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } + } + } +} From 29d8b3ea84bdb1d089c18c5de31f296e452c520c Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:59:01 +0000 Subject: [PATCH 14/15] Preserve stored-input body provenance and cover protocol grammars Carry known per-call encoding metadata through stored input replay as well as output replay. Cover native JSON versus raw history bodies, choice-specific headers, reserved boundary IDs, and native patch grammar admission. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/README.md | 3 +- .../openresponses/response_converter.cc | 9 +- .../internal_api/chat/harmony_decoder_test.cc | 120 ++++++++++++++++++ 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/sdk_v2/cpp/README.md b/sdk_v2/cpp/README.md index f7bdf8928..b0109481a 100644 --- a/sdk_v2/cpp/README.md +++ b/sdk_v2/cpp/README.md @@ -419,7 +419,8 @@ argument bytes are preserved. Custom tools with the admitted stock patch grammar use native raw bodies; unframed text custom tools retain JSON input-object transport. Both reuse the synthesized `{"input":"raw text"}` object for schema admission and deliver unchanged text to callers. Recorded native body encodings survive tool -definition changes during history replay. Tool results are correlated by call ID +definition changes during history replay, including encoding metadata attached to +stored input calls as well as generated output calls. Tool results are correlated by call ID and rendered as named, protected JSON values. Source text and literal marker spellings remain data. Original schemas and custom input formats are supplied as bounded literal prompt information; schema admission remains owned by the existing `ToolCallPlan`. diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 1234543e8..fb83f53de 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -328,7 +328,8 @@ flMessageRole ReplayRole(const std::string& role) { } // namespace /// Replay a hop's own input items — the request the caller sent for that hop, in the shape they sent it. -static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) { +static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items, + const ToolCallEncodings& encodings) { if (!items.is_array()) { return; } @@ -360,12 +361,12 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) } if (type == "function_call") { - request.AddOwnedItem(MakeReplayedToolCall(entry, {})); + request.AddOwnedItem(MakeReplayedToolCall(entry, encodings)); continue; } if (type == "custom_tool_call") { - request.AddOwnedItem(MakeReplayedCustomToolCall(entry, {})); + request.AddOwnedItem(MakeReplayedCustomToolCall(entry, encodings)); continue; } @@ -467,7 +468,7 @@ static void AddHopOutputToRequest(Request& request, const nlohmann::json& output static void AddChainContextToRequest(Request& request, const ResponseChainContext& context) { for (const auto& hop : context) { request.BeginItemSegment(); - AddJsonItemsToRequest(request, hop.input_items); + AddJsonItemsToRequest(request, hop.input_items, hop.call_encodings); request.BeginItemSegment(); AddHopOutputToRequest(request, hop.output_items, hop.call_encodings); } diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc index 26ec9a1f2..8aed30496 100644 --- a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -2,9 +2,13 @@ // Licensed under the MIT License. #include "exception.h" +#include "contracts/tool_definitions.h" #include "inferencing/generative/chat/chat_template.h" #include "inferencing/generative/chat/chat_transcript.h" #include "inferencing/generative/chat/harmony_decoder.h" +#include "inferencing/generative/openresponses/response_converter.h" +#include "inferencing/generative/openresponses/response_store.h" +#include "inferencing/generative/toolcalling/grammar.h" #include "inferencing/generative/toolcalling/tool_call_context.h" #include "inferencing/generative/toolcalling/tool_call_plan.h" @@ -213,6 +217,122 @@ TEST(HarmonyDecoderTest, PromptProjectionPreservesNativeJsonRawAndResultProvenan EXPECT_TRUE(projected_tools[1]["function"]["native_freeform"].get()); } +TEST(HarmonyDecoderTest, StoredInputAndOutputReplayRetainHistoricalCustomBodyEncoding) { + const std::string body = R"({"input":"literal <|call|> and a\\b"})"; + for (const auto encoding : {GeneratedCallEncoding::kStructured, GeneratedCallEncoding::kNativeJson, + GeneratedCallEncoding::kNativeRaw}) { + for (const bool stored_input : {false, true}) { + SCOPED_TRACE(static_cast(encoding)); + SCOPED_TRACE(stored_input); + const auto call = nlohmann::json{{"type", "custom_tool_call"}, {"call_id", "call_history"}, {"name", "patch"}, {"input", body}}; + auto input = nlohmann::json::array({{{"role", "user"}, {"content", "patch"}}}); + if (stored_input) { + input.push_back(call); + } + const auto output = stored_input ? nlohmann::json::array() : nlohmann::json::array({call}); + ResponseStore store; + store.Store("history", {{"output", output}}, input, "harmony", + ToolCallEncodings{{"call_history", encoding}}); + const auto chain = store.BuildChainContext("history"); + ASSERT_TRUE(chain.has_value()); + responses::ResponseCreateParams params; + params.model = "harmony"; + params.input = "continue"; + auto request = ResponseConverter::ToSessionRequest(params, &*chain); + const auto replay = IngestRequestItems(request.items, request.item_segment_starts); + std::size_t call_count = 0; + for (const auto& message : replay.messages) { + for (const auto* historical : message.ToolCalls()) { + ++call_count; + EXPECT_EQ(historical->generated_encoding, encoding); + EXPECT_EQ(historical->kind, ToolKind::kCustom); + EXPECT_EQ(historical->arguments, body); + } + } + EXPECT_EQ(call_count, 1u); + + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"patch", ToolKind::kCustom}}; + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + const auto projected = nlohmann::json::parse( + BuildHarmonyPromptProjection(replay.messages, context, [](std::string value) { return value; }) + .messages_json); + call_count = 0; + for (const auto& message : projected) { + if (!message.contains("tool_calls")) { + continue; + } + for (const auto& historical : message["tool_calls"]) { + ++call_count; + const auto& function = historical["function"]; + const bool raw = encoding != GeneratedCallEncoding::kNativeJson; + const auto expected = raw ? body : nlohmann::json{{"input", body}}.dump(); + EXPECT_EQ(function["native_freeform"], raw); + EXPECT_EQ(function["arguments_json"], expected); + } + } + EXPECT_EQ(call_count, 1u); + } + } +} + +TEST(HarmonyGrammarTest, ChoicesKeepAuthoritativeHeadersBoundariesAndAllowedRecipients) { + for (const bool reasoning : {false, true}) { + for (const std::string choice : {"auto", "required", "forced", "none"}) { + SCOPED_TRACE(reasoning); + SCOPED_TRACE(choice); + ToolCallContext context; + context.harmony = kProtocol; + context.supports_reasoning = reasoning; + context.text_output = choice == "auto" || choice == "none"; + context.tool_output = choice != "none"; + context.eos_token_ids = {99, kProtocol.handoff, 99}; + if (context.tool_output) { + const auto forced = choice == "forced" + ? std::optional{{"lookup", ToolKind::kFunction}} + : std::nullopt; + context.tool_plan = ToolCallPlan::Create(kTools, true, forced); + } + const auto grammar = BuildLarkGrammar(context); + const std::string mode = choice == "none" ? "commentary* final" + : choice == "auto" ? "commentary* (final | tool)" + : "tool"; + EXPECT_TRUE(grammar.starts_with("start: " + std::string(reasoning ? "reasoning* " : "") + mode + "\n")); + EXPECT_NE(grammar.find("plain_token: <[^10,11,12,13,14,15,16,99]>"), std::string::npos); + EXPECT_NE(grammar.find("final: <[11]> \"final\" <[12]> plain_token* <[15]>"), std::string::npos); + if (context.tool_output) { + EXPECT_NE(grammar.find("metadata <[12]> arguments <[14]>"), std::string::npos); + EXPECT_NE(grammar.find("\" to=\" \"functions.lookup\" <[11]> \"commentary\""), std::string::npos); + EXPECT_NE(grammar.find("<[11]> \"commentary to=\" \"functions.lookup\""), std::string::npos); + EXPECT_EQ(grammar.find("\"functions.patch\"") != std::string::npos, choice != "forced"); + EXPECT_NE(grammar.find("arguments:"), std::string::npos); + } else { + EXPECT_EQ(grammar.find("tool:"), std::string::npos); + EXPECT_EQ(grammar.find("arguments:"), std::string::npos); + } + } + } +} + +TEST(HarmonyGrammarTest, NativePatchBodiesUseTheirGrammarAndUnsupportedFormatsFailExplicitly) { + ToolCallContext context; + context.harmony = kProtocol; + context.tool_output = true; + context.text_output = false; + EXPECT_THROW(BuildLarkGrammar(context), fl::Exception); + context.tool_plan = ToolCallPlan::Create(R"([{"type":"function","function":{ + "name":"apply_patch","parameters":{"type":"object","properties":{"input":{"type":"string"}}}}}])"); + context.tool_kinds = {{"apply_patch", ToolKind::kCustom}}; + context.custom_lark_grammars = {{"apply_patch", std::string(tools::kStockGhcpApplyPatchLarkGrammar)}}; + const auto grammar = BuildLarkGrammar(context); + EXPECT_NE(grammar.find("tool_0: header_0 <[12]> native_patch <[14]>"), std::string::npos); + EXPECT_NE(grammar.find("native_patch:"), std::string::npos); + context.custom_lark_grammars["apply_patch"] = "start: /[\\s\\S]+/"; + EXPECT_THROW(BuildLarkGrammar(context), fl::Exception); +} + TEST(HarmonyDecoderTest, PromptProjectionRejectsNonLeadingInstructionRoles) { for (const auto* tools : {"", kTools}) { ToolCallContext context; From 63f98e0ba9e43085a9ce302aa6771dddfc167d33 Mon Sep 17 00:00:00 2001 From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:21:56 +0000 Subject: [PATCH 15/15] Distinguish recorded call encodings from unannotated caller history Historical generated and stored canonical/JSON/raw calls keep their original body encoding when declarations change. Only unannotated caller history infers the current declaration, preserving existing caller behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/README.md | 6 +- .../generative/chat/chat_template.cc | 1 + .../generative/chat/chat_transcript.cc | 2 +- .../generative/chat/chat_transcript.h | 2 + .../internal_api/chat/harmony_decoder_test.cc | 126 +++++++++++------- 5 files changed, 85 insertions(+), 52 deletions(-) diff --git a/sdk_v2/cpp/README.md b/sdk_v2/cpp/README.md index b0109481a..e2f54b15b 100644 --- a/sdk_v2/cpp/README.md +++ b/sdk_v2/cpp/README.md @@ -418,9 +418,11 @@ answer or one directed tool call. Calls are validated before publication; JSON argument bytes are preserved. Custom tools with the admitted stock patch grammar use native raw bodies; unframed text custom tools retain JSON input-object transport. Both reuse the synthesized `{"input":"raw text"}` object for schema admission and -deliver unchanged text to callers. Recorded native body encodings survive tool +deliver unchanged text to callers. Recorded native and canonical body encodings survive tool definition changes during history replay, including encoding metadata attached to -stored input calls as well as generated output calls. Tool results are correlated by call ID +stored input calls as well as generated output calls. +Unannotated caller-supplied calls continue to use the current declaration's body encoding. +Tool results are correlated by call ID and rendered as named, protected JSON values. Source text and literal marker spellings remain data. Original schemas and custom input formats are supplied as bounded literal prompt information; schema admission remains owned by the existing `ToolCallPlan`. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index aaa090f74..45f10cdae 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -359,6 +359,7 @@ HarmonyPromptProjection BuildHarmonyPromptProjection( const auto raw_body = call->kind == ToolKind::kCustom && (call->generated_encoding == GeneratedCallEncoding::kNativeRaw || (call->generated_encoding == GeneratedCallEncoding::kStructured && + !call->has_recorded_encoding && context.UsesNativeRawBody(call->name))); const auto arguments = raw_body || (call->kind == ToolKind::kFunction && !call->arguments.empty()) ? call->arguments diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc index 8156f4d6a..9a87a556f 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc @@ -86,7 +86,7 @@ GeneratedToolCall MakeGeneratedToolCall(std::string call_id, std::string name, s const bool usable = normalized.has_value(); return {{std::move(call_id), std::move(name), std::move(arguments), - usable ? std::move(*normalized) : nlohmann::ordered_json::object(), kind, encoding}, + usable ? std::move(*normalized) : nlohmann::ordered_json::object(), kind, encoding, true}, usable}; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h index 4dd74b919..9b361fab1 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h @@ -37,6 +37,8 @@ struct TranscriptToolCall { /// as what it was, and so a replayed call normalizes the same way the generated one did. ToolKind kind = ToolKind::kFunction; GeneratedCallEncoding generated_encoding = GeneratedCallEncoding::kStructured; + /// Unannotated caller history follows current declarations; generated and stored calls retain their encoding. + bool has_recorded_encoding = false; }; /// One event within a message, stored in the order it occurred. diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc index 8aed30496..72be8d76c 100644 --- a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -221,63 +221,91 @@ TEST(HarmonyDecoderTest, StoredInputAndOutputReplayRetainHistoricalCustomBodyEnc const std::string body = R"({"input":"literal <|call|> and a\\b"})"; for (const auto encoding : {GeneratedCallEncoding::kStructured, GeneratedCallEncoding::kNativeJson, GeneratedCallEncoding::kNativeRaw}) { - for (const bool stored_input : {false, true}) { - SCOPED_TRACE(static_cast(encoding)); - SCOPED_TRACE(stored_input); - const auto call = nlohmann::json{{"type", "custom_tool_call"}, {"call_id", "call_history"}, {"name", "patch"}, {"input", body}}; - auto input = nlohmann::json::array({{{"role", "user"}, {"content", "patch"}}}); - if (stored_input) { - input.push_back(call); - } - const auto output = stored_input ? nlohmann::json::array() : nlohmann::json::array({call}); - ResponseStore store; - store.Store("history", {{"output", output}}, input, "harmony", - ToolCallEncodings{{"call_history", encoding}}); - const auto chain = store.BuildChainContext("history"); - ASSERT_TRUE(chain.has_value()); - responses::ResponseCreateParams params; - params.model = "harmony"; - params.input = "continue"; - auto request = ResponseConverter::ToSessionRequest(params, &*chain); - const auto replay = IngestRequestItems(request.items, request.item_segment_starts); - std::size_t call_count = 0; - for (const auto& message : replay.messages) { - for (const auto* historical : message.ToolCalls()) { - ++call_count; - EXPECT_EQ(historical->generated_encoding, encoding); - EXPECT_EQ(historical->kind, ToolKind::kCustom); - EXPECT_EQ(historical->arguments, body); + for (const bool current_raw : {false, true}) { + for (const bool stored_input : {false, true}) { + SCOPED_TRACE(static_cast(encoding)); + SCOPED_TRACE(current_raw); + SCOPED_TRACE(stored_input); + const auto call = nlohmann::json{{"type", "custom_tool_call"}, {"call_id", "call_history"}, {"name", "patch"}, {"input", body}}; + auto input = nlohmann::json::array({{{"role", "user"}, {"content", "patch"}}}); + if (stored_input) { + input.push_back(call); } - } - EXPECT_EQ(call_count, 1u); - - ToolCallContext context; - context.harmony = kProtocol; - context.tools_json = kTools; - context.tool_kinds = {{"patch", ToolKind::kCustom}}; - context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; - const auto projected = nlohmann::json::parse( - BuildHarmonyPromptProjection(replay.messages, context, [](std::string value) { return value; }) - .messages_json); - call_count = 0; - for (const auto& message : projected) { - if (!message.contains("tool_calls")) { - continue; + const auto output = stored_input ? nlohmann::json::array() : nlohmann::json::array({call}); + ResponseStore store; + store.Store("history", {{"output", output}}, input, "harmony", + ToolCallEncodings{{"call_history", encoding}}); + const auto chain = store.BuildChainContext("history"); + ASSERT_TRUE(chain.has_value()); + responses::ResponseCreateParams params; + params.model = "harmony"; + params.input = "continue"; + auto request = ResponseConverter::ToSessionRequest(params, &*chain); + const auto replay = IngestRequestItems(request.items, request.item_segment_starts); + std::size_t call_count = 0; + for (const auto& message : replay.messages) { + for (const auto* historical : message.ToolCalls()) { + ++call_count; + EXPECT_EQ(historical->generated_encoding, encoding); + EXPECT_TRUE(historical->has_recorded_encoding); + EXPECT_EQ(historical->kind, ToolKind::kCustom); + EXPECT_EQ(historical->arguments, body); + } + } + EXPECT_EQ(call_count, 1u); + + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"patch", ToolKind::kCustom}}; + if (current_raw) { + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; } - for (const auto& historical : message["tool_calls"]) { - ++call_count; - const auto& function = historical["function"]; - const bool raw = encoding != GeneratedCallEncoding::kNativeJson; - const auto expected = raw ? body : nlohmann::json{{"input", body}}.dump(); - EXPECT_EQ(function["native_freeform"], raw); - EXPECT_EQ(function["arguments_json"], expected); + const auto projected = nlohmann::json::parse( + BuildHarmonyPromptProjection(replay.messages, context, [](std::string value) { return value; }) + .messages_json); + call_count = 0; + for (const auto& message : projected) { + if (!message.contains("tool_calls")) { + continue; + } + for (const auto& historical : message["tool_calls"]) { + ++call_count; + const auto& function = historical["function"]; + const bool raw = encoding == GeneratedCallEncoding::kNativeRaw; + const auto expected = raw ? body : nlohmann::json{{"input", body}}.dump(); + EXPECT_EQ(function["native_freeform"], raw); + EXPECT_EQ(function["arguments_json"], expected); + } } + EXPECT_EQ(call_count, 1u); } - EXPECT_EQ(call_count, 1u); } } } +TEST(HarmonyDecoderTest, UnannotatedCallerHistoryStillUsesTheCurrentCustomBodyDeclaration) { + for (const bool current_raw : {false, true}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"patch", ToolKind::kCustom}}; + if (current_raw) { + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + } + const std::string body = R"({"input":"caller text"})"; + TranscriptMessage assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + assistant.AppendToolCall(MakeSuppliedToolCall("caller", "patch", body, ToolKind::kCustom)); + const auto projected = nlohmann::json::parse( + BuildHarmonyPromptProjection({assistant}, context, [](std::string value) { return value; }) + .messages_json); + const auto& function = projected.back()["tool_calls"][0]["function"]; + EXPECT_EQ(function["native_freeform"], current_raw); + const auto expected = current_raw ? body : nlohmann::json{{"input", body}}.dump(); + EXPECT_EQ(function["arguments_json"], expected); + } +} + TEST(HarmonyGrammarTest, ChoicesKeepAuthoritativeHeadersBoundariesAndAllowedRecipients) { for (const bool reasoning : {false, true}) { for (const std::string choice : {"auto", "required", "forced", "none"}) {