diff --git a/.github/instructions/cpp-tool-calling.instructions.md b/.github/instructions/cpp-tool-calling.instructions.md index 8d8ee38c1..cb835dccf 100644 --- a/.github/instructions/cpp-tool-calling.instructions.md +++ b/.github/instructions/cpp-tool-calling.instructions.md @@ -14,9 +14,13 @@ applyTo: "sdk_v2/cpp/src/inferencing/generative/{chat,toolcalling}/**" contents unrestricted and never use global whitespace ignoring inside strings. - Reasoning and executable tool boundaries use published token IDs. Added tokens may have `special:false`; quoted marker spellings are data. Preserve payload ownership once a canonical or explicit raw envelope has opened. -- Automatic structured output must explicitly choose `MESSAGE\n` plus ordinary text or an executable tool boundary. +- Canonical automatic structured output must explicitly choose `MESSAGE\n` plus ordinary text or an executable tool boundary. Strip text framing incrementally and project it into assistant history. Use whole-turn guidance for automatic turns, including nonreasoning Engine models; region-only masks cannot enforce this initial mode selection. +- Qualified native Harmony uses assistant channel/recipient headers instead of canonical text framing or arrays. + Decode its structure by token identity, seed the prepared assistant-header continuation explicitly, and authorize + a call only at the actual sampled handoff. Reuse the same original-schema plan and publication/recovery sinks. + Terminal-token completion metadata is not a retained token and must never be added to the Engine sequence ledger. - Encode transcript/schema/argument data with the independent literal tokenizer. Only template-owned structure, runtime-owned protocol boundaries, and actual historical call boundaries use ordinary encoding. Encoding protocol boundary exemplars literally teaches the model to echo ordinary text instead of invoking tools. diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 56790c2ee..028d79cc1 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -126,6 +126,15 @@ check_cxx_source_compiles( }" FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI ) +unset(FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN CACHE) +check_cxx_source_compiles( + "#include + int main() { + int32_t token = -1; + return OgaEngineEventGetTerminalTokenId(nullptr, &token) != nullptr; + }" + FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN +) unset(CMAKE_REQUIRED_INCLUDES) unset(CMAKE_REQUIRED_LIBRARIES) if(NOT FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI) @@ -262,6 +271,7 @@ set(FOUNDRY_LOCAL_SOURCES src/inferencing/session/session_manager.cc src/inferencing/session/tool_registry.cc src/inferencing/generative/chat/chat_session.cc + src/inferencing/generative/chat/harmony_decoder.cc src/inferencing/generative/chat/chat_template.cc src/inferencing/generative/chat/chat_transcript.cc src/inferencing/generative/chat/media_input.cc @@ -394,7 +404,8 @@ list(APPEND FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_PUBLIC_HEADERS}) function(foundry_local_configure_target TARGET LINK_SCOPE) target_compile_options(${TARGET} PRIVATE ${FOUNDRY_LOCAL_COMPILE_OPTIONS}) target_compile_definitions(${TARGET} ${LINK_SCOPE} - FOUNDRY_LOCAL_HAS_SHARED_HOST=$) + FOUNDRY_LOCAL_HAS_SHARED_HOST=$ + FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN=$) target_include_directories(${TARGET} PUBLIC diff --git a/sdk_v2/cpp/README.md b/sdk_v2/cpp/README.md index d3eb5ccf9..9f605e2e0 100644 --- a/sdk_v2/cpp/README.md +++ b/sdk_v2/cpp/README.md @@ -347,7 +347,7 @@ The unchanged original parameter schema is validated with jsoncons before any call in the batch is published. Structural guidance alone does not guarantee schema compliance or that the model chose the user's intended argument value. -Automatic tool turns select their response mode explicitly: ordinary assistant +Canonical automatic tool turns select their response mode explicitly: ordinary assistant text starts with an internal `MESSAGE` line, while executable calls start with the model's dedicated tool boundary. The runtime removes the text framing line incrementally; callers still receive normal content and tool-call events. @@ -397,6 +397,46 @@ Recovery also shares implicit classic Generator output limits; it cannot acquire fresh default allowance or use the retained conversation's full context as an output budget. Engine text turns remain uncapped when the caller leaves their limit unset. +### Native Harmony + +GPT-OSS packages use their qualified native template and assistant channel/recipient +headers, not canonical `MESSAGE` framing or tool-call arrays. Model loading checks +the tokenizer's distinct control/literal encodings, EOS configuration, protected +argument/result slots, reasoning/commentary projection, and generation prefix. +The model must use the Engine backend. An incompatible template, tokenizer, or +runtime is rejected rather than falling back to string-based parsing. + +Native support requires matching GenAI headers and libraries exposing +`OgaEngineEventGetTerminalTokenId`, plus the matching CUDA add-on when applicable. +The CMake capability probe checks both compilation and linking. Existing pinned +packages without this API still build for canonical models but cannot load a +qualified GPT-OSS package. Sampled handoff and answer completion are metadata, +not extra retained sequence tokens. + +Each generated turn supports analysis, optional visible commentary, then a final +answer or one directed tool call. Calls are validated before publication; JSON +argument bytes are preserved. Custom tools with the admitted stock patch grammar +use native raw bodies; unframed text custom tools retain JSON input-object transport. +Both reuse the synthesized `{"input":"raw text"}` object for schema admission and +deliver unchanged text to callers. Recorded native and canonical body encodings survive tool +definition changes during history replay, including encoding metadata attached to +stored input calls as well as generated output calls. +Unannotated caller-supplied calls continue to use the current declaration's body encoding. +Tool results are correlated by call ID +and rendered as named, protected JSON values. Source text and literal marker spellings remain data. +Original schemas and custom input formats are supplied as bounded literal prompt +information; schema admission remains owned by the existing `ToolCallPlan`. + +GPT-OSS generation is sequential even when the client permits parallel calls. +Native reasoning effort supports `low`, `medium`, and `high`; model-owned reasoning +cannot be disabled with `enable_thinking` or altered with `preserve_thinking`. +Top-level API effort overrides the generic thinking-switch mapping for this +protocol. Explicit whole-response guidance is incompatible with native Harmony. +Rejected selected calls use the shared one-retry policy with a whole-message +checkpoint and a legal, literal developer correction. + +### Local coding qualification + General-chat function qualification is separate from coding-agent qualification. With a locally registered model already loaded, select the ordinary chat cases explicitly: diff --git a/sdk_v2/cpp/src/contracts/chat_completions_converter.cc b/sdk_v2/cpp/src/contracts/chat_completions_converter.cc index 67ee23946..e3c9bd826 100644 --- a/sdk_v2/cpp/src/contracts/chat_completions_converter.cc +++ b/sdk_v2/cpp/src/contracts/chat_completions_converter.cc @@ -313,6 +313,9 @@ void MapRequestParameters(const ChatCompletionRequest& req, Request& session_req session_request.options["chat_template_kwargs"] = ResolveReasoningTemplateKwargs(req.chat_template_kwargs, req.reasoning_effort).dump(); } + if (req.reasoning_effort.has_value()) { + session_request.options["reasoning_effort"] = *req.reasoning_effort; + } // Extract metadata extensions (matching C# GetTopK/GetRandomSeed) if (req.metadata.has_value()) { diff --git a/sdk_v2/cpp/src/hosting/item_codec.cc b/sdk_v2/cpp/src/hosting/item_codec.cc index 091234241..c29703863 100644 --- a/sdk_v2/cpp/src/hosting/item_codec.cc +++ b/sdk_v2/cpp/src/hosting/item_codec.cc @@ -410,7 +410,7 @@ std::unique_ptr Decode(const Json& value, const ItemDecodingContext& conte ? std::nullopt : std::optional(ReadEnum(declared_kind, 0, static_cast(ToolKind::kCustom))); call->generated_encoding = ReadEnum(value.at("encoding"), 0, - static_cast(GeneratedCallEncoding::kRawEnvelope)); + static_cast(GeneratedCallEncoding::kNativeJson)); call->replayed_from_store = value.at("replayed").get(); call->replayed_arguments = value.at("replayed_arguments").get(); const auto& replayed_kind = value.at("replayed_kind"); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h index 07518087a..6769d1bbe 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h @@ -36,6 +36,7 @@ struct ChatTurnUsage { std::optional finish_reason; std::optional termination_cause; int cached_prompt_tokens = 0; + std::optional terminal_token_id = std::nullopt; }; class RetainedPromptMismatchError : public std::runtime_error { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index a795f2ef5..154dd265a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -12,6 +12,7 @@ #include "inferencing/generative/chat/onnx_chat_generator.h" #include "inferencing/generative/chat/onnx_engine_chat_stream.h" #include "inferencing/generative/chat/prepared_chat_prompt.h" +#include "inferencing/generative/chat/harmony_decoder.h" #include "inferencing/generative/chat/reasoning_stream_splitter.h" #include "inferencing/generative/chat/stop_strings.h" #include "inferencing/generative/genai_model_instance.h" @@ -100,7 +101,9 @@ class AssistantTextOutputFilter { public: AssistantTextOutputFilter(const ToolCallContext& context, const SearchOptions& options) : framer_(context), - stop_filter_(context.UsesAssistantTextFraming() ? options.stop_sequences : std::vector{}) {} + stop_filter_(context.UsesAssistantTextFraming() || context.harmony + ? options.stop_sequences + : std::vector{}) {} std::string Push(std::string_view text) { return stop_filter_.Push(framer_.Push(text)); @@ -170,7 +173,7 @@ TranscriptMessage MakeAssistantMessage(const std::vector& const auto& parsed = std::get(event); auto generated = MakeGeneratedToolCall( parsed.id, parsed.name, parsed.arguments, tool_ctx.KindOf(parsed.name), - parsed.raw_envelope ? GeneratedCallEncoding::kRawEnvelope : GeneratedCallEncoding::kStructured); + tool_ctx.CallEncoding(parsed.name, parsed.raw_envelope)); if (!generated.arguments_usable) { logger.Log(LogLevel::Warning, @@ -847,6 +850,87 @@ std::vector ParseStrictGuidedToolCalls(const std::string& text, return {}; } +GuidedToolRetryResult RunHarmonyToolRetry(const chat_internal::PreparedChatMessages& messages, + const SearchOptions& options, GenAIModelInstance& model, + ToolCallContext context, const Request& request, + const TextChatGeneratorFactory& factory, const std::string& validation_error, + std::optional prompt) { + if (!prompt) prompt = PrepareTextChatPrompt(messages, model, context); + if (prompt->harmony_continuation) { + const auto& continuation = *prompt->harmony_continuation; + prompt->token_ids.resize(prompt->token_ids.size() - continuation.header_tokens); + prompt->prompt.resize(prompt->prompt.size() - continuation.header_bytes); + } + auto& preprocessor = model.GetPreprocessor(); + const auto append = [&](const std::string& text, bool literal) { + const auto encoded = literal ? preprocessor.EncodeLiteral(text.c_str()) : preprocessor.Encode(text.c_str()); + prompt->token_ids.insert(prompt->token_ids.end(), encoded->SequenceData(0), + encoded->SequenceData(0) + encoded->SequenceCount(0)); + prompt->prompt += text; + }; + // The checkpoint ends before the provisional assistant message, not inside its generated header. + append("<|start|>developer<|message|>", false); + const auto diagnostic = nlohmann::json(validation_error).dump(-1, ' ', true); + append("The previous unexecuted tool call was rejected: " + diagnostic.substr(0, 4096) + + ". Emit one corrected tool call satisfying the original schemas and requested values. " + "Do not repeat earlier reasoning or commentary.", + true); + append("<|end|><|start|>assistant", false); + prompt->prompt_token_count = static_cast(prompt->token_ids.size()); + prompt->harmony_continuation = HarmonyContinuation{ + "assistant", context.harmony->assistant_prefix_tokens.size(), + std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + context.supports_reasoning = false; + HarmonyDecoder decoder(*context.harmony, context.tool_plan, *prompt, context.NativeRawToolNames()); + auto generator = + CreatePreparedTextChatGenerator(messages, std::move(*prompt), options, model, context, false, factory); + GuidedToolRetryResult result; + std::string error; + const auto capture = [&](HarmonyDecoder::Output output) { + if (output.malformed) error = std::move(output.error); + for (auto& event : output.events) { + if (auto* call = std::get_if(&event)) { + ToolCallStreamAccumulator::Output admitted; + admitted.events.emplace_back(std::move(*call)); + chat_session_internal::NormalizeToolOutputBatch(admitted, context); + if (admitted.malformed || admitted.events.empty()) { + error = "Guided Harmony recovery failed tool-call normalization"; + continue; + } + result.events.emplace_back(std::move(std::get(admitted.events.front()))); + } else { + error = "Guided Harmony recovery emitted text instead of one corrected tool call"; + } + } + }; + while (!generator->IsDone() && !request.IsCancellationRequested()) { + generator->GenerateNextToken(); + const auto token = generator->CurrentTokenId(); + capture(decoder.Push(token, generator->Decode())); + } + result.canceled = request.IsCancellationRequested(); + if (result.canceled) generator->Cancel(); + const auto usage = generator->GetTurnUsage(); + if (!usage) ThrowInvalidGuidedToolRetry("Harmony recovery did not report Engine usage"); + result.prompt_tokens = usage->prompt_tokens; + result.total_tokens = usage->prompt_tokens + usage->generated_tokens; + result.cached_prompt_tokens = usage->cached_prompt_tokens; + result.finish_reason = usage->finish_reason; + result.termination_cause = usage->termination_cause; + result.canceled |= request.IsCancellationRequested(); + if (result.canceled) { + result.events.clear(); + return result; + } + const bool natural = usage->termination_cause == BackendTerminationCause::kNaturalEnd; + capture(decoder.Finish(usage->terminal_token_id, natural)); + if (!natural || !error.empty() || result.events.size() != 1 || + (options.max_output_tokens && usage->generated_tokens > *options.max_output_tokens)) { + ThrowInvalidGuidedToolRetry(error.empty() ? "Incomplete or over-budget Harmony retry" : error); + } + return result; +} + GuidedToolRetryResult RunGuidedToolRetry(const chat_internal::PreparedChatMessages& messages, const SearchOptions& options, GenAIModelInstance& model, ToolCallContext tool_ctx, const Request& request, @@ -861,6 +945,10 @@ GuidedToolRetryResult RunGuidedToolRetry(const chat_internal::PreparedChatMessag tool_ctx.fenced_tool_call_output = false; tool_ctx.guidance_disabled = false; } + if (tool_ctx.harmony) { + return RunHarmonyToolRetry(messages, options, model, std::move(tool_ctx), request, factory, + validation_error, std::move(retry_prompt)); + } if (retry_prompt) { tool_ctx.supports_reasoning = false; } @@ -1087,6 +1175,7 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, GenAIModelInstance& model, ILogger& logger) { ToolCallContext tool_ctx; + tool_ctx.harmony = model.GetHarmonyProtocol(); tool_ctx.tool_call_start = GetOptionOrEmpty(effective_options, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); tool_ctx.tool_call_end = GetOptionOrEmpty(effective_options, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); @@ -1097,9 +1186,26 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "parallel_tool_calls must be true or false"); } tool_ctx.template_kwargs_json = GetOptionOrEmpty(effective_options, "chat_template_kwargs"); - if (!tool_ctx.template_kwargs_json.empty()) { + const auto api_reasoning_effort = GetOptionOrEmpty(effective_options, "reasoning_effort"); + if (!tool_ctx.template_kwargs_json.empty() || (tool_ctx.harmony && !api_reasoning_effort.empty())) { tool_ctx.template_kwargs_json = NormalizeChatTemplateKwargs(tool_ctx.template_kwargs_json); - const auto kwargs = nlohmann::json::parse(tool_ctx.template_kwargs_json); + auto kwargs = nlohmann::json::parse(tool_ctx.template_kwargs_json); + if (tool_ctx.harmony && !api_reasoning_effort.empty()) { + kwargs["reasoning_effort"] = api_reasoning_effort; + kwargs.erase("enable_thinking"); + } + if (tool_ctx.harmony && (kwargs.contains("enable_thinking") || kwargs.contains("preserve_thinking"))) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "native Harmony uses model-owned reasoning; enable_thinking and preserve_thinking are unsupported"); + } + if (tool_ctx.harmony) { + const auto effort = kwargs.find("reasoning_effort"); + if (effort != kwargs.end() && (!effort->is_string() || + (*effort != "low" && *effort != "medium" && *effort != "high"))) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "native Harmony supports low, medium, or high reasoning effort"); + } + tool_ctx.template_kwargs_json = kwargs.dump(); + } const bool uses_reasoning_controls = kwargs.contains("enable_thinking") || kwargs.contains("preserve_thinking") || kwargs.contains("reasoning_effort"); @@ -1228,10 +1334,11 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, ApplyToolChoiceToContext(tool_choice, tool_ctx); if (tool_ctx.tool_output) { - if (!tool_ctx.HasToolCallTokens() && !tool_ctx.ActiveRawEnvelope()) { + if (!tool_ctx.harmony && !tool_ctx.HasToolCallTokens() && !tool_ctx.ActiveRawEnvelope()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "structured tool calling requires model-published opening and closing tool markers"); } + if (tool_ctx.harmony) tool_ctx.parallel_tool_calls = false; tool_ctx.tool_plan = ToolCallPlan::Create(tool_ctx.tools_json, tool_ctx.parallel_tool_calls, tool_ctx.forced_tool); tool_ctx.eos_token_ids = model.GetPreprocessor().GetEosTokenIds(); @@ -1247,6 +1354,20 @@ ToolCallContext BuildToolCallContextForRequest(const Request& request, } } + if (tool_ctx.harmony) { + if (tool_ctx.HasAnyExplicitGuidance()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Explicit whole-response guidance is incompatible with native Harmony"); + } + tool_ctx.eos_token_ids = model.GetPreprocessor().GetEosTokenIds(); + tool_ctx.supports_reasoning = true; + tool_ctx.tool_call_start.clear(); + tool_ctx.tool_call_end.clear(); + tool_ctx.tool_call_start_token_id.reset(); + tool_ctx.tool_call_end_token_id.reset(); + tool_ctx.reasoning_start.clear(); + tool_ctx.reasoning_end.clear(); + } chat_session_internal::ApplyRawEnvelopeGuidance(tool_ctx, logger); return tool_ctx; @@ -1577,12 +1698,11 @@ void ChatSession::ProcessGeneratedOutput(std::vector event flush_segments(); auto& call = std::get(event); const auto kind = tool_ctx.KindOf(call.name); + const auto encoding = tool_ctx.CallEncoding(call.name, call.raw_envelope); response.items.push_back(std::make_unique(std::move(call.id), std::move(call.name), std::move(call.arguments), /*replayed_from_store=*/false, kind, std::nullopt, - call.raw_envelope - ? GeneratedCallEncoding::kRawEnvelope - : GeneratedCallEncoding::kStructured)); + encoding)); has_tool_calls = true; } flush_segments(); @@ -1632,6 +1752,11 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) auto& prepared_messages = *prepared->messages; ToolCallRecoveryCheckpoint recovery_checkpoint(prepared->prompt, turn_tool_ctx.tool_call_start_token_id, turn_tool_ctx.tool_plan != nullptr && !media_turn); + std::optional harmony_decoder; + if (turn_tool_ctx.harmony) { + harmony_decoder.emplace(*turn_tool_ctx.harmony, turn_tool_ctx.tool_plan, prepared->prompt, + turn_tool_ctx.NativeRawToolNames()); + } int prompt_tokens = 0; // Empty until this turn's input is appended to an existing generator. A rebuilt generator bakes the input into its @@ -1739,7 +1864,9 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int output_tokens = 0; StopStringFilter stop_filter(effective_options.stop_sequences); auto* active_stop_filter = - effective_options.stop_sequences.empty() || cached_tool_ctx_.UsesAssistantTextFraming() ? nullptr : &stop_filter; + effective_options.stop_sequences.empty() || cached_tool_ctx_.UsesAssistantTextFraming() || harmony_decoder + ? nullptr + : &stop_filter; bool stop_sequence_matched = false; bool host_output_limit_reached = false; std::vector generated_events; @@ -1830,7 +1957,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) streaming_callback->PushItem(std::make_unique( call.id, call.name, call.arguments, /*replayed_from_store=*/false, cached_tool_ctx_.KindOf(call.name), std::nullopt, - call.raw_envelope ? GeneratedCallEncoding::kRawEnvelope : GeneratedCallEncoding::kStructured)); + cached_tool_ctx_.CallEncoding(call.name, call.raw_envelope))); } generated_events.push_back(std::move(call)); } @@ -1890,13 +2017,16 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) recovery_checkpoint.Observe(token_id, token, splitter.InsideReasoning()); ++output_tokens; - if (chat_session_internal::PushDecodedFragment( - token, token_id, active_stop_filter, splitter, emit_segments, - cached_tool_ctx_.tool_plan && inside_payload) || - stop_sequence_matched) { + if (harmony_decoder) { + DeliverHarmonyOutput(harmony_decoder->Push(token_id, token), emit_segments, emit_tool_output); + } else if (chat_session_internal::PushDecodedFragment( + token, token_id, active_stop_filter, splitter, emit_segments, + cached_tool_ctx_.tool_plan && inside_payload) || + stop_sequence_matched) { stop_sequence_matched = true; break; } + if (stop_sequence_matched) break; if (host_max_output.has_value() && chat_session_internal::DidHostOutputLimitTruncate( @@ -1932,19 +2062,26 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int cached_prompt_tokens = 0; std::optional backend_finish_reason; std::optional backend_termination; + std::optional terminal_token; if (const auto turn_usage = cached_generator_->GetTurnUsage()) { prompt_tokens = turn_usage->prompt_tokens; total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; cached_prompt_tokens = turn_usage->cached_prompt_tokens; backend_finish_reason = turn_usage->finish_reason; backend_termination = turn_usage->termination_cause; + terminal_token = turn_usage->terminal_token_id; } const bool natural_tool_output_end = !turn_guard.TurnEnded() && chat_session_internal::IsNaturalToolOutputEnd( request.IsCancellationRequested(), stop_sequence_matched, host_output_limit_reached, backend_termination); - flush_accumulator(natural_tool_output_end); + if (harmony_decoder) { + DeliverHarmonyOutput( + harmony_decoder->Finish(terminal_token, natural_tool_output_end), emit_segments, emit_tool_output); + } else { + flush_accumulator(natural_tool_output_end); + } emit_visible_text(text_framer.Flush()); if (natural_tool_output_end && cached_tool_ctx_.tool_plan && !cached_tool_ctx_.text_output && !cached_tool_ctx_.ActiveRawEnvelope() && !malformed_tool_output_seen && @@ -1952,7 +2089,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) malformed_tool_output_seen = true; malformed_tool_error = "The model did not emit the required tool call"; } - int accepted_reasoning_tokens = splitter.ReasoningTokenCount(); + int accepted_reasoning_tokens = + harmony_decoder ? harmony_decoder->ReasoningTokenCount() : splitter.ReasoningTokenCount(); if (streaming_callback) { streaming_callback->DrainPending(); @@ -1969,7 +2107,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) bool recovered_tool_output = false; ToolCallContext completed_tool_ctx; if (malformed_tool_output_seen) { - const bool preserve_prefix = recovery_checkpoint.HasBoundary(); + const bool preserve_prefix = + harmony_decoder ? harmony_decoder->HasRecoveryCheckpoint() : recovery_checkpoint.HasBoundary(); const bool tool_call_published = HasGeneratedToolCalls(generated_events); const int attempted_tokens = std::max(output_tokens, total_tokens - prompt_tokens); const bool recovery_eligible = IsGuidedToolRetryEligible( @@ -1999,7 +2138,10 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) prepared_messages, retry_options, Model(), std::move(retry_tool_ctx), request, text_generator_factory_, malformed_tool_error, - preserve_prefix ? std::optional(recovery_checkpoint.TakePrompt()) : std::nullopt); + preserve_prefix ? std::optional( + harmony_decoder ? harmony_decoder->TakeRecoveryPrompt() + : recovery_checkpoint.TakePrompt()) + : std::nullopt); if (!preserve_prefix) { generated_events.clear(); @@ -2021,7 +2163,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) if (streaming_callback) { streaming_callback->PushItem(std::make_unique( call.id, call.name, call.arguments, /*replayed_from_store=*/false, - completed_tool_ctx.KindOf(call.name), std::nullopt, GeneratedCallEncoding::kStructured)); + completed_tool_ctx.KindOf(call.name), std::nullopt, + completed_tool_ctx.CallEncoding(call.name, call.raw_envelope))); } } @@ -2117,6 +2260,9 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons const auto& options = prepared.options; ToolCallRecoveryCheckpoint recovery_checkpoint(prepared.prompt, tool_ctx.tool_call_start_token_id, tool_ctx.tool_plan != nullptr); + std::optional harmony_decoder; + if (tool_ctx.harmony) + harmony_decoder.emplace(*tool_ctx.harmony, tool_ctx.tool_plan, prepared.prompt, tool_ctx.NativeRawToolNames()); const auto& prepared_messages = *prepared.messages; std::unique_ptr generator; if (prepared.prompt.HasMedia()) { @@ -2287,7 +2433,7 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons // Generate token-by-token. StopStringFilter stop_filter(options.stop_sequences); auto* active_stop_filter = - options.stop_sequences.empty() || tool_ctx.UsesAssistantTextFraming() ? nullptr : &stop_filter; + options.stop_sequences.empty() || tool_ctx.UsesAssistantTextFraming() || harmony_decoder ? nullptr : &stop_filter; while (!generator->IsDone() && !original_request.IsCancellationRequested() && !turn_guard.TurnEnded()) { generator->GenerateNextToken(); const auto token_id = generator->CurrentTokenId(); @@ -2296,13 +2442,16 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons splitter.InsideReasoning(), inside_payload); recovery_checkpoint.Observe(token_id, token, splitter.InsideReasoning()); - if (chat_session_internal::PushDecodedFragment( - token, token_id, active_stop_filter, splitter, process_segments, - tool_ctx.tool_plan && inside_payload) || - stop_sequence_matched) { + if (harmony_decoder) { + DeliverHarmonyOutput(harmony_decoder->Push(token_id, token), process_segments, process_tool_output); + } else if (chat_session_internal::PushDecodedFragment( + token, token_id, active_stop_filter, splitter, process_segments, + tool_ctx.tool_plan && inside_payload) || + stop_sequence_matched) { stop_sequence_matched = true; break; } + if (stop_sequence_matched) break; } // Drain any buffered partial-marker bytes at end-of-stream. Reasoning splitter first so any final DEFAULT bytes @@ -2329,28 +2478,36 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons int cached_prompt_tokens = 0; std::optional backend_finish_reason; std::optional backend_termination; + std::optional terminal_token; if (const auto turn_usage = generator->GetTurnUsage()) { prompt_tokens = turn_usage->prompt_tokens; total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; cached_prompt_tokens = turn_usage->cached_prompt_tokens; backend_finish_reason = turn_usage->finish_reason; backend_termination = turn_usage->termination_cause; + terminal_token = turn_usage->terminal_token_id; } const bool natural_tool_output_end = !turn_guard.TurnEnded() && chat_session_internal::IsNaturalToolOutputEnd( original_request.IsCancellationRequested(), stop_sequence_matched, /*host_output_limit_reached=*/false, backend_termination); - process_tool_output(chat_session_internal::FlushToolOutput( - active_raw_detector, tool_accumulator, natural_tool_output_end, - tool_ctx.fenced_tool_call_output)); + if (harmony_decoder) { + DeliverHarmonyOutput( + harmony_decoder->Finish(terminal_token, natural_tool_output_end), process_segments, process_tool_output); + } else { + process_tool_output( + chat_session_internal::FlushToolOutput(active_raw_detector, tool_accumulator, natural_tool_output_end, + tool_ctx.fenced_tool_call_output)); + } emit_visible_text(text_framer.Flush()); if (natural_tool_output_end && tool_ctx.tool_plan && !tool_ctx.text_output && !tool_ctx.ActiveRawEnvelope() && !malformed_tool_output_seen && !HasGeneratedToolCalls(generated_events)) { malformed_tool_output_seen = true; malformed_tool_error = "The model did not emit the required tool call"; } - int accepted_reasoning_tokens = splitter.ReasoningTokenCount(); + int accepted_reasoning_tokens = + harmony_decoder ? harmony_decoder->ReasoningTokenCount() : splitter.ReasoningTokenCount(); if (streaming_callback) { streaming_callback->DrainPending(); @@ -2365,7 +2522,8 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons text_framer.ValidateBoundary(); if (malformed_tool_output_seen) { - const bool preserve_prefix = recovery_checkpoint.HasBoundary(); + const bool preserve_prefix = + harmony_decoder ? harmony_decoder->HasRecoveryCheckpoint() : recovery_checkpoint.HasBoundary(); const bool tool_call_published = HasGeneratedToolCalls(generated_events); const auto attempted_tokens = total_tokens - prompt_tokens; const auto turn_output_budget = prepared.host_max_output_tokens @@ -2395,7 +2553,10 @@ void ChatSession::ProcessChatCompletionsJson(PreparedChatRequest& prepared, cons prepared_messages, retry_options, Model(), tool_ctx, original_request, text_generator_factory_, malformed_tool_error, - preserve_prefix ? std::optional(recovery_checkpoint.TakePrompt()) : std::nullopt); + preserve_prefix ? std::optional( + harmony_decoder ? harmony_decoder->TakeRecoveryPrompt() + : recovery_checkpoint.TakePrompt()) + : std::nullopt); if (!preserve_prefix) { generated_events.clear(); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index 853f7cdbb..1bda33cb9 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/prepared_chat_prompt.h" #include "exception.h" #include "inferencing/generative/genai_model_instance.h" #include "inferencing/generative/toolcalling/tool_call_context.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" #include "items/message_item.h" #include "items/text_item.h" #include "utils.h" @@ -303,8 +305,10 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& } if (const auto* raw = tool_ctx.ActiveRawEnvelope()) { const auto raw_instructions = - "\nThe custom tool \"" + raw->tool_name + "\" also accepts a raw envelope instead of a JSON call. " - "To call this tool in raw form, start with \"" + raw->start_marker + + "\nThe custom tool \"" + raw->tool_name + + "\" also accepts a raw envelope instead of a JSON call. " + "To call this tool in raw form, start with \"" + + raw->start_marker + "\" on its own line, then the exact tool payload, and finish with \"" + raw->end_marker + "\" on its own line. Include both marker lines in the raw input. " "Do not wrap the raw envelope in tool boundary tokens, JSON, or Markdown. " @@ -332,14 +336,189 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& return projected.dump(); } +HarmonyPromptProjection BuildHarmonyPromptProjection( + const std::vector& messages, const ToolCallContext& context, + const std::function& protect_literal) { + using Json = nlohmann::ordered_json; + auto projected = Json::array(); + const auto validate_tool_name = [](std::string_view name) { + if (name.empty() || name.find_first_of(" \t\r\n\v\f\x01\x02") != std::string_view::npos) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Harmony tool names must not contain header separators"); + } + }; + std::unordered_map call_names; + for (const auto& message : messages) { + if (!projected.empty() && + (message.role == FOUNDRY_LOCAL_ROLE_SYSTEM || message.role == FOUNDRY_LOCAL_ROLE_DEVELOPER)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Harmony requires system or developer instructions to be the first message"); + } + Json entry{{"role", Utils::RoleToString(message.role)}, {"content", protect_literal(message.VisibleText())}}; + if (message.role == FOUNDRY_LOCAL_ROLE_ASSISTANT) { + entry["thinking"] = protect_literal(message.ReasoningText()); + auto calls = Json::array(); + for (const auto* call : message.ToolCalls()) { + validate_tool_name(call->name); + if (!call_names.emplace(call->call_id, call->name).second) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Duplicate Harmony history call ID"); + } + const auto raw_body = call->kind == ToolKind::kCustom && + (call->generated_encoding == GeneratedCallEncoding::kNativeRaw || + (call->generated_encoding == GeneratedCallEncoding::kStructured && + !call->has_recorded_encoding && + context.UsesNativeRawBody(call->name))); + const auto arguments = raw_body || (call->kind == ToolKind::kFunction && !call->arguments.empty()) + ? call->arguments + : call->normalized_arguments.dump(); + calls.push_back( + {{"id", call->call_id}, + {"type", "function"}, + {"function", + {{"name", protect_literal(call->name)}, + {"arguments", Json::object()}, + {"arguments_json", protect_literal(arguments)}, + {"native_freeform", raw_body}}}}); + } + + if (!calls.empty()) entry["tool_calls"] = std::move(calls); + } else if (message.role == FOUNDRY_LOCAL_ROLE_TOOL) { + const auto found = call_names.find(message.tool_call_id); + if (found == call_names.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Harmony result has no matching history call ID"); + } + entry["name"] = protect_literal(found->second); + entry["content"] = ""; + entry["content_json"] = protect_literal(Json(message.VisibleText()).dump()); + } + + projected.push_back(std::move(entry)); + } + + if (context.tools_json.size() > ToolCallPlan::kMaxSchemaBytes) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Native tool schema supplement exceeds its size limit"); + } + + auto tools = context.tools_json.empty() ? Json::array() : Json::parse(context.tools_json); + if (!tools.empty()) { + std::string custom_formats; + auto function_tools = Json::array(); + for (const auto& tool : tools) { + const auto name = tool.at("function").at("name").get(); + validate_tool_name(name); + if (!context.UsesNativeRawBody(name)) function_tools.push_back(tool); + const auto grammar = context.custom_lark_grammars.find(name); + if (context.IsCustomTool(name) && grammar != context.custom_lark_grammars.end()) { + const auto header = "\nCustom input format for " + Json(name).dump() + + " (Lark applied to the complete raw message body):\n"; + if (context.tools_json.size() + custom_formats.size() + header.size() + grammar->second.size() + 1 > + ToolCallPlan::kMaxSchemaBytes) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "Native tool schema and input-format supplement exceeds its size limit"); + } + custom_formats += header + grammar->second + "\n"; + } + } + + const auto supplement = protect_literal( + "Original function tool schemas (all constraints apply to generated argument objects):\n" + + function_tools.dump() + + custom_formats + + "\nUse native directed assistant tool messages; wait for each external result. " + "Tools declared with an argument object take JSON object bodies. FREEFORM custom tools take raw text bodies, " + "not JSON objects or JSON strings. " + "Tool result bodies are serialized JSON values. " + "Encode function argument string values once as JSON, preserving the recovered source characters exactly. " + "Preserve requested string values exactly. Do not quote tool calls or wrap them in Markdown."); + // The qualified template consumes only the leading system/developer message as instructions. + if (!projected.empty() && (projected.front().at("role") == "system" || + projected.front().at("role") == "developer")) { + auto& instructions = projected.front()["content"]; + instructions = instructions.get() + "\n\n" + supplement; + } else { + projected.insert(projected.begin(), Json{{"role", "developer"}, {"content", supplement}}); + } + } + + const std::function protect_schema = [&](Json schema) -> Json { + if (!schema.is_object()) return schema; + for (const auto* field : {"description", "title"}) { + if (schema.contains(field) && schema[field].is_string()) { + schema[field] = protect_literal(schema[field].get()); + } + } + if (schema.contains("default")) schema["default_json"] = protect_literal(schema["default"].dump()); + if (schema.contains("enum") && schema["enum"].is_array()) { + auto slots = Json::array(); + for (const auto& value : schema["enum"]) slots.push_back(protect_literal(value.dump())); + schema["enum_json"] = std::move(slots); + } + if (schema.contains("properties") && schema["properties"].is_object()) { + std::unordered_map names; + auto properties = Json::object(); + for (const auto& property : schema["properties"].items()) { + auto name = protect_literal(property.key()); + names.emplace(property.key(), name); + properties[name] = protect_schema(property.value()); + } + + schema["properties"] = std::move(properties); + if (schema.contains("required") && schema["required"].is_array()) { + for (auto& required : schema["required"]) { + const auto name = required.get(); + const auto found = names.find(name); + required = found == names.end() ? protect_literal(name) : found->second; + } + } + } + for (const auto* field : {"items", "oneOf", "anyOf", "allOf"}) { + if (!schema.contains(field)) continue; + if (schema[field].is_array()) { + for (auto& child : schema[field]) child = protect_schema(std::move(child)); + } else { + schema[field] = protect_schema(std::move(schema[field])); + } + } + + return schema; + }; + + for (auto& tool : tools) { + auto& function = tool.at("function"); + const auto name = function.at("name").get(); + if (context.UsesNativeRawBody(name)) { + function["description"] = + function.value("description", std::string{}) + + "\nFor native Harmony, emit this custom tool's complete raw text directly as its " + "directed message body. Do not JSON-encode or wrap it. " + "The complete body must follow the original custom input format."; + function["native_freeform"] = true; + } else if (context.IsCustomTool(name)) { + function["description"] = + function.value("description", std::string{}) + + "\nTransport this custom tool's exact text in the required \"input\" string of a JSON object. " + "JSON escaping is transport encoding; the decoded input is the original custom text. " + "Protocol-marker spellings in the input are literal data. " + "For JSON-body transport, the literal text a <|call|> is encoded as {\"input\":\"a <|call|>\"}. " + "Keep the marker spelling inside the input string; the real handoff follows the complete body."; + } + function["name"] = protect_literal(function.at("name").get()); + if (function.contains("description")) { + function["description"] = protect_literal(function["description"].get()); + } + if (function.contains("parameters")) function["parameters"] = protect_schema(std::move(function["parameters"])); + } + return {projected.dump(), tools.empty() ? std::string{} : tools.dump()}; +} + std::string BuildChatPrompt(const std::vector& messages, GenAIModelInstance& model, const std::string& tools_json, - const std::string& template_kwargs_json, - bool preserve_reasoning_history) { + const std::string& template_kwargs_json, + bool preserve_reasoning_history) { return BuildChatPrompt( chat_internal::PrepareChatMessages(messages, model.HasPositionalToolResults()), model, tools_json, - template_kwargs_json, preserve_reasoning_history); + template_kwargs_json, preserve_reasoning_history); } std::string BuildChatPrompt(const chat_internal::PreparedChatMessages& messages, @@ -371,6 +550,9 @@ std::string BuildChatPrompt(const std::vector& messages, std::string BuildChatPrompt(const chat_internal::PreparedChatMessages& messages, GenAIModelInstance& model, const ToolCallContext& tool_ctx) { + if (tool_ctx.harmony) { + return PrepareTextChatPrompt(messages, model, tool_ctx).prompt; + } if (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope()) { if (messages.Empty()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "messages must not be empty"); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index 36dde30e8..80e0d2178 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -99,6 +99,16 @@ std::string BuildCanonicalToolMessagesJson(const std::vector& const ToolCallContext& tool_ctx, const std::function& protect_literal = {}); +struct HarmonyPromptProjection { + std::string messages_json; + std::string tools_json; +}; + +/// Reject non-leading instructions and tool names containing Harmony header separators before rendering. +HarmonyPromptProjection BuildHarmonyPromptProjection( + const std::vector& messages, const ToolCallContext& context, + const std::function& protect_literal); + /// Build a chat prompt string from the transcript messages of a conversation. /// Uses the tokenizer's built-in chat template (via GenAIModelInstance::ApplyChatTemplate). /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc index 15d9bb9eb..9a87a556f 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc @@ -58,7 +58,7 @@ std::optional ParseToolCallArguments(const std::string& /// /// kFunction defers to ParseToolCallArguments and can fail. kCustom never fails: the payload is free-form text by /// definition, so it is wrapped as `{"input": }` — the exact shape the synthesized custom-tool schema -/// asks the model for, and therefore the shape a replayed call must render back to. Wrapping unconditionally keeps +/// uses for admission. Native raw-body history bypasses this object during rendering. Wrapping unconditionally keeps /// the rule unambiguous: a payload that happens to look like `{"input": "..."}` is still just text. static std::optional NormalizeToolCallArguments(ToolKind kind, const std::string& arguments) { @@ -86,7 +86,7 @@ GeneratedToolCall MakeGeneratedToolCall(std::string call_id, std::string name, s const bool usable = normalized.has_value(); return {{std::move(call_id), std::move(name), std::move(arguments), - usable ? std::move(*normalized) : nlohmann::ordered_json::object(), kind, encoding}, + usable ? std::move(*normalized) : nlohmann::ordered_json::object(), kind, encoding, true}, usable}; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h index dbce3dac8..9b361fab1 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h @@ -30,13 +30,15 @@ struct TranscriptToolCall { std::string arguments; /// Object form used for template projection. Always a JSON object, so projecting a committed transcript can never /// fail. For kFunction this is the parsed `arguments`, or an empty object when model output failed to produce one. - /// For kCustom it is the payload rewrapped as `{"input": arguments}` — the exact shape the model emitted and the - /// only shape the chat template and the tool-call grammar understand. + /// For kCustom it is the payload rewrapped as `{"input": arguments}` for schema admission and JSON transport. + /// A native raw call instead replays `arguments` using its recorded encoding. nlohmann::ordered_json normalized_arguments = nlohmann::ordered_json::object(); /// Which of the session's tool kinds produced this call. Recorded so the authoritative transcript reports the call /// as what it was, and so a replayed call normalizes the same way the generated one did. ToolKind kind = ToolKind::kFunction; GeneratedCallEncoding generated_encoding = GeneratedCallEncoding::kStructured; + /// Unannotated caller history follows current declarations; generated and stored calls retain their encoding. + bool has_recorded_encoding = false; }; /// One event within a message, stored in the order it occurred. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc new file mode 100644 index 000000000..2a58c788c --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.cc @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "inferencing/generative/chat/harmony_decoder.h" + +#include "exception.h" +#include "inferencing/generative/toolcalling/tool_call_stream_accumulator.h" + +#include +#include +#include +#include + +namespace fl { +namespace { + +bool HasRecipientField(std::string_view header) { + size_t position = 0; + while ((position = header.find("to=", position)) != std::string_view::npos) { + if (position == 0 || std::string_view(" \t\r\n\x01\x02").find(header[position - 1]) != std::string_view::npos) { + return true; + } + position += 3; + } + return false; +} + +} // namespace + +HarmonyDecoder::HarmonyDecoder(HarmonyProtocol protocol, std::shared_ptr plan, + const PreparedChatPrompt& prompt, + std::unordered_set raw_tool_names) + : protocol_(std::move(protocol)), plan_(std::move(plan)), raw_tool_names_(std::move(raw_tool_names)) { + if (!prompt.harmony_continuation) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Harmony decoding requires an explicit prepared continuation"); + } + const auto& continuation = *prompt.harmony_continuation; + if (continuation.header_tokens > prompt.token_ids.size() || continuation.header_bytes > prompt.prompt.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Invalid prepared Harmony header extent"); + } + header_ = continuation.header; + checkpoint_.prompt = prompt.prompt; + checkpoint_.token_ids = prompt.token_ids; + message_token_offset_ = prompt.token_ids.size() - continuation.header_tokens; + message_byte_offset_ = prompt.prompt.size() - continuation.header_bytes; +} + +HarmonyDecoder::Output HarmonyDecoder::Fail(std::string error) { + failed_ = true; + arguments_.clear(); + return {{}, true, std::move(error)}; +} + +void HarmonyDecoder::CaptureCheckpoint() { + if (!checkpoint_captured_ && plan_) { + checkpoint_.token_ids.resize(message_token_offset_); + checkpoint_.prompt.resize(message_byte_offset_); + checkpoint_captured_ = true; + } +} + +void HarmonyDecoder::RecordToken(int32_t token, const std::string& text) { + if (checkpoint_captured_ || !plan_) { + return; + } + if (token == protocol_.start) { + message_token_offset_ = checkpoint_.token_ids.size(); + message_byte_offset_ = checkpoint_.prompt.size(); + } + checkpoint_.token_ids.push_back(token); + if (token == protocol_.start) + checkpoint_.prompt += "<|start|>"; + else if (token == protocol_.channel) + checkpoint_.prompt += "<|channel|>"; + else if (token == protocol_.message) + checkpoint_.prompt += "<|message|>"; + else if (token == protocol_.end) + checkpoint_.prompt += "<|end|>"; + else if (token == protocol_.format) + checkpoint_.prompt += "<|constrain|>"; + else + checkpoint_.prompt += text; +} + +void HarmonyDecoder::ParseHeader() { + if (HasRecipientField(header_)) { + CaptureCheckpoint(); + } + if (std::count(header_.begin(), header_.end(), '\x01') != 1 || + std::count(header_.begin(), header_.end(), '\x02') > 1) { + throw std::runtime_error("Harmony header requires exactly one channel"); + } + const auto channel_position = header_.find('\x01'); + const auto format_position = header_.find('\x02'); + if (format_position != std::string::npos && format_position < channel_position) { + throw std::runtime_error("Harmony format metadata must follow the channel"); + } + std::istringstream prefix(header_.substr(0, channel_position)); + std::string field; + if (!(prefix >> field) || field != "assistant") { + throw std::runtime_error("Harmony output participant must be assistant"); + } + channel_.clear(); + recipient_.clear(); + const auto read_recipient = [&](const std::string& value) { + if (!recipient_.empty() || !value.starts_with("to=functions.") || value.size() == 13) { + throw std::runtime_error("Invalid Harmony tool recipient"); + } + recipient_ = value.substr(13); + }; + while (prefix >> field) read_recipient(field); + std::istringstream channel(header_.substr(channel_position + 1, + format_position == std::string::npos + ? std::string::npos + : format_position - channel_position - 1)); + if (!(channel >> channel_) || + (channel_ != "analysis" && channel_ != "commentary" && channel_ != "final")) { + throw std::runtime_error("Unsupported Harmony channel"); + } + bool has_format = false; + while (channel >> field) { + if (field == "json" && !recipient_.empty() && format_position == std::string::npos) { + has_format = true; + if (channel >> field) throw std::runtime_error("Unexpected field after Harmony format metadata"); + } else { + read_recipient(field); + } + } + if (format_position != std::string::npos) { + has_format = true; + std::istringstream format(header_.substr(format_position + 1)); + if (!(format >> field) || field != "json" || (format >> field)) { + throw std::runtime_error("Unsupported Harmony format metadata"); + } + } + if ((!recipient_.empty() && channel_ != "commentary") || + (has_format && recipient_.empty()) || (channel_ == "analysis" && visible_started_)) { + throw std::runtime_error("Unsupported Harmony assistant-channel transition"); + } + if (has_format && raw_tool_names_.contains(recipient_)) { + throw std::runtime_error("Native custom tools require a raw body without JSON metadata"); + } + visible_started_ |= channel_ != "analysis"; +} + +HarmonyDecoder::Output HarmonyDecoder::Push(std::optional token, const std::string& text) { + if (failed_) return {}; + if (!token) return text.empty() ? Output{} : Fail("Harmony decoding requires token identity"); + if (phase_ == Phase::kDone) { + return token == ending_ ? Output{} : Fail("Output after Harmony turn completion"); + } + RecordToken(*token, text); + try { + if (*token == protocol_.start) { + if (phase_ != Phase::kBetween) return Fail("Harmony message started before the previous message ended"); + header_.clear(); + phase_ = Phase::kHeader; + } else if (*token == protocol_.channel || *token == protocol_.format) { + if (phase_ != Phase::kHeader) return Fail("Harmony header control in a message body"); + header_ += *token == protocol_.channel ? '\x01' : '\x02'; + } else if (*token == protocol_.message) { + if (phase_ != Phase::kHeader) return Fail("Unexpected Harmony message-body boundary"); + ParseHeader(); + phase_ = Phase::kBody; + } else if (*token == protocol_.end) { + if (phase_ != Phase::kBody || !recipient_.empty() || channel_ == "final") { + return Fail("Harmony message completion cannot authorize a tool call or final answer"); + } + phase_ = Phase::kBetween; + } else if (*token == protocol_.handoff) { + if (phase_ != Phase::kBody || recipient_.empty() || !plan_) { + return Fail("Harmony handoff requires a selected declared tool"); + } + ParsedToolCall call; + call.name = recipient_; + call.arguments = raw_tool_names_.contains(recipient_) + ? nlohmann::json{{"input", arguments_}}.dump() + : arguments_; + call.argument_source = call.arguments; + plan_->ValidateBatch({call}); + call.id = GenerateToolCallId(); + phase_ = Phase::kDone; + ending_ = token; + return {{std::move(call)}, false, {}}; + } else if (*token == protocol_.answer) { + if (phase_ != Phase::kBody || channel_ != "final" || !recipient_.empty()) { + return Fail("Harmony answer completion requires a final message"); + } + phase_ = Phase::kDone; + ending_ = token; + } else if (phase_ == Phase::kHeader) { + if (HasRecipientField(header_ + text.substr(0, 1025))) { + CaptureCheckpoint(); + } + if (text.find_first_of("\x01\x02") != std::string::npos) { + return Fail("Ordinary header bytes cannot manufacture Harmony metadata controls"); + } + if (header_.size() + text.size() > 1024) return Fail("Harmony header exceeds its size limit"); + header_ += text; + } else if (phase_ == Phase::kBody && !recipient_.empty()) { + if (arguments_.size() + text.size() > ToolCallPlan::kMaxArgumentBytes) { + return Fail("Harmony argument body exceeds its size limit"); + } + arguments_ += text; + } else if (phase_ == Phase::kBody) { + const auto type = channel_ == "analysis" ? FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING + : FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT; + reasoning_tokens_ += channel_ == "analysis"; + if (!text.empty()) return {{ReasoningStreamSplitter::Segment{text, type, token}}, false, {}}; + } else if (text.find_first_not_of(" \t\r\n") != std::string::npos) { + return Fail("Unexpected bytes between Harmony messages"); + } + } catch (const std::exception& error) { + return Fail(error.what()); + } + return {}; +} + +HarmonyDecoder::Output HarmonyDecoder::Finish(std::optional terminal_token, bool natural_end) { + if (failed_) return {}; + if (natural_end && terminal_token && + (*terminal_token == protocol_.handoff || *terminal_token == protocol_.answer)) { + return Push(terminal_token, {}); + } + if (phase_ == Phase::kDone) return {}; + if (!recipient_.empty() || natural_end) return Fail("Incomplete or interrupted Harmony turn"); + return {}; +} + +PreparedChatPrompt HarmonyDecoder::TakeRecoveryPrompt() { + if (!checkpoint_captured_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "Harmony recovery has no selected-call checkpoint"); + } + checkpoint_.prompt_token_count = static_cast(checkpoint_.token_ids.size()); + return std::move(checkpoint_); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h new file mode 100644 index 000000000..949d65d5f --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_decoder.h @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "inferencing/generative/chat/harmony_protocol.h" +#include "inferencing/generative/chat/prepared_chat_prompt.h" +#include "inferencing/generative/chat/reasoning_stream_splitter.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" +#include "inferencing/generative/toolcalling/tool_call_stream_accumulator.h" + +#include +#include +#include +#include +#include +#include + +namespace fl { + +/// Decodes native message structure above payload admission. No session, retry policy, or publication ownership. +class HarmonyDecoder { + public: + using Event = std::variant; + struct Output { + std::vector events; + bool malformed = false; + std::string error; + }; + + HarmonyDecoder(HarmonyProtocol protocol, std::shared_ptr plan, + const PreparedChatPrompt& prompt, + std::unordered_set raw_tool_names = {}); + Output Push(std::optional token, const std::string& text); + Output Finish(std::optional terminal_token, bool natural_end); + bool HasRecoveryCheckpoint() const noexcept { return checkpoint_captured_; } + PreparedChatPrompt TakeRecoveryPrompt(); + int ReasoningTokenCount() const noexcept { return reasoning_tokens_; } + + private: + enum class Phase { kHeader, + kBody, + kBetween, + kDone }; + void ParseHeader(); + void RecordToken(int32_t token, const std::string& text); + void CaptureCheckpoint(); + Output Fail(std::string error); + + const HarmonyProtocol protocol_; + const std::shared_ptr plan_; + const std::unordered_set raw_tool_names_; + Phase phase_ = Phase::kHeader; + std::string header_; + std::string channel_; + std::string recipient_; + std::string arguments_; + bool visible_started_ = false; + bool failed_ = false; + bool checkpoint_captured_ = false; + size_t message_token_offset_ = 0; + size_t message_byte_offset_ = 0; + PreparedChatPrompt checkpoint_; + std::optional ending_; + int reasoning_tokens_ = 0; +}; + +template +void DeliverHarmonyOutput(HarmonyDecoder::Output output, SegmentSink&& segments, ToolSink&& tools) { + for (auto& event : output.events) { + if (auto* segment = std::get_if(&event)) { + segments(std::vector{std::move(*segment)}); + } else { + ToolCallStreamAccumulator::Output admitted; + admitted.events.emplace_back(std::move(std::get(event))); + tools(std::move(admitted)); + } + } + if (output.malformed) { + ToolCallStreamAccumulator::Output rejected; + rejected.malformed = true; + rejected.error = std::move(output.error); + tools(std::move(rejected)); + } +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h new file mode 100644 index 000000000..6158590a1 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/harmony_protocol.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include +#include + +namespace fl { + +struct HarmonyProtocol { + static constexpr const char* kAssistantPrefix = "<|start|>assistant"; + int32_t start; + int32_t channel; + int32_t message; + int32_t end; + int32_t handoff; + int32_t answer; + int32_t format; + std::vector assistant_prefix_tokens; +}; + +struct HarmonyContinuation { + std::string header; + size_t header_tokens = 0; + size_t header_bytes = 0; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc index 98bd81b94..923a3836e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -447,6 +447,9 @@ void OnnxChatEngine::RouteEvents() { conversation->result.generated_tokens = usage.GeneratedTokens(); conversation->result.cached_prompt_tokens = usage.CachedPromptTokens(); conversation->result.finish_reason = event->FinishReason(); +#if FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN + conversation->result.terminal_token_id = event->TerminalTokenId(); +#endif conversation->turn_finished = true; } if ((flags & OgaEngineEventFlag_Failed) != 0) { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h index f58582caa..0a40c4137 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h @@ -45,6 +45,7 @@ class OnnxChatEngine { uint64_t generated_tokens = 0; uint64_t cached_prompt_tokens = 0; uint32_t finish_reason = 0; + std::optional terminal_token_id = std::nullopt; }; class Conversation { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 22a51a777..3fc56eaf7 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -208,6 +208,8 @@ std::optional OnnxChatGenerator::GetTurnUsage() const { generated_tokens, termination.finish_reason, termination.cause, + 0, + eos_matched ? last_generated_token_ : std::nullopt, }; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc index cc9c78bdf..09d999b8b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_stream.cc @@ -62,6 +62,7 @@ ChatTurnUsage BuildTurnUsage(int prompt_tokens, const OnnxChatEngine::TurnResult MapFinishReason(static_cast(result.finish_reason)), MapTerminationCause(result.finish_reason), static_cast(result.cached_prompt_tokens), + result.terminal_token_id, }; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc index a4d6aa41a..e4e3cd8b5 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc @@ -14,6 +14,7 @@ #include #include +#include namespace fl { @@ -30,7 +31,7 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag } PreparedChatPrompt prepared; - if (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope()) { + if (tool_ctx.harmony || (tool_ctx.tool_plan && !tool_ctx.HasForcedRawEnvelope())) { const auto original = BuildChatMessagesJson(messages.Messages(), tool_ctx.supports_reasoning_history); std::string prefix; do { @@ -39,19 +40,25 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag tool_ctx.template_kwargs_json.find(prefix) != std::string::npos); std::unordered_map literals; - const auto projected = BuildCanonicalToolMessagesJson( - messages.Messages(), tool_ctx, [&](std::string text) { - if (text.empty()) { - return text; - } - auto marker = prefix + std::to_string(literals.size()) + "__"; - literals.emplace(marker, std::move(text)); - return marker; - }); + const auto protect = [&](std::string text) { + if (text.empty()) { + return text; + } + auto marker = prefix + std::to_string(literals.size()) + "__"; + literals.emplace(marker, std::move(text)); + return marker; + }; + const auto native = tool_ctx.harmony + ? BuildHarmonyPromptProjection(messages.Messages(), tool_ctx, protect) + : HarmonyPromptProjection{}; + const auto projected = tool_ctx.harmony ? native.messages_json + : BuildCanonicalToolMessagesJson(messages.Messages(), tool_ctx, protect); auto& preprocessor = model.GetPreprocessor(); const auto* kwargs = tool_ctx.template_kwargs_json.empty() ? nullptr : tool_ctx.template_kwargs_json.c_str(); const auto rendered = - preprocessor.ApplyChatTemplateWithOptions(projected.c_str(), nullptr, kwargs, true); + preprocessor.ApplyChatTemplateWithOptions(projected.c_str(), + native.tools_json.empty() ? nullptr : native.tools_json.c_str(), + kwargs, true); const auto append = [&](const std::string& text, bool literal) { if (text.empty()) { return; @@ -84,6 +91,15 @@ PreparedChatPrompt PrepareTextChatPrompt(const chat_internal::PreparedChatMessag } prepared.prompt_token_count = static_cast(prepared.token_ids.size()); + if (tool_ctx.harmony) { + const auto& assistant_prefix = tool_ctx.harmony->assistant_prefix_tokens; + if (!rendered.ends_with(HarmonyProtocol::kAssistantPrefix) || prepared.token_ids.size() < assistant_prefix.size() || + !std::equal(assistant_prefix.rbegin(), assistant_prefix.rend(), prepared.token_ids.rbegin())) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Harmony template produced an unqualified generation continuation"); + } + prepared.harmony_continuation = HarmonyContinuation{ + "assistant", assistant_prefix.size(), std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + } return prepared; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h index 8fccdb531..1cb104f10 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.h @@ -3,6 +3,7 @@ #pragma once #include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include #include @@ -33,6 +34,7 @@ struct PreparedChatPrompt { std::vector token_ids; std::unique_ptr media_tensors; int64_t prompt_token_count = 0; + std::optional harmony_continuation; bool HasMedia() const noexcept { return media_tensors != nullptr; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc index 66ccc16fa..b8ae5822d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc @@ -155,7 +155,7 @@ std::optional ResolveTurnGuidanceOptions(const ToolCallCont const bool tool_call_only = tool_ctx.tool_output && !tool_ctx.text_output; if (!guidance_type.empty() && !guidance_data.empty() && - (user_specified_guidance || tool_call_only || tool_ctx.tool_plan)) { + (user_specified_guidance || tool_call_only || tool_ctx.tool_plan || tool_ctx.harmony)) { return TurnGuidanceOptions{ std::move(guidance_type), std::move(guidance_data), @@ -248,8 +248,8 @@ EngineTurnOptionsPlan BuildEngineTurnOptionsPlan(const SearchOptions& options, plan.seed = *options.seed; } - // Native stops see hidden MESSAGE framing too; framed turns instead match ordinary visible text on the host. - if (ShouldForwardStopSequencesToEngine(backend_kind) && !tool_ctx.UsesAssistantTextFraming()) { + // Backend stops also see hidden framing and reasoning; protocol-aware turns match only visible text on the host. + if (ShouldForwardStopSequencesToEngine(backend_kind) && !tool_ctx.UsesAssistantTextFraming() && !tool_ctx.harmony) { plan.stop_sequences = options.stop_sequences; } diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index b53cbc096..f37af264b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -10,8 +10,8 @@ #include #include - #include +#include namespace fl { namespace { @@ -122,6 +122,86 @@ std::vector FindDelimitedRegions(std::string_view text, return regions; } +std::optional QualifyHarmonyProtocol(std::string_view model_type, Preprocessor& preprocessor) { + if (model_type != "gptoss") return std::nullopt; +#if !FOUNDRY_LOCAL_OGA_HAS_TERMINAL_TOKEN + (void)preprocessor; + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS Harmony requires an ORT GenAI runtime exposing sampled terminal-token identity"); +#else + const auto resolve = [&](const char* spelling) { + const auto encoded = preprocessor.Encode(spelling); + const auto literal = preprocessor.EncodeLiteral(spelling); + if (encoded->SequenceCount(0) != 1 || encoded->SequenceData(0)[0] < 0 || + (literal->SequenceCount(0) == 1 && literal->SequenceData(0)[0] == encoded->SequenceData(0)[0])) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Harmony control must have distinct literal/control encoding: ", spelling); + } + return encoded->SequenceData(0)[0]; + }; + HarmonyProtocol protocol{ + resolve("<|start|>"), + resolve("<|channel|>"), + resolve("<|message|>"), + resolve("<|end|>"), + resolve("<|call|>"), + resolve("<|return|>"), + resolve("<|constrain|>"), + {}, + }; + const std::set controls{protocol.start, protocol.channel, protocol.message, protocol.end, + protocol.handoff, protocol.answer, protocol.format}; + const auto& eos = preprocessor.GetEosTokenIds(); + if (controls.size() != 7 || std::find(eos.begin(), eos.end(), protocol.handoff) == eos.end() || + std::find(eos.begin(), eos.end(), protocol.answer) == eos.end() || + std::find(eos.begin(), eos.end(), protocol.end) != eos.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid Harmony control-token or EOS configuration"); + } + constexpr const char* probe = R"([ + {"role":"user","content":"probe"}, + {"role":"assistant","content":"","thinking":"probe-private","tool_calls":[ + {"function":{"name":"probe_function","arguments":{},"arguments_json":"__probe_arguments__"}} + ]}, + {"role":"tool","name":"probe_function","content":"","content_json":"__probe_result__"} + ])"; + const auto rendered = preprocessor.ApplyChatTemplate(probe, nullptr, true); + if (!ContainsInOrder(rendered, {"<|channel|>analysis<|message|>probe-private<|end|>", + "<|start|>assistant to=functions.probe_function<|channel|>commentary json" + "<|message|>__probe_arguments__<|call|>", + "<|start|>functions.probe_function to=assistant<|channel|>commentary" + "<|message|>__probe_result__<|end|>"}) || + !rendered.ends_with(HarmonyProtocol::kAssistantPrefix)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS requires the qualified protected Harmony transcript template"); + } + constexpr const char* custom_probe = R"([ + {"role":"user","content":"probe"}, + {"role":"assistant","content":"","tool_calls":[ + {"function":{"name":"probe_custom","native_freeform":true, + "arguments":{},"arguments_json":"__probe_raw_input__"}} + ]}, + {"role":"tool","name":"probe_custom","content":"","content_json":"__probe_custom_result__"} + ])"; + constexpr const char* custom_tools = R"([ + {"type":"function","function":{"name":"probe_custom","description":"probe", + "native_freeform":true,"parameters":{}}} + ])"; + const auto custom_rendered = preprocessor.ApplyChatTemplate(custom_probe, custom_tools, true); + if (!ContainsInOrder(custom_rendered, {"type probe_custom = (_: FREEFORM) => any;", + "<|start|>assistant to=functions.probe_custom<|channel|>commentary" + "<|message|>__probe_raw_input__<|call|>", + "<|start|>functions.probe_custom to=assistant<|channel|>commentary" + "<|message|>__probe_custom_result__<|end|>"}) || + !custom_rendered.ends_with(HarmonyProtocol::kAssistantPrefix)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "GPT-OSS requires protected native custom-tool declarations and history"); + } + const auto prefix = preprocessor.Encode(HarmonyProtocol::kAssistantPrefix); + protocol.assistant_prefix_tokens.assign(prefix->SequenceData(0), prefix->SequenceData(0) + prefix->SequenceCount(0)); + return protocol; +#endif +} + ModelCapabilities ResolveModelCapabilities(std::string_view model_type, Preprocessor& preprocessor, std::string_view tool_call_start, std::string_view tool_call_end, @@ -311,6 +391,13 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, capabilities_ = ResolveModelCapabilities(model_type_, *preprocessor_, tag_info.bot_str, tag_info.eot_str, tag_info.bor_str, tag_info.eor_str); + capabilities_.harmony = QualifyHarmonyProtocol(model_type_, *preprocessor_); + if (capabilities_.harmony) { + capabilities_.supports_tool_calling = true; + capabilities_.supports_reasoning_history = true; + capabilities_.positional_tool_results = true; + capabilities_.supports_reasoning_controls = true; + } if (IsMultiModal() && genai_config_.GetChatBackendKind() == ChatBackendKind::kEngine) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 65df74b3d..f83a20ef9 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -4,6 +4,7 @@ #include "inferencing/execution_provider.h" #include "inferencing/generative/genai_config.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include "inferencing/generative/preprocessor.h" #include "logger.h" @@ -33,6 +34,7 @@ struct ModelCapabilities { bool supports_reasoning_history = false; bool supports_preserve_thinking = false; bool supports_reasoning_controls = false; + std::optional harmony = std::nullopt; }; namespace model_capabilities_internal { @@ -73,6 +75,7 @@ class GenAIModelInstance { bool SupportsReasoningHistory() const { return capabilities_.supports_reasoning_history; } bool SupportsPreserveThinking() const { return capabilities_.supports_preserve_thinking; } bool SupportsReasoningControls() const { return capabilities_.supports_reasoning_controls; } + const std::optional& GetHarmonyProtocol() const { return capabilities_.harmony; } /// Cached tag token IDs and decoded strings for tool/reasoning detection. /// Populated once on first access using OGA tokenizer APIs. diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h index 2859c7cb4..e8f9288c5 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h @@ -2,15 +2,19 @@ // Licensed under the MIT License. #pragma once +#include "inferencing/session/types.h" + #include #include #include -#include +#include #include namespace fl { +using ToolCallEncodings = std::unordered_map; + /// One hop of a reconstructed Responses chain: the input items that request was given, followed by the output items /// it produced. /// @@ -24,14 +28,14 @@ namespace fl { struct ResponseChainHop { ResponseChainHop() = default; ResponseChainHop(nlohmann::json input, nlohmann::json output, - std::unordered_set raw_call_ids = {}) + ToolCallEncodings encodings = {}) : input_items(std::move(input)), output_items(std::move(output)), - raw_envelope_call_ids(std::move(raw_call_ids)) {} + call_encodings(std::move(encodings)) {} nlohmann::json input_items = nlohmann::json::array(); nlohmann::json output_items = nlohmann::json::array(); - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// A complete chain, oldest hop first. diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 33d9660b7..fb83f53de 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -251,6 +251,11 @@ std::unique_ptr MakeReasoningMessage(std::string reasoning) { return message; } +GeneratedCallEncoding ReplayedCallEncoding(const std::string& call_id, const ToolCallEncodings& encodings) { + const auto found = encodings.find(call_id); + return found == encodings.end() ? GeneratedCallEncoding::kStructured : found->second; +} + /// Rebuild a stored `function_call` item. /// /// A hop's stored items are replayed as the hop recorded them, whether the call came from the model's own output or @@ -258,7 +263,7 @@ std::unique_ptr MakeReasoningMessage(std::string reasoning) { /// validated strictly at the time, and a model-emitted call whose argument bytes were not a JSON object was already /// presented to the model as having none. Replay reproduces that rather than re-admitting bytes the strict /// caller-supplied path would now reject, which would fail a continuation of a conversation that already happened. -std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { +std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item, const ToolCallEncodings& encodings) { const auto call_id = RequiredReplayString(item, "call_id", "stored function_call"); const auto name = RequiredReplayString(item, "name", "stored function_call"); std::string arguments; @@ -271,7 +276,7 @@ std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { } auto call = std::make_unique(call_id, name, arguments, /*replayed_from_store=*/true, - ToolKind::kFunction); + ToolKind::kFunction, std::nullopt, ReplayedCallEncoding(call_id, encodings)); call->replayed_arguments = std::move(arguments); call->replayed_kind = ToolKind::kFunction; return call; @@ -283,15 +288,13 @@ std::unique_ptr MakeReplayedToolCall(const nlohmann::json& item) { /// produced it in, and the same form the tool would receive. It is never parsed or re-encoded: doing so would change /// the bytes for a payload that is itself JSON, or reject one that is not. std::unique_ptr MakeReplayedCustomToolCall( - const nlohmann::json& item, const std::unordered_set& raw_envelope_call_ids) { + const nlohmann::json& item, const ToolCallEncodings& encodings) { const auto call_id = RequiredReplayString(item, "call_id", "stored custom_tool_call"); const auto name = RequiredReplayString(item, "name", "stored custom_tool_call"); auto input = RequiredReplayString(item, "input", "stored custom_tool_call", true); auto call = std::make_unique(call_id, name, input, /*replayed_from_store=*/true, ToolKind::kCustom, std::nullopt, - raw_envelope_call_ids.contains(call_id) - ? GeneratedCallEncoding::kRawEnvelope - : GeneratedCallEncoding::kStructured); + ReplayedCallEncoding(call_id, encodings)); call->replayed_arguments = std::move(input); call->replayed_kind = ToolKind::kCustom; return call; @@ -325,7 +328,8 @@ flMessageRole ReplayRole(const std::string& role) { } // namespace /// Replay a hop's own input items — the request the caller sent for that hop, in the shape they sent it. -static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) { +static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items, + const ToolCallEncodings& encodings) { if (!items.is_array()) { return; } @@ -357,12 +361,12 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) } if (type == "function_call") { - request.AddOwnedItem(MakeReplayedToolCall(entry)); + request.AddOwnedItem(MakeReplayedToolCall(entry, encodings)); continue; } if (type == "custom_tool_call") { - request.AddOwnedItem(MakeReplayedCustomToolCall(entry, {})); + request.AddOwnedItem(MakeReplayedCustomToolCall(entry, encodings)); continue; } @@ -411,7 +415,7 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) /// text moved in front of the call. A hop that produced nothing replayable still emits the assistant boundary it /// committed live. static void AddHopOutputToRequest(Request& request, const nlohmann::json& output_items, - const std::unordered_set& raw_envelope_call_ids) { + const ToolCallEncodings& encodings) { bool emitted = false; if (output_items.is_array()) { @@ -423,13 +427,13 @@ static void AddHopOutputToRequest(Request& request, const nlohmann::json& output const std::string type = entry.value("type", ""); if (type == "function_call") { - request.AddOwnedItem(MakeReplayedToolCall(entry)); + request.AddOwnedItem(MakeReplayedToolCall(entry, encodings)); emitted = true; continue; } if (type == "custom_tool_call") { - request.AddOwnedItem(MakeReplayedCustomToolCall(entry, raw_envelope_call_ids)); + request.AddOwnedItem(MakeReplayedCustomToolCall(entry, encodings)); emitted = true; continue; } @@ -464,9 +468,9 @@ static void AddHopOutputToRequest(Request& request, const nlohmann::json& output static void AddChainContextToRequest(Request& request, const ResponseChainContext& context) { for (const auto& hop : context) { request.BeginItemSegment(); - AddJsonItemsToRequest(request, hop.input_items); + AddJsonItemsToRequest(request, hop.input_items, hop.call_encodings); request.BeginItemSegment(); - AddHopOutputToRequest(request, hop.output_items, hop.raw_envelope_call_ids); + AddHopOutputToRequest(request, hop.output_items, hop.call_encodings); } } @@ -741,6 +745,7 @@ Request ToSessionRequest(const ResponseCreateParams& params, const ResponseChain if (params.reasoning.has_value() && params.reasoning->effort.has_value()) { request.options["chat_template_kwargs"] = ResolveReasoningTemplateKwargs(std::nullopt, params.reasoning->effort).dump(); + request.options["reasoning_effort"] = *params.reasoning->effort; } // Text format / grammar guidance → metadata parameters diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc index cabd988de..06495e066 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc @@ -50,15 +50,27 @@ void ResponseStore::Store(const std::string& response_id, nlohmann::json response, nlohmann::json input_items, std::string model_id, - std::unordered_set raw_envelope_call_ids) { + ToolCallEncodings call_encodings) { std::lock_guard lock(mutex_); auto state = PrepareStoreLocked({response_id, std::move(model_id), std::move(response), std::move(input_items), - std::move(raw_envelope_call_ids)}); + std::move(call_encodings)}); DropEvictedArtifactsLocked(state); CommitStoreLocked(std::move(state)); } +void ResponseStore::Store(const std::string& response_id, + nlohmann::json response, + nlohmann::json input_items, + std::string model_id, + std::unordered_set raw_envelope_call_ids) { + ToolCallEncodings encodings; + for (const auto& id : raw_envelope_call_ids) { + encodings.emplace(id, GeneratedCallEncoding::kRawEnvelope); + } + Store(response_id, std::move(response), std::move(input_items), std::move(model_id), std::move(encodings)); +} + ResponseStore::MetadataState ResponseStore::PrepareStoreLocked( StoredResponse response, const std::shared_ptr& replay_snapshot) { const auto response_id = response.id; @@ -116,7 +128,7 @@ void ResponseStore::StoreLocked(StoredResponse response, MetadataState& state) { .model_id = std::move(response.model_id), .response = std::move(response.response), .input_items = std::move(response.input_items), - .raw_envelope_call_ids = std::move(response.raw_envelope_call_ids), + .call_encodings = std::move(response.call_encodings), .replay_prefix = std::move(replay_prefix), .insertion_sequence = state.next_insertion_sequence++}); state.index[response.id] = state.entries.begin(); @@ -597,7 +609,7 @@ ResponseChainHop ResponseStore::ToReplayHop(const Entry& entry) { if (output != entry.response.end() && output->is_array()) { replay.output_items = *output; } - replay.raw_envelope_call_ids = entry.raw_envelope_call_ids; + replay.call_encodings = entry.call_encodings; return replay; } diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h index 6d90ca1ca..ebaf2a400 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h @@ -120,12 +120,12 @@ class ResponseStore { struct StoredResponse { StoredResponse(std::string id_in, std::string model_id_in, nlohmann::json response_in, nlohmann::json input_items_in, - std::unordered_set raw_call_ids_in = {}) + ToolCallEncodings call_encodings_in = {}) : id(std::move(id_in)), model_id(std::move(model_id_in)), response(std::move(response_in)), input_items(std::move(input_items_in)), - raw_envelope_call_ids(std::move(raw_call_ids_in)) {} + call_encodings(std::move(call_encodings_in)) {} std::string id; /// Resolved model that produced it. A continuation must resolve to the same model; empty means unbound, which @@ -135,7 +135,7 @@ class ResponseStore { nlohmann::json response; /// This hop's own request items — exactly what the /input_items endpoint returns. nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// Outcome of ResponseStore::BeginResponse. @@ -191,7 +191,14 @@ class ResponseStore { nlohmann::json response, nlohmann::json input_items, std::string model_id = {}, - std::unordered_set raw_envelope_call_ids = {}); + ToolCallEncodings call_encodings = {}); + + /// Compatibility entry point for callers that record only legacy raw-envelope provenance. + void Store(const std::string& response_id, + nlohmann::json response, + nlohmann::json input_items, + std::string model_id, + std::unordered_set raw_envelope_call_ids); /// Retrieve a stored response by ID. Returns nullopt if not found. std::optional Get(const std::string& response_id); @@ -268,7 +275,7 @@ class ResponseStore { std::string model_id; nlohmann::json response; nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; std::shared_ptr replay_prefix; uint64_t insertion_sequence = 0; }; diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc index e74b2acf6..de3b43acf 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc @@ -3,6 +3,7 @@ #include "inferencing/generative/toolcalling/grammar.h" #include "exception.h" +#include "contracts/tool_definitions.h" #include "inferencing/generative/toolcalling/tool_call_plan.h" #include @@ -12,6 +13,74 @@ namespace fl { +namespace { + +std::string BuildHarmonyGrammar(const ToolCallContext& context) { + if (context.tool_output && !context.tool_plan) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Harmony tool guidance requires an original-schema plan"); + } + const auto& protocol = *context.harmony; + const auto token = [](int32_t id) { return "<[" + std::to_string(id) + "]>"; }; + auto excluded = context.eos_token_ids; + for (const auto id : {protocol.start, protocol.channel, protocol.message, protocol.end, + protocol.handoff, protocol.answer, protocol.format}) excluded.push_back(id); + std::sort(excluded.begin(), excluded.end()); + excluded.erase(std::unique(excluded.begin(), excluded.end()), excluded.end()); + std::ostringstream grammar; + grammar << "start: " << (context.supports_reasoning ? "reasoning* " : ""); + if (!context.tool_output) { + grammar << "commentary* final\n"; + } else { + grammar << (context.text_output ? "commentary* (final | tool)\n" : "tool\n"); + } + grammar << "reasoning: " << token(protocol.channel) << " \"analysis\" " << token(protocol.message) + << " plain_token* " << token(protocol.end) << " " << token(protocol.start) << " \"assistant\"\n"; + grammar << "commentary: " << token(protocol.channel) << " \"commentary\" " << token(protocol.message) + << " plain_token* " << token(protocol.end) << " " << token(protocol.start) << " \"assistant\"\n"; + grammar << "final: " << token(protocol.channel) << " \"final\" " << token(protocol.message) + << " plain_token* " << token(protocol.answer) << "\n"; + if (context.tool_output) { + const auto names = context.tool_plan->AllowedNames(); + grammar << "tool: "; + for (size_t i = 0; i < names.size(); ++i) grammar << (i ? " | " : "") << "tool_" << i; + grammar << "\n"; + for (size_t i = 0; i < names.size(); ++i) { + const auto custom = context.UsesNativeRawBody(names[i]); + grammar << "tool_" << i << ": header_" << i << (custom ? " " : " metadata ") + << token(protocol.message) << " "; + const auto format = context.custom_lark_grammars.find(names[i]); + if (custom && format != context.custom_lark_grammars.end()) { + if (names[i] != "apply_patch" || format->second != tools::kStockGhcpApplyPatchLarkGrammar) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Unsupported native custom-tool input grammar"); + } + grammar << "native_patch"; + } else { + grammar << "arguments"; + } + grammar << " " << token(protocol.handoff) << "\n"; + const auto recipient = EscapeLarkLiteral("functions." + names[i]); + grammar << "header_" << i << ": \" to=\" " << recipient << " " << token(protocol.channel) + << " \"commentary\" | " << token(protocol.channel) << " \"commentary to=\" " << recipient << "\n"; + } + grammar << "metadata: (\" json\" | " << token(protocol.format) << " \"json\" | \" \" " + << token(protocol.format) << " \"json\")?\n"; + if (std::any_of(names.begin(), names.end(), [&](const auto& name) { + return context.UsesNativeRawBody(name); + })) { + auto patch = std::string(tools::kStockGhcpApplyPatchLarkGrammar); + patch.replace(0, std::string_view("start").size(), "native_patch"); + grammar << patch << "\n"; + } + } + grammar << "plain_token: <[^"; + for (size_t i = 0; i < excluded.size(); ++i) grammar << (i ? "," : "") << excluded[i]; + grammar << "]>\n"; + if (context.tool_output) grammar << context.tool_plan->ObjectGrammar("arguments"); + return grammar.str(); +} + +} // namespace + std::string EscapeLarkLiteral(const std::string& text) { return nlohmann::json(text).dump(); } @@ -28,6 +97,7 @@ std::string BuildLarkGrammar(const ToolCallContext& ctx, bool prompt_opens_reaso if (!ctx.text_output && !ctx.tool_output) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "neither text output nor tool calling output is enabled"); } + if (ctx.harmony) return BuildHarmonyGrammar(ctx); if (!ctx.tool_output) { return {}; } diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h index b7cd173d1..30e761610 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_context.h @@ -3,6 +3,7 @@ #pragma once #include "inferencing/session/types.h" +#include "inferencing/generative/chat/harmony_protocol.h" #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include namespace fl { @@ -25,6 +27,7 @@ struct ToolCallContext { /// Whether the model supports tool calling, using explicit catalog metadata when present /// and otherwise the loaded model's rendered-template probe. bool supports_tool_calling = false; + std::optional harmony; /// The special token that marks the start of a tool call in generated text. /// Empty if the model does not advertise tool call tokens. @@ -102,6 +105,7 @@ struct ToolCallContext { } const RawEnvelopeDescriptor* ActiveRawEnvelope() const { + if (harmony) return nullptr; if (!tool_output || !raw_envelope.has_value() || !IsCustomTool(raw_envelope->tool_name)) { return nullptr; } @@ -132,6 +136,25 @@ struct ToolCallContext { /// Whether the named tool takes a raw text payload rather than JSON arguments. bool IsCustomTool(const std::string& name) const { return KindOf(name) == ToolKind::kCustom; } + /// Unframed custom text keeps JSON transport; native raw bodies require an admitted input grammar. + bool UsesNativeRawBody(const std::string& name) const { + return harmony && IsCustomTool(name) && custom_lark_grammars.find(name) != custom_lark_grammars.end(); + } + + std::unordered_set NativeRawToolNames() const { + std::unordered_set names; + for (const auto& entry : custom_lark_grammars) { + if (UsesNativeRawBody(entry.first)) names.insert(entry.first); + } + return names; + } + + GeneratedCallEncoding CallEncoding(const std::string& name, bool uses_raw_envelope) const { + if (uses_raw_envelope) return GeneratedCallEncoding::kRawEnvelope; + if (!harmony) return GeneratedCallEncoding::kStructured; + return UsesNativeRawBody(name) ? GeneratedCallEncoding::kNativeRaw : GeneratedCallEncoding::kNativeJson; + } + /// User-specified guidance type from response_format (e.g., "lark_grammar", "json_schema"). /// Empty means no explicit guidance — the generator may still apply auto-generated tool guidance. std::string guidance_type; @@ -152,7 +175,7 @@ struct ToolCallContext { } bool UsesAssistantTextFraming() const { - return tool_plan && text_output && tool_output && !HasAnyExplicitGuidance() && !ActiveRawEnvelope(); + return !harmony && tool_plan && text_output && tool_output && !HasAnyExplicitGuidance() && !ActiveRawEnvelope(); } /// Whether any tools were provided in the request. diff --git a/sdk_v2/cpp/src/inferencing/session/types.h b/sdk_v2/cpp/src/inferencing/session/types.h index 6488ff30f..0d85f6191 100644 --- a/sdk_v2/cpp/src/inferencing/session/types.h +++ b/sdk_v2/cpp/src/inferencing/session/types.h @@ -28,6 +28,8 @@ enum class ToolKind { enum class GeneratedCallEncoding { kStructured, kRawEnvelope, + kNativeRaw, + kNativeJson, }; struct ForcedToolChoice { diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index fcc652bea..639359a66 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -22,7 +22,6 @@ #include #include #include -#include namespace fl { @@ -61,7 +60,7 @@ struct PublishRequest { std::string model_id; nlohmann::json response; nlohmann::json input_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; }; /// Publish a completed response: store its metadata and cache its session as one indivisible step. @@ -79,7 +78,7 @@ void PublishResponse(PublishRequest request) { const bool published = request.store.Commit( request.lease, ResponseStore::StoredResponse{request.response_id, request.model_id, std::move(request.response), - std::move(request.input_items), std::move(request.raw_envelope_call_ids)}, + std::move(request.input_items), std::move(request.call_encodings)}, &admission); if (!published) { @@ -426,11 +425,13 @@ std::shared_ptr ResponsesHandler::HandleNo session->ProcessRequest(session_request, session_response); auto [output, output_text] = ResponseConverter::FromSessionResponse(session_response); - std::unordered_set raw_envelope_call_ids; - for (const auto& item : output) { - if (const auto* custom = std::get_if(&item); - custom != nullptr && custom->generated_encoding == GeneratedCallEncoding::kRawEnvelope) { - raw_envelope_call_ids.insert(custom->call_id); + ToolCallEncodings call_encodings; + for (const auto& item : session_response.items) { + if (item->type == FOUNDRY_LOCAL_ITEM_TOOL_CALL) { + const auto& call = static_cast(*item); + if (call.generated_encoding != GeneratedCallEncoding::kStructured) { + call_encodings.emplace(call.call_id, call.generated_encoding); + } } } @@ -450,7 +451,7 @@ std::shared_ptr ResponsesHandler::HandleNo .model_id = turn.model_id, .response = response_json, .input_items = ResponseConverter::ToInputItems(req_json), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .call_encodings = std::move(call_encodings)}); } return JsonResponse(Status::CODE_200, response_json); @@ -531,7 +532,7 @@ std::shared_ptr ResponsesHandler::HandleSt // Items that have been *closed* (or, for the currently-open item at end-of-stream, finalized in place). // Used to construct the final `output[]` array for the response.completed event. std::vector closed_items; - std::unordered_set raw_envelope_call_ids; + ToolCallEncodings call_encodings; auto push_event = [&](const std::string& event_name, const StreamEvent& ev) { stream->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); @@ -669,8 +670,8 @@ std::shared_ptr ResponsesHandler::HandleSt } closed_items.push_back(std::move(output.completed_item)); - if (call.generated_encoding == GeneratedCallEncoding::kRawEnvelope) { - raw_envelope_call_ids.insert(call.call_id); + if (call.generated_encoding != GeneratedCallEncoding::kStructured) { + call_encodings.emplace(call.call_id, call.generated_encoding); } }; @@ -779,7 +780,7 @@ std::shared_ptr ResponsesHandler::HandleSt .model_id = turn.model_id, .response = std::move(response_json), .input_items = ResponseConverter::ToInputItems(req_copy), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .call_encodings = std::move(call_encodings)}); })) { stream->Finish(); return; diff --git a/sdk_v2/cpp/test/CMakeLists.txt b/sdk_v2/cpp/test/CMakeLists.txt index 012dbe138..587721db9 100644 --- a/sdk_v2/cpp/test/CMakeLists.txt +++ b/sdk_v2/cpp/test/CMakeLists.txt @@ -97,6 +97,7 @@ add_executable(foundry_local_tests internal_api/callback_handler_test.cc internal_api/catalog_cache_test.cc internal_api/chat/chat_generator_test.cc + internal_api/chat/harmony_decoder_test.cc internal_api/chat/reasoning_stream_splitter_test.cc internal_api/chat/chat_session_test.cc internal_api/chat/chat_template_test.cc diff --git a/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc new file mode 100644 index 000000000..72be8d76c --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/chat/harmony_decoder_test.cc @@ -0,0 +1,406 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "exception.h" +#include "contracts/tool_definitions.h" +#include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/chat/chat_transcript.h" +#include "inferencing/generative/chat/harmony_decoder.h" +#include "inferencing/generative/openresponses/response_converter.h" +#include "inferencing/generative/openresponses/response_store.h" +#include "inferencing/generative/toolcalling/grammar.h" +#include "inferencing/generative/toolcalling/tool_call_context.h" +#include "inferencing/generative/toolcalling/tool_call_plan.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace fl; + +namespace { + +const HarmonyProtocol kProtocol{ + 10, 11, 12, 13, 14, 15, 16, {10, 17}}; + +constexpr const char* kTools = R"([ + {"type":"function","function":{ + "name":"lookup", + "parameters":{ + "type":"object", + "properties":{"city":{"type":"string"}}, + "required":["city"] + } + }}, + {"type":"function","function":{ + "name":"patch", + "parameters":{ + "type":"object", + "properties":{"input":{"type":"string"}}, + "required":["input"] + } + }} +])"; + +PreparedChatPrompt MakePrompt() { + PreparedChatPrompt prompt; + prompt.prompt = "prefix<|start|>assistant"; + prompt.token_ids = {99, kProtocol.start, 17}; + prompt.harmony_continuation = HarmonyContinuation{ + "assistant", kProtocol.assistant_prefix_tokens.size(), + std::char_traits::length(HarmonyProtocol::kAssistantPrefix)}; + return prompt; +} + +void PushHeader(HarmonyDecoder& decoder, std::string recipient, bool json) { + EXPECT_FALSE(decoder.Push(100, " to=functions.").malformed); + EXPECT_FALSE(decoder.Push(101, std::move(recipient)).malformed); + EXPECT_FALSE(decoder.Push(kProtocol.channel, {}).malformed); + EXPECT_FALSE(decoder.Push(102, "commentary").malformed); + if (json) { + EXPECT_FALSE(decoder.Push(kProtocol.format, {}).malformed); + EXPECT_FALSE(decoder.Push(103, "json").malformed); + } + EXPECT_FALSE(decoder.Push(kProtocol.message, {}).malformed); +} + +const ParsedToolCall& OnlyCall(const HarmonyDecoder::Output& output) { + EXPECT_FALSE(output.malformed) << output.error; + EXPECT_EQ(output.events.size(), 1u); + const auto* call = output.events.empty() ? nullptr : std::get_if(&output.events.front()); + EXPECT_NE(call, nullptr); + static const ParsedToolCall empty; + return call == nullptr ? empty : *call; +} + +} // namespace + +TEST(HarmonyDecoderTest, UsesControlIdsWithEmptyDecodedTextAndAcceptsSplitJsonHeader) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + PushHeader(decoder, "lookup", true); + constexpr std::string_view arguments = R"({"city": "Paris"})"; + EXPECT_FALSE(decoder.Push(104, std::string(arguments)).malformed); + const auto output = decoder.Finish(kProtocol.handoff, true); + const auto& call = OnlyCall(output); + + EXPECT_EQ(call.name, "lookup"); + EXPECT_EQ(call.arguments, arguments); + EXPECT_EQ(call.argument_source, arguments); + EXPECT_FALSE(call.id.empty()); +} + +TEST(HarmonyDecoderTest, LiteralControlSpellingsCannotManufactureHeaderMetadata) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + EXPECT_FALSE(decoder.Push(100, "<|channel|>commentary").malformed); + const auto output = decoder.Push(kProtocol.message, {}); + + EXPECT_TRUE(output.malformed); + EXPECT_NE(output.error.find("exactly one channel"), std::string::npos); +} + +TEST(HarmonyDecoderTest, TerminalMetadataCompletesToolAndFinalMessages) { + auto tool_prompt = MakePrompt(); + HarmonyDecoder tool_decoder(kProtocol, ToolCallPlan::Create(kTools), tool_prompt); + PushHeader(tool_decoder, "lookup", true); + EXPECT_FALSE(tool_decoder.Push(104, R"({"city":"Paris"})").malformed); + EXPECT_EQ(OnlyCall(tool_decoder.Finish(kProtocol.handoff, true)).name, "lookup"); + + auto final_prompt = MakePrompt(); + HarmonyDecoder final_decoder(kProtocol, nullptr, final_prompt); + EXPECT_FALSE(final_decoder.Push(kProtocol.channel, {}).malformed); + EXPECT_FALSE(final_decoder.Push(105, "final").malformed); + EXPECT_FALSE(final_decoder.Push(kProtocol.message, {}).malformed); + EXPECT_FALSE(final_decoder.Push(106, "done").malformed); + const auto final = final_decoder.Finish(kProtocol.answer, true); + EXPECT_FALSE(final.malformed) << final.error; + EXPECT_TRUE(final.events.empty()); +} + +TEST(HarmonyDecoderTest, InterruptedOrMalformedCallsFailWithoutPublishing) { + auto interrupted_prompt = MakePrompt(); + HarmonyDecoder interrupted(kProtocol, ToolCallPlan::Create(kTools), interrupted_prompt); + PushHeader(interrupted, "lookup", true); + EXPECT_FALSE(interrupted.Push(104, R"({"city":"Paris"})").malformed); + const auto interrupted_output = interrupted.Finish(std::nullopt, true); + EXPECT_TRUE(interrupted_output.malformed); + EXPECT_TRUE(interrupted_output.events.empty()); + + auto unauthorized_prompt = MakePrompt(); + HarmonyDecoder unauthorized(kProtocol, ToolCallPlan::Create(kTools), unauthorized_prompt); + PushHeader(unauthorized, "missing", true); + EXPECT_FALSE(unauthorized.Push(104, R"({"city":"Paris"})").malformed); + const auto unauthorized_output = unauthorized.Push(kProtocol.handoff, {}); + EXPECT_TRUE(unauthorized_output.malformed); + EXPECT_TRUE(unauthorized_output.events.empty()); +} + +TEST(HarmonyDecoderTest, SelectedRecipientCapturesARecoveryCheckpointBeforeItsHeader) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt); + + EXPECT_FALSE(decoder.Push(100, " to=func").malformed); + EXPECT_TRUE(decoder.HasRecoveryCheckpoint()); + EXPECT_FALSE(decoder.Push(101, "tions.lookup").malformed); + ASSERT_TRUE(decoder.HasRecoveryCheckpoint()); + + const auto recovered = decoder.TakeRecoveryPrompt(); + EXPECT_EQ(recovered.prompt, "prefix"); + EXPECT_EQ(recovered.token_ids, (std::vector{99})); +} + +TEST(HarmonyDecoderTest, NativeRawBodyUsesInputEnvelopeWithoutLosingSourceBytes) { + auto prompt = MakePrompt(); + HarmonyDecoder decoder(kProtocol, ToolCallPlan::Create(kTools), prompt, {"patch"}); + + PushHeader(decoder, "patch", false); + const std::string body = "*** Begin Patch\n+literal <|call|>\n*** End Patch"; + EXPECT_FALSE(decoder.Push(104, body).malformed); + const auto output = decoder.Push(kProtocol.handoff, {}); + const auto& call = OnlyCall(output); + const auto expected = nlohmann::json{{"input", body}}.dump(); + + EXPECT_EQ(call.name, "patch"); + EXPECT_EQ(call.arguments, expected); + EXPECT_EQ(call.argument_source, expected); +} + +TEST(HarmonyDecoderTest, PromptProjectionPreservesNativeJsonRawAndResultProvenance) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"lookup", ToolKind::kFunction}, {"patch", ToolKind::kCustom}}; + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + + TranscriptMessage raw_assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + raw_assistant.AppendToolCall( + MakeGeneratedToolCall("call_raw", "patch", "raw <|call|> text", + ToolKind::kCustom, GeneratedCallEncoding::kNativeRaw) + .call); + TranscriptMessage json_assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + const std::string json_arguments = R"({"path":"a\\b","text":"<|call|>"})"; + json_assistant.AppendToolCall( + MakeGeneratedToolCall("call_json", "lookup", json_arguments, + ToolKind::kFunction, GeneratedCallEncoding::kNativeJson) + .call); + const std::vector messages{ + std::move(raw_assistant), + TranscriptMessage::ToolResult("call_raw", "raw result"), + std::move(json_assistant), + TranscriptMessage::ToolResult("call_json", "json result"), + }; + + const auto projection = BuildHarmonyPromptProjection(messages, context, [](std::string value) { + return value; + }); + const auto projected = nlohmann::json::parse(projection.messages_json); + ASSERT_EQ(projected.size(), 5u); + EXPECT_EQ(projected[1]["tool_calls"][0]["function"]["arguments_json"], "raw <|call|> text"); + EXPECT_TRUE(projected[1]["tool_calls"][0]["function"]["native_freeform"]); + EXPECT_EQ(projected[2]["content_json"], nlohmann::json("raw result").dump()); + EXPECT_EQ(projected[3]["tool_calls"][0]["function"]["arguments_json"], json_arguments); + EXPECT_FALSE(projected[3]["tool_calls"][0]["function"]["native_freeform"]); + EXPECT_EQ(projected[4]["content_json"], nlohmann::json("json result").dump()); + + const auto projected_tools = nlohmann::json::parse(projection.tools_json); + EXPECT_FALSE(projected_tools[0]["function"].contains("native_freeform")); + EXPECT_TRUE(projected_tools[1]["function"]["native_freeform"].get()); +} + +TEST(HarmonyDecoderTest, StoredInputAndOutputReplayRetainHistoricalCustomBodyEncoding) { + const std::string body = R"({"input":"literal <|call|> and a\\b"})"; + for (const auto encoding : {GeneratedCallEncoding::kStructured, GeneratedCallEncoding::kNativeJson, + GeneratedCallEncoding::kNativeRaw}) { + for (const bool current_raw : {false, true}) { + for (const bool stored_input : {false, true}) { + SCOPED_TRACE(static_cast(encoding)); + SCOPED_TRACE(current_raw); + SCOPED_TRACE(stored_input); + const auto call = nlohmann::json{{"type", "custom_tool_call"}, {"call_id", "call_history"}, {"name", "patch"}, {"input", body}}; + auto input = nlohmann::json::array({{{"role", "user"}, {"content", "patch"}}}); + if (stored_input) { + input.push_back(call); + } + const auto output = stored_input ? nlohmann::json::array() : nlohmann::json::array({call}); + ResponseStore store; + store.Store("history", {{"output", output}}, input, "harmony", + ToolCallEncodings{{"call_history", encoding}}); + const auto chain = store.BuildChainContext("history"); + ASSERT_TRUE(chain.has_value()); + responses::ResponseCreateParams params; + params.model = "harmony"; + params.input = "continue"; + auto request = ResponseConverter::ToSessionRequest(params, &*chain); + const auto replay = IngestRequestItems(request.items, request.item_segment_starts); + std::size_t call_count = 0; + for (const auto& message : replay.messages) { + for (const auto* historical : message.ToolCalls()) { + ++call_count; + EXPECT_EQ(historical->generated_encoding, encoding); + EXPECT_TRUE(historical->has_recorded_encoding); + EXPECT_EQ(historical->kind, ToolKind::kCustom); + EXPECT_EQ(historical->arguments, body); + } + } + EXPECT_EQ(call_count, 1u); + + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"patch", ToolKind::kCustom}}; + if (current_raw) { + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + } + const auto projected = nlohmann::json::parse( + BuildHarmonyPromptProjection(replay.messages, context, [](std::string value) { return value; }) + .messages_json); + call_count = 0; + for (const auto& message : projected) { + if (!message.contains("tool_calls")) { + continue; + } + for (const auto& historical : message["tool_calls"]) { + ++call_count; + const auto& function = historical["function"]; + const bool raw = encoding == GeneratedCallEncoding::kNativeRaw; + const auto expected = raw ? body : nlohmann::json{{"input", body}}.dump(); + EXPECT_EQ(function["native_freeform"], raw); + EXPECT_EQ(function["arguments_json"], expected); + } + } + EXPECT_EQ(call_count, 1u); + } + } + } +} + +TEST(HarmonyDecoderTest, UnannotatedCallerHistoryStillUsesTheCurrentCustomBodyDeclaration) { + for (const bool current_raw : {false, true}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = kTools; + context.tool_kinds = {{"patch", ToolKind::kCustom}}; + if (current_raw) { + context.custom_lark_grammars = {{"patch", "start: /[\\s\\S]+/"}}; + } + const std::string body = R"({"input":"caller text"})"; + TranscriptMessage assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + assistant.AppendToolCall(MakeSuppliedToolCall("caller", "patch", body, ToolKind::kCustom)); + const auto projected = nlohmann::json::parse( + BuildHarmonyPromptProjection({assistant}, context, [](std::string value) { return value; }) + .messages_json); + const auto& function = projected.back()["tool_calls"][0]["function"]; + EXPECT_EQ(function["native_freeform"], current_raw); + const auto expected = current_raw ? body : nlohmann::json{{"input", body}}.dump(); + EXPECT_EQ(function["arguments_json"], expected); + } +} + +TEST(HarmonyGrammarTest, ChoicesKeepAuthoritativeHeadersBoundariesAndAllowedRecipients) { + for (const bool reasoning : {false, true}) { + for (const std::string choice : {"auto", "required", "forced", "none"}) { + SCOPED_TRACE(reasoning); + SCOPED_TRACE(choice); + ToolCallContext context; + context.harmony = kProtocol; + context.supports_reasoning = reasoning; + context.text_output = choice == "auto" || choice == "none"; + context.tool_output = choice != "none"; + context.eos_token_ids = {99, kProtocol.handoff, 99}; + if (context.tool_output) { + const auto forced = choice == "forced" + ? std::optional{{"lookup", ToolKind::kFunction}} + : std::nullopt; + context.tool_plan = ToolCallPlan::Create(kTools, true, forced); + } + const auto grammar = BuildLarkGrammar(context); + const std::string mode = choice == "none" ? "commentary* final" + : choice == "auto" ? "commentary* (final | tool)" + : "tool"; + EXPECT_TRUE(grammar.starts_with("start: " + std::string(reasoning ? "reasoning* " : "") + mode + "\n")); + EXPECT_NE(grammar.find("plain_token: <[^10,11,12,13,14,15,16,99]>"), std::string::npos); + EXPECT_NE(grammar.find("final: <[11]> \"final\" <[12]> plain_token* <[15]>"), std::string::npos); + if (context.tool_output) { + EXPECT_NE(grammar.find("metadata <[12]> arguments <[14]>"), std::string::npos); + EXPECT_NE(grammar.find("\" to=\" \"functions.lookup\" <[11]> \"commentary\""), std::string::npos); + EXPECT_NE(grammar.find("<[11]> \"commentary to=\" \"functions.lookup\""), std::string::npos); + EXPECT_EQ(grammar.find("\"functions.patch\"") != std::string::npos, choice != "forced"); + EXPECT_NE(grammar.find("arguments:"), std::string::npos); + } else { + EXPECT_EQ(grammar.find("tool:"), std::string::npos); + EXPECT_EQ(grammar.find("arguments:"), std::string::npos); + } + } + } +} + +TEST(HarmonyGrammarTest, NativePatchBodiesUseTheirGrammarAndUnsupportedFormatsFailExplicitly) { + ToolCallContext context; + context.harmony = kProtocol; + context.tool_output = true; + context.text_output = false; + EXPECT_THROW(BuildLarkGrammar(context), fl::Exception); + context.tool_plan = ToolCallPlan::Create(R"([{"type":"function","function":{ + "name":"apply_patch","parameters":{"type":"object","properties":{"input":{"type":"string"}}}}}])"); + context.tool_kinds = {{"apply_patch", ToolKind::kCustom}}; + context.custom_lark_grammars = {{"apply_patch", std::string(tools::kStockGhcpApplyPatchLarkGrammar)}}; + const auto grammar = BuildLarkGrammar(context); + EXPECT_NE(grammar.find("tool_0: header_0 <[12]> native_patch <[14]>"), std::string::npos); + EXPECT_NE(grammar.find("native_patch:"), std::string::npos); + context.custom_lark_grammars["apply_patch"] = "start: /[\\s\\S]+/"; + EXPECT_THROW(BuildLarkGrammar(context), fl::Exception); +} + +TEST(HarmonyDecoderTest, PromptProjectionRejectsNonLeadingInstructionRoles) { + for (const auto* tools : {"", kTools}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = tools; + for (const auto role : {FOUNDRY_LOCAL_ROLE_SYSTEM, FOUNDRY_LOCAL_ROLE_DEVELOPER}) { + const auto protect = [](std::string value) { return value; }; + EXPECT_NO_THROW(BuildHarmonyPromptProjection( + {{role, "Follow these instructions."}, {FOUNDRY_LOCAL_ROLE_USER, "Hello"}}, context, protect)); + try { + BuildHarmonyPromptProjection( + {{FOUNDRY_LOCAL_ROLE_USER, "Hello"}, {role, "Do not omit this instruction."}, {FOUNDRY_LOCAL_ROLE_USER, "Continue"}}, + context, protect); + FAIL() << "Expected non-leading instructions to be rejected"; + } catch (const fl::Exception& error) { + EXPECT_EQ(error.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } + } + } +} + +TEST(HarmonyDecoderTest, PromptProjectionRejectsHeaderSeparatorsInToolAndHistoryNames) { + for (const char separator : std::string(" \t\r\n\v\f\x01\x02")) { + const std::string name = "lookup" + std::string(1, separator) + "city"; + for (const bool history : {false, true}) { + ToolCallContext context; + context.harmony = kProtocol; + context.tools_json = history ? kTools : nlohmann::json::array({{{"type", "function"}, {"function", {{"name", name}, {"parameters", {{"type", "object"}}}}}}}).dump(); + std::vector messages{{FOUNDRY_LOCAL_ROLE_USER, "Hello"}}; + if (history) { + TranscriptMessage assistant(FOUNDRY_LOCAL_ROLE_ASSISTANT, ""); + assistant.AppendToolCall(MakeSuppliedToolCall("call_bad", name, "{}")); + messages.push_back(std::move(assistant)); + } + try { + BuildHarmonyPromptProjection(messages, context, [](std::string value) { return value; }); + FAIL() << "Expected an unrepresentable Harmony recipient to be rejected"; + } catch (const fl::Exception& error) { + EXPECT_EQ(error.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } + } + } +}