Honor per-model external prefill chunking capability

sdk_v2/cpp/CMakeLists.txt

@@ -126,6 +126,18 @@ check_cxx_source_compiles(
      }"
     FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI
 )
+unset(FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY CACHE)
+check_cxx_source_compiles(
+    "#include <ort_genai.h>
+     int main() {
+       const OgaModel* model = nullptr;
+       return model->SupportsExternalPrefillChunking();
+     }"
+    FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY
+)
+if(NOT FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY)
+    message(STATUS "ORT GenAI has no per-model prefill capability API; retaining legacy chunk-size defaults")
+endif()
 unset(CMAKE_REQUIRED_INCLUDES)
 unset(CMAKE_REQUIRED_LIBRARIES)
 if(NOT FOUNDRY_LOCAL_HAS_REQUIRED_NON_GENERATIVE_ORT_GENAI)
@@ -394,7 +406,8 @@ list(APPEND FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_PUBLIC_HEADERS})
 function(foundry_local_configure_target TARGET LINK_SCOPE)
     target_compile_options(${TARGET} PRIVATE ${FOUNDRY_LOCAL_COMPILE_OPTIONS})
     target_compile_definitions(${TARGET} ${LINK_SCOPE}
-        FOUNDRY_LOCAL_HAS_SHARED_HOST=$<BOOL:${FOUNDRY_LOCAL_HAS_SHARED_HOST}>)
+        FOUNDRY_LOCAL_HAS_SHARED_HOST=$<BOOL:${FOUNDRY_LOCAL_HAS_SHARED_HOST}>
+        FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY=$<BOOL:${FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY}>)
 
     target_include_directories(${TARGET}
         PUBLIC

sdk_v2/cpp/README.md

@@ -760,6 +760,14 @@ prefill step cannot be interrupted before it returns.
 Configure with `--skip_service` when the web service is not required; this removes its
 oat++ dependency from the build.
 
+## Generator prefill capability
+
+Automatic Generator prefill chunking uses GenAI's per-model capability when the configured headers and library
+provide it. Models that manage prefill internally can opt out of host-driven chunking without changing tool protocols.
+With older packages, including the pinned 0.19.0 package, the build explicitly reports that legacy chunk-size
+defaults are retained. Model-specific opt-out requires a runtime package containing the capability API.
+Explicit positive model chunk sizes are preserved.
+
 ## Examples
 
 The build produces these example executables in the build output directory:

sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc

@@ -412,9 +412,15 @@ std::unique_ptr<OnnxChatGenerator> OnnxChatGenerator::CreatePrepared(
   const int input_token_count = static_cast<int>(prepared.prompt_token_count);
   auto gen_params = OgaGeneratorParams::Create(model.GetOgaModel());
   const int default_max_output_tokens = GetDefaultMaxOutputTokens(prepared.HasMedia());
+#if FOUNDRY_LOCAL_OGA_HAS_PREFILL_CHUNKING_CAPABILITY
+  const bool supports_external_prefill_chunking = model.GetOgaModel().SupportsExternalPrefillChunking();
+#else
+  // Older runtimes retain the automatic chunk-size behavior they had before this capability was added.
+  constexpr bool supports_external_prefill_chunking = true;
+#endif
   const int max_length =
       ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, model.EP(),
-                         use_full_context, default_max_output_tokens);
+                         use_full_context, default_max_output_tokens, supports_external_prefill_chunking);
   const int max_output_tokens = ResolveMaxOutputTokens(options, default_max_output_tokens);
 
   auto reasoning_markers = ResolveReasoningMarkers(tool_ctx, model);

sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc

@@ -281,7 +281,8 @@ int ApplySearchOptions(const SearchOptions& options,
                        OgaGeneratorParams& gen_params,
                        ExecutionProvider ep,
                        bool use_full_context,
-                       int default_max_output_tokens) {
+                       int default_max_output_tokens,
+                       bool supports_external_prefill_chunking) {
   ValidatePenalties(options);
 
   const int model_max_length = GetModelMaxContextLength(config);
@@ -336,7 +337,7 @@ int ApplySearchOptions(const SearchOptions& options,
 
   // Preserve a positive model setting. ORT GenAI reports both an absent setting and explicit zero as zero; Foundry
   // Local intentionally treats both as unset. ORT GenAI decides whether the model consumes the resulting option.
-  if (gen_params.GetSearchNumber("chunk_size") <= 0) {
+  if (supports_external_prefill_chunking && gen_params.GetSearchNumber("chunk_size") <= 0) {
     // The model's resolved EP is kDefault for the common load path, so use the provider declared in
     // genai_config.json. An empty provider means ORT's CPU fallback.
     ExecutionProvider effective_ep = ep;

sdk_v2/cpp/src/inferencing/generative/chat/search_options.h

@@ -164,7 +164,7 @@ EngineTurnOptionsPlan BuildEngineTurnOptionsPlan(const SearchOptions& options,
 ///                           on providers that benefit from it (CUDA, NvTensorRtRtx, WebGPU, CPU).
 ///                           When kDefault, the effective provider is taken from
 ///                           config.DefaultProvider() (empty ⇒ CPU).
-///                           ORT GenAI determines whether the model consumes this option.
+/// @param supports_external_prefill_chunking  Whether the loaded model accepts host-driven chunking.
 /// @param use_full_context   When true, set max_length to the model's full context window
 ///                           instead of input+output. Used for continuous decoding (cached generators).
 /// @param default_max_output_tokens  Default applied when the request does not specify
@@ -175,7 +175,8 @@ int ApplySearchOptions(const SearchOptions& options,
                        OgaGeneratorParams& gen_params,
                        ExecutionProvider ep,
                        bool use_full_context = false,
-                       int default_max_output_tokens = kDefaultChatTextMaxOutputTokens);
+                       int default_max_output_tokens = kDefaultChatTextMaxOutputTokens,
+                       bool supports_external_prefill_chunking = true);
 
 /// Applies request-level grammar guidance to generator parameters when the tool context requires tool-only output.
 void ApplyGuidanceOptions(const ToolCallContext& tool_ctx,

sdk_v2/cpp/test/internal_api/chat/search_options_test.cc

@@ -894,6 +894,16 @@ TEST_F(SearchOptionsTest, ChunkedPrefillSkippedForUnlistedEp) {
   EXPECT_EQ(params->GetSearchNumber("chunk_size"), 0);
 }
 
+TEST_F(SearchOptionsTest, ChunkedPrefillSkippedWhenModelDoesNotSupportIt) {
+  SearchOptions opts;
+  auto params = MakeParams();
+
+  ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kCUDA,
+                     /*use_full_context=*/false, kDefaultChatTextMaxOutputTokens,
+                     /*supports_external_prefill_chunking=*/false);
+  EXPECT_EQ(params->GetSearchNumber("chunk_size"), 0);
+}
+
 TEST_F(SearchOptionsTest, ChunkedPrefillResolvesSupportedProvidersFromConfig) {
   SearchOptions opts;
   GenAIConfig config;