diff --git a/.pipelines/templates/stages-build-native.yml b/.pipelines/templates/stages-build-native.yml index 5ce457838..4f98ec8dd 100644 --- a/.pipelines/templates/stages-build-native.yml +++ b/.pipelines/templates/stages-build-native.yml @@ -66,6 +66,7 @@ stages: - compute_version jobs: - job: build + timeoutInMinutes: 120 pool: name: onnxruntime-medium-arm64-scus os: windows diff --git a/.pipelines/templates/steps-build-linux.yml b/.pipelines/templates/steps-build-linux.yml index 92b5a3256..d989a2f2a 100644 --- a/.pipelines/templates/steps-build-linux.yml +++ b/.pipelines/templates/steps-build-linux.yml @@ -116,7 +116,8 @@ steps: "GENAI_FETCH_URL=$NUGET_CACHE/genai.zip" \ "ORT_GENAI_VERSION=${{ parameters.genaiVersion }}" \ "ORT_FETCH_URL=$NUGET_CACHE/ort.zip" \ - "FOUNDRY_LOCAL_PACKAGE_VERSION=$SDK_VERSION" + "FOUNDRY_LOCAL_PACKAGE_VERSION=$SDK_VERSION" \ + "FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=ON" # Model-free loader checks run on every Linux build, even when the # model-dependent suite is disabled for this lane. TF_BUILD=true "$HOME/venv/bin/ctest" \ diff --git a/.pipelines/templates/steps-build-macos.yml b/.pipelines/templates/steps-build-macos.yml index 52a2516fc..42a18e868 100644 --- a/.pipelines/templates/steps-build-macos.yml +++ b/.pipelines/templates/steps-build-macos.yml @@ -66,7 +66,7 @@ steps: set -euo pipefail python3 build.py --configure --build \ --config ${{ parameters.buildConfig }} \ - --cmake_extra_defines $(cmakeFetchDefines) + --cmake_extra_defines $(cmakeFetchDefines) FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=ON displayName: 'Configure and build' workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp env: diff --git a/.pipelines/templates/steps-build-windows.yml b/.pipelines/templates/steps-build-windows.yml index b44c54044..663c81cda 100644 --- a/.pipelines/templates/steps-build-windows.yml +++ b/.pipelines/templates/steps-build-windows.yml @@ -98,12 +98,13 @@ steps: --config ${{ parameters.buildConfig }} --cmake_generator "Visual Studio 17 2022" --winml_sdk_version ${{ parameters.winmlVersion }} - --cmake_extra_defines $(cmakeFetchDefines) + --cmake_extra_defines $(cmakeFetchDefines) FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=ON displayName: 'Configure and build (x64)' workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg VCPKG_DOWNLOADS: $(Build.BinariesDirectory)\vcpkg-downloads + VCPKG_MAX_CONCURRENCY: '1' PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat - ${{ if eq(parameters.arch, 'arm64') }}: @@ -113,12 +114,13 @@ steps: --config ${{ parameters.buildConfig }} --cmake_generator "Visual Studio 17 2022" --winml_sdk_version ${{ parameters.winmlVersion }} - --cmake_extra_defines $(cmakeFetchDefines) + --cmake_extra_defines $(cmakeFetchDefines) FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=ON displayName: 'Configure and build (arm64)' workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg VCPKG_DOWNLOADS: $(Build.BinariesDirectory)\vcpkg-downloads + VCPKG_MAX_CONCURRENCY: '1' PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat - ${{ if eq(parameters.arch, 'x64') }}: diff --git a/sdk_v2/cpp/README.md b/sdk_v2/cpp/README.md index 5a37febc0..ebcc8d61d 100644 --- a/sdk_v2/cpp/README.md +++ b/sdk_v2/cpp/README.md @@ -117,6 +117,79 @@ required runtime directories when using it. It is not needed solely for the x64 Foundry Local leaves runtime dependency loading to ORT and GenAI. On Linux, use the system loader configuration or `LD_LIBRARY_PATH`. +## Catalog GPU Memory Compatibility + +Native catalog filtering interprets an explicit `variantMetadata.vRamFootprintBytes` as the variant's minimum +total GPU memory requirement, in bytes. It is retained as the optional integer property `min_vram_bytes`. +Only variants explicitly marked as GPU are checked. CPU and NPU compatibility rules are unchanged. +Model file size is not used as a memory requirement. Missing, null, malformed, negative, or out-of-range +requirements are unknown, not zero. + +A variant is excluded only when every candidate device for its execution provider has known total VRAM below +the requirement. Multiple devices are not summed. A fitting device, an unknown capacity, an unsupported query, +or a query failure leaves capacity compatibility undecided and does not exclude the variant. An alias remains +visible when a compatible variant remains. Full metadata snapshots are saved before capacity filtering so +offline catalogs can be reevaluated on different hardware. Previously returned handles remain stable across +refreshes; memory-incompatible retained variants are hidden from discovery and cannot become automatic defaults. +They remain accessible by ID and in cached/loaded inventory, and become visible again if capacity permits. + +Capacity queries use ORT's device-memory total only, never current available VRAM. This is not a guarantee +that loading will succeed. CPU RAM and unified-memory checks are deliberately not implemented and require +separate metadata and measurement contracts. + +The query requires the actual `EpDevice_GetMemoryCapacity` API and matching CUDA provider built with +[ORT PR #33447](https://github.com/microsoft/onnxruntime/pull/33447). An API 32 version alone is insufficient: +pre-merge 1.32 nightlies lack the member. CMake checks the member and warns if capacity filtering is inactive. +The API is required by default: configuration fails when the selected headers lack the member. +The pinned 1.31 runtime and CUDA bundles predate the API, so they cannot satisfy the shipping requirement. +The dependency rollout remains a merge requirement: wait for the ORT 1.32.0 release, then update the runtime +and matching provider packages together, including verified provider versions and hashes. Do not substitute +an API version number or a pre-release nightly for a verified release rollout. +Shipping native build templates explicitly require the API, overriding cached or supplied compatibility settings. +Until the release packages and matching providers are available, use a source-built ORT with matching headers +in `/include` and runtime libraries in `/lib`: + +```bash +python sdk_v2/cpp/build.py --configure --build --ort_home +``` + +For source compatibility only, explicitly opt out when building against an older ORT: + +```bash +python sdk_v2/cpp/build.py --configure --build \ + --cmake_extra_defines FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=OFF +``` + +This opt-out warns and treats capacity as unknown if the API is absent; it does not disable the query when +the selected headers support it. It is not a shipping configuration. Before completing the rollout, exercise +the enabled query path with the released runtime and matching provider; a compiled-out or skipped test does +not validate packaged filtering. + +Register the matching CUDA provider before refreshing the catalog. Do not mix provider and runtime builds. +Hosted catalogs use the execution host's provider/device/capacity snapshot, never the client's local GPU. +Older hosts that report only provider names retain unknown-capacity behavior. + +### Real-GPU catalog end-to-end test + +`AzureModelCatalogCapacityEndToEndTest.RealCudaCapacityFiltersParsedCatalogAndOfflineSnapshot` uses the real +ORT environment, registered CUDA provider, EP detector, Azure catalog parser, selection logic, and disk snapshot. +Only the catalog HTTP response is replaced with controlled metadata; no model downloads or inference are required. +Requirements are derived from the detected devices, covering exact total-capacity fit, oversized GPU rejection, +CPU fallback, aliases with no fitting variants, missing requirements, explicit zero, and offline snapshot reuse. + +Build with the capacity-enabled ORT described above and run from the repository root on Linux: + +```bash +FOUNDRY_LOCAL_TEST_CUDA_EP_LIBRARY=/absolute/path/to/libonnxruntime_providers_cuda.so \ + sdk_v2/cpp/build/Linux/Release/bin/foundry_local_tests \ + --gtest_filter='AzureModelCatalogCapacityEndToEndTest.*' +``` + +The provider must match the ORT runtime used to build the tests. Configure CUDA and cuDNN library search paths +as for normal CUDA usage. The test skips when built without the capacity API or when the provider path is unset. +Once the provider path is set, registration failures, missing visible GPUs, and failed capacity queries fail +the test. Set `CUDA_VISIBLE_DEVICES=7,2` (or another valid subset) to verify reordered device visibility. + ## Clearing a Model Cache Use `Model::RemoveFromCache()` to clear a completed, interrupted, or corrupted download. diff --git a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake index 8a2bf3772..f020e1e50 100644 --- a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake +++ b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake @@ -134,6 +134,36 @@ if(NOT EXISTS "${_ORT_HEADER_DIR}/onnxruntime_c_api.h") message(FATAL_ERROR "OnnxRuntime header not found at ${_ORT_HEADER_DIR}/onnxruntime_c_api.h") endif() +option(FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY + "Require an ORT build supporting EP device total-memory queries" ON) +include(CheckCXXSourceCompiles) +include(CMakePushCheckState) +cmake_push_check_state(RESET) +set(CMAKE_REQUIRED_INCLUDES "${_ORT_HEADER_DIR}") +unset(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY CACHE) +check_cxx_source_compiles(" + #include + int main() { + using CapacityQuery = decltype(&OrtApi::EpDevice_GetMemoryCapacity); + return sizeof(CapacityQuery) == 0; + }" FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) +cmake_pop_check_state() +if(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) + add_compile_definitions(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY=1) + message(STATUS "ORT EP device-memory capacity API enabled; provider support is checked at runtime") +elseif(FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY) + message(FATAL_ERROR + "ORT lacks EpDevice_GetMemoryCapacity. Use --ort_home with a source build containing ORT PR #33447, " + "including matching runtime headers/libraries and CUDA provider. An API 32 version alone is insufficient. " + "Shipping builds require this API and matching ORT 1.32.0 release runtime/provider packages. " + "For source compatibility only, explicitly set FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=OFF.") +else() + message(WARNING + "ORT lacks EpDevice_GetMemoryCapacity: catalog VRAM capacity filtering is inactive. " + "FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=OFF is for source compatibility only, not shipping. " + "Use a post-PR #33447 ORT build via --ort_home with matching runtime headers/libraries and provider.") +endif() + # Create imported target add_library(OnnxRuntime::OnnxRuntime SHARED IMPORTED) set_target_properties(OnnxRuntime::OnnxRuntime PROPERTIES diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h index 735f17045..788e7da5e 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h @@ -280,6 +280,7 @@ typedef enum flTensorDataType { #define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT "supports_tool_calling" ///< optional bool (not set or -1=unknown, 0=false, 1=true) #define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT "supports_reasoning" ///< optional bool (not set or -1=unknown, 0=false, 1=true) #define FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT "filesize_mb" ///< optional int32_t +#define FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT "min_vram_bytes" ///< optional int64_t #define FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT "max_output_tokens" ///< optional int32_t #define FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT "created_at_unix" ///< Unix timestamp. default=0 #define FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT "is_test_model" ///< bool (0=false, 1=true) diff --git a/sdk_v2/cpp/src/api/backend.h b/sdk_v2/cpp/src/api/backend.h index b6a5e1bce..faa3b38b1 100644 --- a/sdk_v2/cpp/src/api/backend.h +++ b/sdk_v2/cpp/src/api/backend.h @@ -4,6 +4,7 @@ #include +#include "ep_detection/execution_target.h" #include "inferencing/session/streaming_callback.h" #include @@ -45,6 +46,11 @@ struct InputOutputInfo { using EpProgressCallback = std::function; using DeviceProviders = std::map>; +struct RuntimeDeviceSnapshot { + DeviceProviders devices; + std::vector execution_targets; +}; + class BackendObject { public: BackendKind Kind() const noexcept { return kind_; } @@ -136,7 +142,9 @@ class ManagerBackend : public BackendObject { virtual void StopWebService() = 0; virtual std::span GetDiscoverableEps() const = 0; virtual DeviceProviders GetRuntimeDevices() const = 0; - virtual void SetCatalogRuntimeDevices(DeviceProviders devices) = 0; + virtual RuntimeDeviceSnapshot GetRuntimeDeviceSnapshot() const { return {GetRuntimeDevices(), {}}; } + virtual void SetCatalogRuntimeDevices(DeviceProviders devices, + std::vector execution_targets = {}) = 0; virtual EpDownloadResult DownloadAndRegisterEps(const std::vector* names, const EpProgressCallback& progress) = 0; virtual bool IsEpDownloadInProgress() const = 0; diff --git a/sdk_v2/cpp/src/api/embedded_backend.cc b/sdk_v2/cpp/src/api/embedded_backend.cc index f8b4159ed..d68b019d0 100644 --- a/sdk_v2/cpp/src/api/embedded_backend.cc +++ b/sdk_v2/cpp/src/api/embedded_backend.cc @@ -256,9 +256,13 @@ class EmbeddedManager final : public ManagerBackend { return owner_.manager.GetEpDetector().GetAvailableDevicesToEPs(); } - void SetCatalogRuntimeDevices(DeviceProviders devices) override { + RuntimeDeviceSnapshot GetRuntimeDeviceSnapshot() const override { + return {GetRuntimeDevices(), owner_.manager.GetEpDetector().GetExecutionTargets()}; + } + + void SetCatalogRuntimeDevices(DeviceProviders devices, std::vector execution_targets) override { static_cast(owner_.manager.GetCatalog(CatalogType::kPublic)) - .SetRuntimeDevices(std::move(devices)); + .SetRuntimeDevices(std::move(devices), std::move(execution_targets)); } EpDownloadResult DownloadAndRegisterEps(const std::vector* names, diff --git a/sdk_v2/cpp/src/api/hosted_manager.cc b/sdk_v2/cpp/src/api/hosted_manager.cc index 844bbb631..013242ca6 100644 --- a/sdk_v2/cpp/src/api/hosted_manager.cc +++ b/sdk_v2/cpp/src/api/hosted_manager.cc @@ -93,9 +93,10 @@ class HostedState { }); if (catalog == CatalogType::kPublic) { const auto context = Connect(false); - metadata->SetCatalogRuntimeDevices( - context ? context->Call(Operation::RuntimeDevices, Json::object()).get() - : metadata->GetRuntimeDevices()); + auto snapshot = context ? DecodeRuntimeDeviceSnapshot( + context->Call(Operation::RuntimeDevices, {{"include_execution_targets", true}})) + : RuntimeDeviceSnapshot{}; + metadata->SetCatalogRuntimeDevices(std::move(snapshot.devices), std::move(snapshot.execution_targets)); } } @@ -605,14 +606,19 @@ class HostedManager final : public ManagerBackend { } DeviceProviders GetRuntimeDevices() const override { + return GetRuntimeDeviceSnapshot().devices; + } + + RuntimeDeviceSnapshot GetRuntimeDeviceSnapshot() const override { const auto context = state_->Connect(false); - return context ? context->Call(Operation::RuntimeDevices, Json::object()).get() - : state_->metadata->GetRuntimeDevices(); + return context ? DecodeRuntimeDeviceSnapshot( + context->Call(Operation::RuntimeDevices, {{"include_execution_targets", true}})) + : RuntimeDeviceSnapshot{}; } - void SetCatalogRuntimeDevices(DeviceProviders devices) override { + void SetCatalogRuntimeDevices(DeviceProviders devices, std::vector execution_targets) override { state_->RequireOpen(); - state_->metadata->SetCatalogRuntimeDevices(std::move(devices)); + state_->metadata->SetCatalogRuntimeDevices(std::move(devices), std::move(execution_targets)); } EpDownloadResult DownloadAndRegisterEps(const std::vector* names, diff --git a/sdk_v2/cpp/src/catalog/azure_catalog_models.cc b/sdk_v2/cpp/src/catalog/azure_catalog_models.cc index eb28ea4a0..c8ba885ca 100644 --- a/sdk_v2/cpp/src/catalog/azure_catalog_models.cc +++ b/sdk_v2/cpp/src/catalog/azure_catalog_models.cc @@ -213,6 +213,14 @@ void from_json(const nlohmann::json& j, VariantMetadata& v) { opt_str(j, "device", v.device); opt_str(j, "executionProvider", v.execution_provider); opt_int64(j, "fileSizeBytes", v.file_size_bytes); + const auto footprint = j.find("vRamFootprintBytes"); + if (footprint != j.end() && footprint->is_number_integer() && + (!footprint->is_number_unsigned() || footprint->get() <= uint64_t{INT64_MAX})) { + const auto bytes = footprint->get(); + if (bytes >= 0) { + v.vram_footprint_bytes = bytes; + } + } } void from_json(const nlohmann::json& j, VariantParent& v) { @@ -457,6 +465,9 @@ std::optional CatalogModelToModelInfo(const CatalogLocalModel& cm) { if (vm.file_size_bytes) { info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT] = *vm.file_size_bytes / (1024 * 1024); } + if (vm.vram_footprint_bytes) { + info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = *vm.vram_footprint_bytes; + } } else { info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX"; } diff --git a/sdk_v2/cpp/src/catalog/azure_catalog_models.h b/sdk_v2/cpp/src/catalog/azure_catalog_models.h index 25cc495bc..3d5097587 100644 --- a/sdk_v2/cpp/src/catalog/azure_catalog_models.h +++ b/sdk_v2/cpp/src/catalog/azure_catalog_models.h @@ -56,6 +56,7 @@ struct VariantMetadata { std::optional device; std::optional execution_provider; std::optional file_size_bytes; + std::optional vram_footprint_bytes; }; struct VariantParent { diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index fcd44c07e..528581a67 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "catalog/azure_model_catalog.h" +#include "catalog/capacity_compatibility.h" #include "catalog/azure_catalog_client.h" #include "catalog/catalog_cache.h" #include "catalog/catalog_client.h" @@ -158,13 +159,47 @@ class AzureModelCatalog::CatalogDevices final : public IEpDetector { return hosted_ ? *hosted_ : local_.GetAvailableDevicesToEPs(); } - bool Set(Devices devices) { + std::map> GetDeviceCapacities() const override { std::lock_guard lock(mutex_); - if (hosted_ && *hosted_ == devices) { + if (!hosted_) { + return local_.GetDeviceCapacities(); + } + + std::map> capacities; + for (const auto& target : execution_targets_) { + capacities[target.provider].push_back(target.capacity); + } + + return capacities; + } + + std::vector GetExecutionTargets() const override { + std::lock_guard lock(mutex_); + if (!hosted_) { + return local_.GetExecutionTargets(); + } + auto targets = execution_targets_; + for (const auto& [type, providers] : *hosted_) { + for (const auto& provider : providers) { + if (std::none_of(targets.begin(), targets.end(), [&](const auto& target) { + return ToLower(EPUtils::NormalizeRegistrationName(target.provider)) == + ToLower(EPUtils::NormalizeRegistrationName(provider)); + })) { + targets.push_back({provider, 0, type, {}}); + } + } + } + return targets; + } + + bool Set(Devices devices, std::vector execution_targets) { + std::lock_guard lock(mutex_); + if (hosted_ && *hosted_ == devices && execution_targets_ == execution_targets) { return false; } hosted_ = std::move(devices); + execution_targets_ = std::move(execution_targets); return true; } @@ -172,6 +207,7 @@ class AzureModelCatalog::CatalogDevices final : public IEpDetector { const IEpDetector& local_; mutable std::mutex mutex_; std::optional hosted_; + std::vector execution_targets_; }; AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, @@ -200,28 +236,56 @@ AzureModelCatalog::AzureModelCatalog(std::vector> devices) { - if (catalog_devices_->Set(std::move(devices))) { +void AzureModelCatalog::SetRuntimeDevices(std::map> devices, + std::vector execution_targets) { + if (catalog_devices_->Set(std::move(devices), std::move(execution_targets))) { InvalidateCache(); } } BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility() const { - std::unordered_set providers; - for (const auto& [device, eps] : catalog_devices_->GetAvailableDevicesToEPs()) { - for (const auto& ep : eps) { - providers.insert(ToLower(EPUtils::NormalizeRegistrationName(ep))); + return MakeCompatibilityPredicate(false); +} + +BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantVisibility() const { + return [this, compatible = VariantEligibility{}](const Model& variant) mutable { + if (variant.Info().device_type != DeviceType::kGPU || + !variant.Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT)) { + return true; } + if (!compatible) { + compatible = MakeCompatibilityPredicate(true); + } + return compatible(variant); + }; +} + +BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeCompatibilityPredicate(bool capacity_only) const { + auto targets = catalog_devices_->GetExecutionTargets(); + std::unordered_set providers; + for (const auto& target : targets) { + providers.insert(ToLower(EPUtils::NormalizeRegistrationName(target.provider))); } // Cache config requirements only within this pass so artifact edits and provider changes are seen on refresh. - return [this, providers = std::move(providers), + return [this, capacity_only, providers = std::move(providers), targets = std::move(targets), configured_providers = std::unordered_map>{}]( const Model& variant) mutable { const auto& info = variant.Info(); + ModelCapacityRequirements requirements; + if (const auto* bytes = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); bytes && *bytes >= 0) { + requirements.minimum_vram_bytes = static_cast(*bytes); + } + if (capacity_only && (info.device_type != DeviceType::kGPU || !requirements.minimum_vram_bytes)) { + return true; + } + const auto meets_capacity = [&](const std::string& provider, bool generic_gpu = false) { + return EvaluateProviderCapacity(info.device_type == DeviceType::kGPU, requirements, targets, + provider, generic_gpu).state != CompatibilityState::kDoesNotFit; + }; if (info.model_id.find("generic-gpu") != std::string::npos && (providers.contains("cudaexecutionprovider") || providers.contains("webgpuexecutionprovider"))) { - return true; + return meets_capacity({}, true); } auto provider = info.execution_provider; @@ -240,7 +304,7 @@ BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility() } if (!entry->second) { - return false; + return capacity_only; } provider = *entry->second; @@ -252,10 +316,31 @@ BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility() } provider = ToLower(EPUtils::NormalizeRegistrationName(provider)); - return provider.empty() || provider == "cpuexecutionprovider" || providers.contains(provider); + return (capacity_only || provider.empty() || provider == "cpuexecutionprovider" || providers.contains(provider)) && + meets_capacity(provider); }; } +void AzureModelCatalog::FilterCapacityIncompatibleModels(std::vector& models) const { + const bool has_requirements = std::any_of(models.begin(), models.end(), [](const Model& model) { + return model.Info().device_type == DeviceType::kGPU && + model.Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT) != nullptr; + }); + if (!has_requirements) { + return; + } + auto compatible = MakeCompatibilityPredicate(true); + std::erase_if(models, [&](const Model& model) { + if (compatible(model)) { + return false; + } + logger_.Log(LogLevel::Information, + fmt::format("Catalog: filtering variant '{}' because its minimum total GPU VRAM exceeds capacity.", + model.Info().model_id)); + return true; + }); +} + std::unique_ptr AzureModelCatalog::CreateCatalogClient(const std::string& url, const std::string& filter) const { return MakeCatalogClient(url, filter, *catalog_devices_, logger_, cache_dir_); @@ -384,6 +469,7 @@ std::vector AzureModelCatalog::FetchModelVersions( out.push_back(model_factory_(std::move(info), /*local_path=*/"")); } + FilterCapacityIncompatibleModels(out); logger_.Log(LogLevel::Information, fmt::format("FetchModelVersions('{}') returned {} variant(s).", model_alias, out.size())); diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.h b/sdk_v2/cpp/src/catalog/azure_model_catalog.h index 7b9c4593e..9384c2bbe 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.h @@ -37,7 +37,8 @@ class AzureModelCatalog : public BaseModelCatalog { ~AzureModelCatalog() override; // Metadata-only override for hosted clients; never changes the native runtime's registered providers. - void SetRuntimeDevices(std::map> devices); + void SetRuntimeDevices(std::map> devices, + std::vector execution_targets = {}); protected: std::vector FetchModels() const override; @@ -48,6 +49,7 @@ class AzureModelCatalog : public BaseModelCatalog { virtual std::unique_ptr CreateCatalogClient(const std::string& url, const std::string& filter) const; VariantEligibility MakeVariantEligibility() const override; + VariantEligibility MakeVariantVisibility() const override; void RefreshRetainedVariantCache(Model& model) const override; private: @@ -66,6 +68,9 @@ class AzureModelCatalog : public BaseModelCatalog { static constexpr const char* kDefaultCatalogUrl = Configuration::kDefaultCatalogUrl; static constexpr const char* kDefaultCatalogFilter = Configuration::kDefaultCatalogFilter; + VariantEligibility MakeCompatibilityPredicate(bool capacity_only) const; + void FilterCapacityIncompatibleModels(std::vector& models) const; + CatalogResult GetLiveCatalogOrLocalSnapshot(const std::vector& cached_model_ids) const; std::vector CreateModelsWithLocalPaths(const std::vector& model_infos, const LocalModels& local_models) const; diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 9051e0ddd..117b81756 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -22,7 +22,9 @@ BaseModelCatalog::BaseModelCatalog(std::string name, CatalogType type, ILogger& BaseModelCatalog::~BaseModelCatalog() = default; void BaseModelCatalog::SelectDefaultVariant(Model& model, const VariantEligibility& is_eligible, + const VariantEligibility& is_visible, bool preserve_explicit_selection) const { + model.UpdateVariantVisibility(is_visible); if (type_ == CatalogType::kPublic) { model.SetDefaultVariantRefresh([this] { EnsurePopulated(); }); } @@ -35,7 +37,7 @@ void BaseModelCatalog::SelectDefaultVariant(Model& model, const VariantEligibili logger_.Log(LogLevel::Information, fmt::format("Catalog '{}': skipping variant '{}' for default selection because " - "execution provider '{}' is unavailable.", + "execution provider '{}' is unavailable or device capacity is incompatible.", name_, variant.Id(), variant.Info().execution_provider)); return false; }, @@ -59,6 +61,7 @@ void BaseModelCatalog::SelectDefaultVariant(Model& model, const VariantEligibili void BaseModelCatalog::PopulateModels(std::vector variants) const { const auto is_eligible = MakeVariantEligibility(); + const auto is_visible = MakeVariantVisibility(); // Group variants by alias into Model containers. // Matches C# Catalog.UpdateModels() pattern: @@ -122,11 +125,11 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { } std::unordered_map existing_variants; - for (auto* variant : existing->second->Variants()) { + for (auto* variant : existing->second->Variants(/*include_hidden=*/true)) { existing_variants.emplace(variant->Id(), variant); } - for (auto* variant : incoming.Variants()) { + for (auto* variant : incoming.Variants(/*include_hidden=*/true)) { const auto known = existing_variants.find(variant->Id()); if (known == existing_variants.end()) { existing->second->AddVariant(std::move(*variant)); @@ -143,7 +146,7 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { for (const auto& [alias, existing] : existing_aliases) { if (!alias_to_model.contains(alias)) { - for (auto* variant : existing->Variants()) { + for (auto* variant : existing->Variants(/*include_hidden=*/true)) { RefreshRetainedVariantCache(*variant); } } @@ -152,14 +155,14 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { for (auto& stored : models_) { if (stored.active) { - SelectDefaultVariant(*stored.model, is_eligible, /*preserve_explicit_selection=*/true); + SelectDefaultVariant(*stored.model, is_eligible, is_visible, /*preserve_explicit_selection=*/true); } } size_t new_count = 0; for (auto& [alias, model] : alias_to_model) { if (!existing_aliases.contains(alias)) { - SelectDefaultVariant(model, is_eligible); + SelectDefaultVariant(model, is_eligible, is_visible); models_.push_back({std::make_unique(std::move(model)), true}); ++new_count; } @@ -178,7 +181,7 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { // Initial population: move all models into stable storage. models_.reserve(alias_to_model.size()); for (auto& [alias, model] : alias_to_model) { - SelectDefaultVariant(model, is_eligible); + SelectDefaultVariant(model, is_eligible, is_visible); models_.push_back({std::make_unique(std::move(model)), true}); } @@ -198,6 +201,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { } const auto is_eligible = MakeVariantEligibility(); + const auto is_visible = MakeVariantVisibility(); // Build a lookup of existing aliases -> containers so we can merge new // variants in O(1) per incoming variant. @@ -215,7 +219,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { if (!stored.active) { continue; } - for (auto* v : stored.model->Variants()) { + for (auto* v : stored.model->Variants(/*include_hidden=*/true)) { existing_ids.insert(v->Info().model_id); } } @@ -251,7 +255,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { ++added_variants; } - SelectDefaultVariant(*it->second, is_eligible, /*preserve_explicit_selection=*/true); + SelectDefaultVariant(*it->second, is_eligible, is_visible, /*preserve_explicit_selection=*/true); } else { // New alias: build a container and choose default after all variants are added. auto first = std::move(alias_variants.front()); @@ -260,7 +264,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { container.AddVariant(std::move(alias_variants[i])); } - SelectDefaultVariant(container, is_eligible); + SelectDefaultVariant(container, is_eligible, is_visible); models_.push_back({std::make_unique(std::move(container)), true}); ++added_aliases; @@ -288,7 +292,7 @@ void BaseModelCatalog::RebuildIndex() const { auto& m = stored.model; new_index->alias_index[m->Alias()] = m.get(); - for (auto* variant : m->Variants()) { + for (auto* variant : m->Variants(/*include_hidden=*/true)) { const auto& info = variant->Info(); if (!new_index->id_index.contains(info.model_id)) { @@ -369,6 +373,7 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { std::vector BaseModelCatalog::ListModels() const { EnsurePopulated(/*allow_refresh=*/true); + const auto is_visible = MakeVariantVisibility(); // PopulateModels appends to models_ under mutex_; iterate under the same lock so a // concurrent EnsurePopulated cannot reallocate the vector mid-iteration. @@ -377,6 +382,11 @@ std::vector BaseModelCatalog::ListModels() const { result.reserve(models_.size()); for (auto& stored : models_) { if (stored.active) { + const auto variants = stored.model->Variants(); + if (is_visible && !std::any_of(variants.begin(), variants.end(), + [&](const Model* variant) { return is_visible(*variant); })) { + continue; + } result.push_back(stored.model.get()); } } @@ -488,7 +498,7 @@ std::vector BaseModelCatalog::GetCachedModels() const { continue; } - for (auto* variant : stored.model->Variants()) { + for (auto* variant : stored.model->Variants(/*include_hidden=*/true)) { if (variant->IsCached()) { result.push_back(variant); } @@ -508,7 +518,7 @@ std::vector BaseModelCatalog::GetLoadedModels() const { continue; } - for (auto* variant : stored.model->Variants()) { + for (auto* variant : stored.model->Variants(/*include_hidden=*/true)) { if (variant->IsLoaded()) { result.push_back(variant); } @@ -522,7 +532,7 @@ Model* BaseModelCatalog::AppendActiveModel(Model model) { EnsurePopulated(); std::lock_guard lock(mutex_); auto container = std::make_unique(Model::MakeContainer(std::move(model))); - SelectDefaultVariant(*container, MakeVariantEligibility()); + SelectDefaultVariant(*container, MakeVariantEligibility(), MakeVariantVisibility()); auto* result = container.get(); models_.push_back({std::move(container), true}); RebuildIndex(); @@ -538,7 +548,7 @@ bool BaseModelCatalog::RetireModel(const std::string& alias_or_model_id) { } bool matches = stored.model->Alias() == alias_or_model_id; - for (auto* variant : stored.model->Variants()) { + for (auto* variant : stored.model->Variants(/*include_hidden=*/true)) { matches = matches || variant->Id() == alias_or_model_id; } @@ -617,7 +627,7 @@ std::vector BaseModelCatalog::GetModelVersions(const std::string& model_ container.AddVariant(std::move(fetched[i])); } - SelectDefaultVariant(container, MakeVariantEligibility()); + SelectDefaultVariant(container, MakeVariantEligibility(), MakeVariantVisibility()); version_query_models_.push_back(std::make_unique(std::move(container))); // Return variant pointers from the container (like Model_GetVariantsImpl). diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.h b/sdk_v2/cpp/src/catalog/base_model_catalog.h index e8e551f9c..d199c92c5 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.h @@ -100,6 +100,7 @@ class BaseModelCatalog : public ICatalog { /// Each selection pass uses one provider snapshot; an empty predicate makes every variant eligible. using VariantEligibility = std::function; virtual VariantEligibility MakeVariantEligibility() const { return {}; } + virtual VariantEligibility MakeVariantVisibility() const { return {}; } private: /// Lookup indices into the stable models_ storage. @@ -136,6 +137,7 @@ class BaseModelCatalog : public ICatalog { void PopulateModels(std::vector variants) const; void SelectDefaultVariant(Model& model, const VariantEligibility& is_eligible, + const VariantEligibility& is_visible, bool preserve_explicit_selection = false) const; /// Merge new variants into the catalog's stable storage. For an diff --git a/sdk_v2/cpp/src/catalog/capacity_compatibility.h b/sdk_v2/cpp/src/catalog/capacity_compatibility.h new file mode 100644 index 000000000..9e0c1b7e2 --- /dev/null +++ b/sdk_v2/cpp/src/catalog/capacity_compatibility.h @@ -0,0 +1,93 @@ +#pragma once + +#include "ep_detection/execution_target.h" +#include "inferencing/execution_provider.h" +#include "utils.h" + +#include +#include +#include +#include + +namespace fl { + +struct ModelCapacityRequirements { + std::optional minimum_vram_bytes; +}; + +enum class CompatibilityState { kFits, kDoesNotFit, kUnknown }; +enum class CompatibilityReason { kSufficientCapacity, kInsufficientCapacity, kMissingRequirement, kUnknownCapacity }; + +struct CompatibilityResult { + CompatibilityState state; + CompatibilityReason reason; + std::vector eligible_targets; +}; + +inline CompatibilityResult EvaluateCapacity(bool is_gpu, const ModelCapacityRequirements& requirements, + std::span targets) { + CompatibilityResult result{CompatibilityState::kUnknown, CompatibilityReason::kUnknownCapacity, {}}; + if (!is_gpu || !requirements.minimum_vram_bytes) { + result.reason = CompatibilityReason::kMissingRequirement; + result.eligible_targets.assign(targets.begin(), targets.end()); + return result; + } + bool unknown = targets.empty(); + bool fits = false; + for (const auto& target : targets) { + if (target.device_type != "GPU" || !target.capacity.total_vram_bytes) { + unknown = true; + result.eligible_targets.push_back(target); + } else if (*target.capacity.total_vram_bytes >= *requirements.minimum_vram_bytes) { + fits = true; + result.eligible_targets.push_back(target); + } + } + if (fits) { + result.state = CompatibilityState::kFits; + result.reason = CompatibilityReason::kSufficientCapacity; + std::stable_partition(result.eligible_targets.begin(), result.eligible_targets.end(), [&](const auto& target) { + return target.device_type == "GPU" && target.capacity.total_vram_bytes && + *target.capacity.total_vram_bytes >= *requirements.minimum_vram_bytes; + }); + } else if (!unknown) { + result.state = CompatibilityState::kDoesNotFit; + result.reason = CompatibilityReason::kInsufficientCapacity; + } + return result; +} + +inline CompatibilityResult EvaluateProviderCapacity(bool is_gpu, const ModelCapacityRequirements& requirements, + std::span targets, + std::string_view required_provider, bool generic_gpu = false) { + const auto normalize = [](std::string_view provider) { + return ToLower(EPUtils::NormalizeRegistrationName(std::string(provider))); + }; + std::vector candidates; + const auto append_provider = [&](std::string_view provider) { + for (const auto& target : targets) { + if (normalize(target.provider) == normalize(provider)) { + candidates.push_back(target); + } + } + }; + if (generic_gpu) { + append_provider("CUDAExecutionProvider"); + append_provider("WebGpuExecutionProvider"); + } + if (candidates.empty()) { + append_provider(required_provider); + } + return EvaluateCapacity(is_gpu, requirements, candidates); +} + +inline bool MeetsCapacityRequirements(bool is_gpu, const ModelCapacityRequirements& requirements, + std::span devices) { + std::vector targets; + for (const auto& device : devices) { + targets.push_back({{}, 0, "GPU", device}); + } + return EvaluateCapacity(is_gpu, requirements, targets).state != CompatibilityState::kDoesNotFit; +} + +} // namespace fl \ No newline at end of file diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index c2e3dc413..fecc7fd34 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -35,6 +35,34 @@ bool IsTrtRtxEp(std::string_view name) { } // namespace +std::vector IEpDetector::GetExecutionTargets() const { + const auto capacities = GetDeviceCapacities(); + std::map provider_types; + for (const auto& [device_type, providers] : GetAvailableDevicesToEPs()) { + for (const auto& provider : providers) { + provider_types.emplace(provider, device_type); + } + } + + std::vector targets; + for (const auto& [provider, device_capacities] : capacities) { + const auto type = provider_types.find(provider); + for (size_t index = 0; index < device_capacities.size(); ++index) { + targets.push_back({provider, static_cast(index), + type == provider_types.end() ? std::string{} : type->second, device_capacities[index]}); + } + } + + for (const auto& [provider, device_type] : provider_types) { + const auto found = capacities.find(provider); + if (found == capacities.end() || found->second.empty()) { + targets.push_back({provider, 0, device_type, {}}); + } + } + + return targets; +} + EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, ILogger& logger, @@ -129,6 +157,76 @@ std::map> EpDetector::GetAvailableDevicesT return devices; } +std::map> EpDetector::GetDeviceCapacities() const { + std::map> capacities; + for (const auto& target : GetExecutionTargets()) { + if (target.device_type == "GPU") { + capacities[target.provider].push_back(target.capacity); + } + } + + return capacities; +} + +std::vector EpDetector::GetExecutionTargets() const { + std::vector targets; + const OrtEpDevice* const* devices = nullptr; + size_t count = 0; + if (auto* status = ort_api_.GetEpDevices(&ort_env_, &devices, &count)) { + logger_.Log(LogLevel::Warning, "Cannot enumerate execution targets"); + ort_api_.ReleaseStatus(status); + return targets; + } + + targets.reserve(count); + for (size_t index = 0; index < count; ++index) { + const auto* device = devices[index]; + const auto* provider = ort_api_.EpDevice_EpName(device); + if (!provider) { + continue; + } + + const auto* hardware = ort_api_.EpDevice_Device(device); + ExecutionTarget target{provider, ort_api_.HardwareDevice_DeviceId(hardware), {}, {}}; + if (const auto* options = ort_api_.EpDevice_EpOptions(device)) { + const char* const* keys = nullptr; + const char* const* values = nullptr; + size_t option_count = 0; + ort_api_.GetKeyValuePairs(options, &keys, &values, &option_count); + for (size_t option_index = 0; option_index < option_count; ++option_index) { + target.options.emplace(keys[option_index], values[option_index]); + } + } + const auto type = ort_api_.HardwareDevice_Type(hardware); + switch (type) { + case OrtHardwareDeviceType_CPU: + target.device_type = "CPU"; + break; + case OrtHardwareDeviceType_GPU: + target.device_type = "GPU"; + break; + case OrtHardwareDeviceType_NPU: + target.device_type = "NPU"; + break; + default: + break; + } + +#if defined(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) + uint64_t total = 0; + if (auto* status = ort_api_.EpDevice_GetMemoryCapacity(device, OrtDeviceMemoryType_DEFAULT, &total, nullptr)) { + logger_.Log(LogLevel::Debug, std::string("Total memory capacity unavailable for ") + provider); + ort_api_.ReleaseStatus(status); + } else if (type == OrtHardwareDeviceType_GPU) { + target.capacity.total_vram_bytes = total; + } +#endif + targets.push_back(std::move(target)); + } + + return targets; +} + const std::vector& EpDetector::GetDiscoverableEps() const { // Take the cache lock for strict correctness of the is_registered field reads. // Vector size and element addresses are immutable after construction; only diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index 7232e71d0..cbce3bcc4 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -16,6 +16,7 @@ #include "ep_detection/ep_types.h" #include "ep_detection/ep_bootstrapper.h" +#include "ep_detection/execution_target.h" // Forward declarations — avoid pulling onnxruntime_c_api.h into this header. struct OrtApi; @@ -35,6 +36,10 @@ class IEpDetector { /// Example: { "CPU" → ["CPUExecutionProvider"], "GPU" → ["CudaExecutionProvider"] } virtual std::map> GetAvailableDevicesToEPs() const = 0; + virtual std::map> GetDeviceCapacities() const { return {}; } + + virtual std::vector GetExecutionTargets() const; + /// Returns metadata for all discoverable EPs (those with bootstrappers). /// Default: empty — no discoverable EPs beyond what's already registered. virtual const std::vector& GetDiscoverableEps() const { @@ -86,6 +91,8 @@ class EpDetector : public IEpDetector { EpDetector& operator=(const EpDetector&) = delete; std::map> GetAvailableDevicesToEPs() const override; + std::map> GetDeviceCapacities() const override; + std::vector GetExecutionTargets() const override; const std::vector& GetDiscoverableEps() const override; std::span GetDiscoverableEpsCApi() const override; EpDownloadResult DownloadAndRegisterEps(const std::vector* names, diff --git a/sdk_v2/cpp/src/ep_detection/execution_target.h b/sdk_v2/cpp/src/ep_detection/execution_target.h new file mode 100644 index 000000000..69131322c --- /dev/null +++ b/sdk_v2/cpp/src/ep_detection/execution_target.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include +#include + +namespace fl { + +struct DeviceCapacity { + std::optional total_vram_bytes; + bool operator==(const DeviceCapacity&) const = default; +}; + +struct ExecutionTarget { + std::string provider; + uint32_t device_id = 0; + std::string device_type; + DeviceCapacity capacity; + std::map options = {}; + bool operator==(const ExecutionTarget&) const = default; +}; + +} // namespace fl \ No newline at end of file diff --git a/sdk_v2/cpp/src/hosting/manager_dispatcher.cc b/sdk_v2/cpp/src/hosting/manager_dispatcher.cc index 1ff6b8cd2..c726f6405 100644 --- a/sdk_v2/cpp/src/hosting/manager_dispatcher.cc +++ b/sdk_v2/cpp/src/hosting/manager_dispatcher.cc @@ -4,6 +4,7 @@ #include "ep_detection/ep_types.h" #include "exception.h" +#include "hosting/sdk_values.h" namespace fl::hosting { namespace { @@ -58,6 +59,10 @@ void ManagerDispatcher::Dispatch(const std::shared_ptr owner) { switch (call.Op()) { case Operation::RuntimeDevices: + if (call.Payload().value("include_execution_targets", false)) { + return EncodeRuntimeDeviceSnapshot(manager_->GetRuntimeDeviceSnapshot()); + } + return manager_->GetRuntimeDevices(); case Operation::WebServiceStart: { if (!owner) { diff --git a/sdk_v2/cpp/src/hosting/sdk_values.cc b/sdk_v2/cpp/src/hosting/sdk_values.cc index 10574ba8c..ab02d058d 100644 --- a/sdk_v2/cpp/src/hosting/sdk_values.cc +++ b/sdk_v2/cpp/src/hosting/sdk_values.cc @@ -82,6 +82,56 @@ void RequireArray(const Json& value) { } // namespace +Json EncodeRuntimeDeviceSnapshot(const api::RuntimeDeviceSnapshot& snapshot) { + Json targets = Json::array(); + for (const auto& target : snapshot.execution_targets) { + targets.push_back({{"provider", target.provider}, + {"device_id", target.device_id}, + {"device_type", target.device_type}, + {"options", target.options}, + {"capacity", {{"total_vram_bytes", target.capacity.total_vram_bytes + ? Json(*target.capacity.total_vram_bytes) + : Json(nullptr)}}}}); + } + + return {{"devices", snapshot.devices}, {"execution_targets", std::move(targets)}}; +} + +api::RuntimeDeviceSnapshot DecodeRuntimeDeviceSnapshot(const Json& value) { + return DecodeChecked([&]() { + api::RuntimeDeviceSnapshot snapshot; + if (!value.contains("devices")) { + snapshot.devices = value.get(); + return snapshot; + } + + snapshot.devices = value.at("devices").get(); + if (const auto targets = value.find("execution_targets"); targets != value.end()) { + RequireArray(*targets); + for (const auto& entry : *targets) { + ExecutionTarget target; + target.provider = entry.at("provider").get(); + target.device_id = Integer(entry.at("device_id")); + target.device_type = entry.at("device_type").get(); + if (const auto options = entry.find("options"); options != entry.end()) { + target.options = options->get>(); + } + const auto capacity = entry.find("capacity"); + if (capacity != entry.end()) { + const auto total = capacity->find("total_vram_bytes"); + if (total != capacity->end() && !total->is_null()) { + target.capacity.total_vram_bytes = Integer(*total); + } + } + + snapshot.execution_targets.push_back(std::move(target)); + } + } + + return snapshot; + }); +} + Json EncodeModelInfo(const ModelInfo& info) { return {{"id", info.model_id}, {"name", info.name}, diff --git a/sdk_v2/cpp/src/hosting/sdk_values.h b/sdk_v2/cpp/src/hosting/sdk_values.h index 6ad420feb..bc0ea0695 100644 --- a/sdk_v2/cpp/src/hosting/sdk_values.h +++ b/sdk_v2/cpp/src/hosting/sdk_values.h @@ -12,6 +12,7 @@ struct RequestBudget; enum class CatalogType; namespace api { class ModelBackend; +struct RuntimeDeviceSnapshot; } } // namespace fl @@ -23,6 +24,9 @@ struct ModelSelection { std::string path; }; +nlohmann::json EncodeRuntimeDeviceSnapshot(const api::RuntimeDeviceSnapshot& snapshot); +api::RuntimeDeviceSnapshot DecodeRuntimeDeviceSnapshot(const nlohmann::json& value); + nlohmann::json EncodeModelInfo(const ModelInfo& info); ModelInfo DecodeModelInfo(const nlohmann::json& value); nlohmann::json EncodeModelSelection(const ModelSelection& selection); diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index b53cbc096..a28042c2a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -240,7 +240,8 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, std::string effective_model_path, GenAIConfig genai_config, ExecutionProvider resolved_ep, - ILogger& logger) + ILogger& logger, + const std::optional& target) : model_id_(std::move(model_id)), model_path_(std::move(effective_model_path)), genai_config_(std::move(genai_config)), @@ -289,6 +290,20 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, model_id_, configured_provider.empty() ? "implicit CPU" : configured_provider)); } + if (target && !target->options.empty()) { + const auto provider = ep_ == ExecutionProvider::kDefault + ? genai_config_.DefaultProvider() + : std::string(EPUtils::EPtoGenAI(ep_)); + try { + for (const auto& [key, value] : target->options) { + oga_config->SetProviderOption(provider.c_str(), key.c_str(), value.c_str()); + } + } catch (const std::runtime_error& error) { + FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to configure selected execution target for model ", model_id_, ": ", error.what()); + } + } + // Create OGA Model try { oga_model_ = OgaModel::Create(*oga_config); diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 65df74b3d..30a481e1f 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #pragma once +#include "ep_detection/execution_target.h" #include "inferencing/execution_provider.h" #include "inferencing/generative/genai_config.h" @@ -115,7 +116,8 @@ class GenAIModelInstance { std::string effective_model_path, GenAIConfig genai_config, ExecutionProvider resolved_ep, - ILogger& logger); + ILogger& logger, + const std::optional& target); std::string model_id_; std::string model_path_; diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index 9733f3078..997d349a5 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "inferencing/model_load_manager.h" +#include "catalog/capacity_compatibility.h" #include "inferencing/predictive/non_generative_runtime.h" #include "model.h" @@ -503,7 +504,8 @@ bool ModelLoadManager::UnloadModelArtifact( ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_path, std::string_view model_id, - ExecutionProvider ep_override) { + ExecutionProvider ep_override, + const ModelInfo* model_info) { if (shutdown_.load()) { FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot load model during shutdown"); @@ -576,6 +578,35 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ : EPUtils::RequiredEpForConfigProvider(config_provider); } + std::optional selected_target; + if (model_info && model_info->device_type == DeviceType::kGPU && resolved_ep != ExecutionProvider::kCPU) { + ModelCapacityRequirements requirements; + if (const auto* bytes = model_info->GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); + bytes && *bytes >= 0) { + requirements.minimum_vram_bytes = static_cast(*bytes); + } + if (requirements.minimum_vram_bytes) { + const auto targets = ep_detector_.GetExecutionTargets(); + const bool generic_gpu = ep_override == ExecutionProvider::kDefault && + id_str.find("generic-gpu") != std::string::npos; + const auto result = EvaluateProviderCapacity(true, requirements, targets, required_ep, generic_gpu); + if (result.state == CompatibilityState::kDoesNotFit) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "no execution target has sufficient total GPU memory for model: ", id_str); + } + if (!result.eligible_targets.empty()) { + selected_target = result.eligible_targets.front(); + required_ep = EPUtils::NormalizeRegistrationName(selected_target->provider); + if (generic_gpu) { + resolved_ep = EPUtils::StringtoEP(required_ep); + } + logger_.Log(LogLevel::Information, + fmt::format("model {} selected target {} hardware_id={} options={}", id_str, required_ep, + selected_target->device_id, nlohmann::json(selected_target->options).dump())); + } + } + } + // OGA can crash or hang if a model is loaded with an unregistered EP. PrepareProviderForModelLoad(required_ep, id_str); @@ -584,7 +615,8 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ path_str, std::move(genai_config), resolved_ep, - logger_)); + logger_, + selected_target)); auto* raw_ptr = loaded.get(); loaded_runtimes_[runtime_key] = diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.h b/sdk_v2/cpp/src/inferencing/model_load_manager.h index f09d86ded..f7a336e26 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.h +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.h @@ -27,6 +27,7 @@ namespace fl { class Model; +struct ModelInfo; class ModelLoadManager; class ManagedModelRuntime; class ModelLoadManagerTestAccess; @@ -101,7 +102,8 @@ class ModelLoadManager { /// @returns LoadResult with status and non-owning pointer to the loaded model. LoadResult LoadModel(std::string_view model_path, std::string_view model_id, - ExecutionProvider ep_override = ExecutionProvider::kDefault); + ExecutionProvider ep_override = ExecutionProvider::kDefault, + const ModelInfo* model_info = nullptr); /// Verify that a required provider is registered and prepare its runtime /// dependencies before constructing an ORT GenAI model or session. diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index c52a37811..71ef8191f 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -147,6 +147,7 @@ Model::Model(Model&& other) noexcept model_load_manager_(other.model_load_manager_), variants_(std::move(other.variants_)), retired_variants_(std::move(other.retired_variants_)), + catalog_visible_(other.catalog_visible_.load()), selected_variant_(other.selected_variant_.load(std::memory_order_relaxed)), selection_is_explicit_(other.selection_is_explicit_), has_eligible_default_variant_(other.has_eligible_default_variant_), @@ -171,6 +172,7 @@ Model& Model::operator=(Model&& other) noexcept { model_load_manager_ = other.model_load_manager_; variants_ = std::move(other.variants_); retired_variants_ = std::move(other.retired_variants_); + catalog_visible_.store(other.catalog_visible_.load()); selected_variant_.store(other.selected_variant_.load(std::memory_order_relaxed), std::memory_order_relaxed); selection_is_explicit_ = other.selection_is_explicit_; has_eligible_default_variant_ = other.has_eligible_default_variant_; @@ -477,13 +479,23 @@ const ModelInfo& Model::Info() const { return *info_; } -std::vector Model::Variants() const { +void Model::UpdateVariantVisibility(const std::function& is_visible) { + std::lock_guard lock(state_mutex_); + for (auto& variant : variants_) { + variant->catalog_visible_.store(!is_visible || is_visible(*variant)); + } +} + +std::vector Model::Variants(bool include_hidden) const { std::lock_guard lock(state_mutex_); std::vector result; if (IsContainer()) { result.reserve(variants_.size()); for (auto& v : variants_) { + if (!include_hidden && !v->catalog_visible_.load()) { + continue; + } // const_cast: the *set* of variants is fixed (this method is const), but each // variant is independently mutable. See header. result.push_back(const_cast(v.get())); @@ -773,7 +785,7 @@ void Model::Load(ExecutionProvider ep, std::optional expected_ *this, [&] { lifecycle_lock.unlock(); }); return; } - auto result = model_load_manager_->LoadModel(path, Info().model_id, ep); + auto result = model_load_manager_->LoadModel(path, info.model_id, ep, &info); if (result.status == ModelLoadManager::LoadStatus::kModelNotFound) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model not found at path: " + path); diff --git a/sdk_v2/cpp/src/model.h b/sdk_v2/cpp/src/model.h index 04103cf88..d18e8bb8f 100644 --- a/sdk_v2/cpp/src/model.h +++ b/sdk_v2/cpp/src/model.h @@ -104,6 +104,7 @@ class Model { bool preserve_explicit_selection = false) const; bool HasEligibleDefaultVariant() const; void SetDefaultVariantRefresh(std::function refresh); + void UpdateVariantVisibility(const std::function& is_visible); /// Hosted loads pin the selected artifact without loading weights into the metadata manager. /// Matching ID/path handles share the pin so any of them can release it. bool PreserveLoadedSelection(bool preserve); @@ -124,7 +125,8 @@ class Model { const ModelInfo& Info() const; /// Returns the variants within this model. For a leaf, returns {this}. - /// For a container, returns pointers to all contained variants. + /// For a container, returns catalog-visible variants. Internal inventory and reconciliation callers + /// may include hidden leaves to preserve cached/loaded access and stable handles across refreshes. /// /// Returned by value because the underlying variants_ vector could change during catalog refresh. /// @@ -132,7 +134,7 @@ class Model { /// SelectVariant). The returned variants are themselves mutable (Download, Load, etc.) — /// the const_cast in the implementation is intentional and mirrors the /// `std::unique_ptr::get() const → T*` idiom. - std::vector Variants() const; + std::vector Variants(bool include_hidden = false) const; // --- Query methods --- @@ -243,6 +245,7 @@ class Model { // stable across vector growth/reordering. std::vector> variants_; std::vector> retired_variants_; + std::atomic catalog_visible_{true}; mutable std::atomic selected_variant_{nullptr}; // non-null = this is a container mutable bool selection_is_explicit_ = false; // guarded by state_mutex_ mutable bool has_eligible_default_variant_ = true; // guarded by state_mutex_ diff --git a/sdk_v2/cpp/test/CMakeLists.txt b/sdk_v2/cpp/test/CMakeLists.txt index 012dbe138..b4244e063 100644 --- a/sdk_v2/cpp/test/CMakeLists.txt +++ b/sdk_v2/cpp/test/CMakeLists.txt @@ -96,6 +96,7 @@ add_executable(foundry_local_tests internal_api/c_api_test.cc internal_api/callback_handler_test.cc internal_api/catalog_cache_test.cc + internal_api/capacity_compatibility_test.cc internal_api/chat/chat_generator_test.cc internal_api/chat/reasoning_stream_splitter_test.cc internal_api/chat/chat_session_test.cc diff --git a/sdk_v2/cpp/test/internal_api/api_backend_test.cc b/sdk_v2/cpp/test/internal_api/api_backend_test.cc index b693df314..630a2fa5f 100644 --- a/sdk_v2/cpp/test/internal_api/api_backend_test.cc +++ b/sdk_v2/cpp/test/internal_api/api_backend_test.cc @@ -159,7 +159,7 @@ class RecordingManager final : public fl::api::ManagerBackend { void StopWebService() override {} std::span GetDiscoverableEps() const override { return {}; } fl::api::DeviceProviders GetRuntimeDevices() const override { return {}; } - void SetCatalogRuntimeDevices(fl::api::DeviceProviders) override {} + void SetCatalogRuntimeDevices(fl::api::DeviceProviders, std::vector) override {} fl::EpDownloadResult DownloadAndRegisterEps(const std::vector*, const fl::api::EpProgressCallback&) override { return {}; diff --git a/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc b/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc index 52d354b8a..364651931 100644 --- a/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc @@ -89,6 +89,32 @@ class AllDevicesEpDetector final : public IEpDetector { } // namespace +TEST(AzureCatalogClientTest, VramRequirementPreservesOnlyExplicitNonnegativeIntegerBytes) { + for (const nlohmann::json& footprint : {nlohmann::json(nullptr), nlohmann::json(-1), nlohmann::json("16"), + nlohmann::json(1.5), nlohmann::json(UINT64_MAX), nlohmann::json(0), + nlohmann::json(16)}) { + SCOPED_TRACE(footprint.dump()); + auto response = nlohmann::json::parse(MakeSummaryResponse({{"model", 1}})); + response["summaries"][0]["variantInformation"]["variantMetadata"]["vRamFootprintBytes"] = footprint; + CpuOnlyEpDetector detector; + StderrLogger logger; + AzureCatalogClient client("https://catalog.test", "", detector, logger, + [&](const std::string&, const std::string&) { return MakeOkResponse(response.dump()); }); + const auto infos = client.FetchAllModelInfos(); + ASSERT_EQ(infos.size(), 1u); + const auto* bytes = infos.front().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); + if (footprint == nlohmann::json(0) || footprint == nlohmann::json(16)) { + ASSERT_NE(bytes, nullptr); + EXPECT_EQ(*bytes, footprint.get()); + const auto restored = ModelInfoFromJson(ModelInfoToJson(infos.front())); + ASSERT_NE(restored.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT), nullptr); + EXPECT_EQ(*restored.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT), *bytes); + } else { + EXPECT_EQ(bytes, nullptr); + } + } +} + TEST(AzureCatalogClientTest, RequestUsesAssetGalleryContract) { CpuOnlyEpDetector ep; StderrLogger logger; diff --git a/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc index bd7eb0bc7..b2fabfa17 100644 --- a/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "catalog/azure_model_catalog.h" +#include "catalog/azure_catalog_client.h" #include "catalog/catalog_cache.h" #include "catalog/catalog_client.h" #include "internal_api/test_helpers.h" @@ -14,12 +15,15 @@ #include #include #include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -144,6 +148,12 @@ class CountingCatalogEpDetector final : public IEpDetector { } mutable int calls = 0; + mutable int capacity_calls = 0; + std::map> GetDeviceCapacities() const override { + ++capacity_calls; + return capacities; + } + std::map> capacities; std::map> devices{{"CPU", {"CPUExecutionProvider"}}}; }; @@ -272,6 +282,143 @@ class AzureModelCatalogTest : public ::testing::Test { int model_handle_factory_calls_ = 0; }; +TEST(AzureModelCatalogCapacityEndToEndTest, RealCudaCapacityFiltersParsedCatalogAndOfflineSnapshot) { +#if !defined(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) + GTEST_SKIP() << "Requires ORT with EpDevice_GetMemoryCapacity (PR #33447)"; +#else + const char* provider_library = std::getenv("FOUNDRY_LOCAL_TEST_CUDA_EP_LIBRARY"); + if (!provider_library || !*provider_library) { + GTEST_SKIP() << "Set FOUNDRY_LOCAL_TEST_CUDA_EP_LIBRARY to the matching CUDA provider library"; + } + const auto* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ASSERT_NE(api, nullptr); + const auto check_status = [api](OrtStatus* status) { + if (!status) { + return ::testing::AssertionSuccess(); + } + const std::string message = api->GetErrorMessage(status); + api->ReleaseStatus(status); + return ::testing::AssertionFailure() << message; + }; + OrtEnv* env = nullptr; + ASSERT_TRUE(check_status(api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "catalog-capacity-e2e", &env))); + ScopeGuard release_env([&]() noexcept { api->ReleaseEnv(env); }); + const auto library_path = fs::path(provider_library); + ASSERT_TRUE(check_status(api->RegisterExecutionProviderLibrary(env, "catalog-capacity-cuda", library_path.c_str()))); + ScopeGuard unregister_provider([&]() noexcept { + if (auto* status = api->UnregisterExecutionProviderLibrary(env, "catalog-capacity-cuda")) { + api->ReleaseStatus(status); + } + }); + + fl::test::FakeServiceBindings services; + EpDetector detector(*api, *env, {}, services.logger, services.telemetry); + const auto capacities = detector.GetDeviceCapacities(); + const auto cuda = capacities.find("CUDAExecutionProvider"); + ASSERT_NE(cuda, capacities.end()) << "Configured CUDA provider must expose at least one visible GPU"; + ASSERT_FALSE(cuda->second.empty()); + const OrtEpDevice* const* ort_devices = nullptr; + size_t device_count = 0; + ASSERT_TRUE(check_status(api->GetEpDevices(env, &ort_devices, &device_count))); + uint64_t largest_total = 0; + size_t cuda_index = 0; + for (size_t index = 0; index < device_count; ++index) { + const auto* device = ort_devices[index]; + const char* provider = api->EpDevice_EpName(device); + if (!provider || std::string_view(provider) != "CUDAExecutionProvider" || + api->HardwareDevice_Type(api->EpDevice_Device(device)) != OrtHardwareDeviceType_GPU) { + continue; + } + uint64_t total = 0; + ASSERT_TRUE(check_status(api->EpDevice_GetMemoryCapacity(device, OrtDeviceMemoryType_DEFAULT, &total, nullptr))); + ASSERT_LT(cuda_index, cuda->second.size()); + const auto& capacity = cuda->second[cuda_index++]; + ASSERT_TRUE(capacity.total_vram_bytes.has_value()) << "Configured CUDA provider must support total capacity"; + EXPECT_EQ(*capacity.total_vram_bytes, total); + largest_total = std::max(largest_total, total); + } + ASSERT_EQ(cuda_index, cuda->second.size()); + ASSERT_GT(largest_total, 0u); + ASSERT_LT(largest_total, static_cast(std::numeric_limits::max())); + const auto oversized = static_cast(largest_total + 1); + + nlohmann::json summaries = nlohmann::json::array(); + const auto add_variant = [&](const std::string& name, const std::string& alias, const std::string& device, + const std::string& provider, std::optional required) { + nlohmann::json metadata = {{"device", device}, {"executionProvider", provider}, {"modelType", "ONNX"}}; + if (required) { + metadata["vRamFootprintBytes"] = *required; + } + summaries.push_back({ + {"assetId", "azureml://registries/azureml/models/" + name + "/versions/1"}, + {"name", name}, + {"alias", alias}, + {"version", "1"}, + {"foundryLocal", ""}, + {"deploymentOptions", {"Foundry Local on Devices"}}, + {"variantInformation", {{"variantMetadata", std::move(metadata)}}}, + }); + }; + add_variant("exact-cuda-gpu", "exact", "gpu", "CUDAExecutionProvider", static_cast(largest_total)); + add_variant("mixed-cuda-gpu", "mixed", "gpu", "CUDAExecutionProvider", oversized); + add_variant("mixed-generic-cpu", "mixed", "cpu", "CPUExecutionProvider", oversized); + add_variant("hidden-cuda-gpu", "hidden", "gpu", "CUDAExecutionProvider", oversized); + add_variant("unknown-cuda-gpu", "unknown", "gpu", "CUDAExecutionProvider", std::nullopt); + add_variant("zero-cuda-gpu", "zero", "gpu", "CUDAExecutionProvider", 0); + const auto body = nlohmann::json({{"summaries", summaries}}).dump(); + int requests = 0; + const auto client_factory = [&](const std::string& url, const std::string& filter) { + return std::make_unique(url, filter, detector, services.logger, + [&](const std::string&, const std::string&) { + ++requests; + http::HttpResponse response; + response.status = 200; + response.body = body; + return response; + }); + }; + const auto model_factory = [&](ModelInfo info, std::string local_path) { + return Model::FromModelInfo(std::move(info), std::move(local_path), services.download_manager, + services.model_load_manager); + }; + const auto cache = fl::test::TempPath::CreateTempDir("fl_capacity_e2e_"); + const std::vector>> urls{{"https://catalog.test", std::nullopt}}; + const auto verify = [&](AzureModelCatalog& catalog) { + const auto models = catalog.ListModels(); + ASSERT_EQ(models.size(), 4u); + EXPECT_EQ(FindVariant(models, "mixed-cuda-gpu:1"), nullptr); + EXPECT_EQ(FindVariant(models, "hidden-cuda-gpu:1"), nullptr); + EXPECT_EQ(catalog.GetModel("hidden"), nullptr); + auto* exact = catalog.GetModel("exact"); + ASSERT_NE(exact, nullptr); + EXPECT_EQ(exact->Id(), "exact-cuda-gpu:1"); + const auto* requirement = exact->Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); + ASSERT_NE(requirement, nullptr); + EXPECT_EQ(*requirement, static_cast(largest_total)); + auto* mixed = catalog.GetModel("mixed"); + ASSERT_NE(mixed, nullptr); + EXPECT_EQ(mixed->Id(), "mixed-generic-cpu:1"); + EXPECT_NE(catalog.GetModel("unknown"), nullptr); + EXPECT_NE(catalog.GetModel("zero"), nullptr); + }; + TestAzureModelCatalog live(urls, cache.string(), model_factory, detector, services.logger, false, + services.telemetry, client_factory); + ASSERT_NO_FATAL_FAILURE(verify(live)); + EXPECT_GT(requests, 0); + + CatalogCache snapshot(cache.string(), services.logger); + snapshot.Load(); + const auto persisted = snapshot.GetCachedModels(); + ASSERT_TRUE(persisted.has_value()); + EXPECT_EQ(persisted->size(), summaries.size()); + const int live_requests = requests; + TestAzureModelCatalog offline(urls, cache.string(), model_factory, detector, services.logger, true, + services.telemetry, client_factory); + ASSERT_NO_FATAL_FAILURE(verify(offline)); + EXPECT_EQ(requests, live_requests) << "Offline filtering must not fetch catalog metadata"; +#endif +} + TEST(AzureModelCatalogRuntimeTest, HostedDeviceChangesInvalidateMetadataWithoutChangingLocalProviderAvailability) { static std::atomic sequence{0}; const auto scratch = fs::path(FOUNDRY_LOCAL_TEST_SCRATCH_DIR); @@ -403,6 +550,261 @@ TEST_F(AzureModelCatalogTest, CachedModelLookupSharesRefreshCorrelation) { EXPECT_EQ(telemetry_.calls[1].correlation_id, telemetry_.calls[0].correlation_id); } +TEST_F(AzureModelCatalogTest, GpuCapacityFiltersDefaultWithoutHidingCpuFallback) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto cpu = MakeModelInfo("model-cpu:1", "model-cpu", 1, "model", "Azure"); + cpu.device_type = DeviceType::kCPU; + cpu.execution_provider = "CPUExecutionProvider"; + cpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto gpu_only = gpu; + gpu_only.model_id = "gpu-only:1"; + gpu_only.name = "gpu-only"; + gpu_only.alias = "gpu-only"; + auto behavior = AddBehavior(url); + behavior->all_models = {gpu, cpu, gpu_only}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + + auto* model = catalog->GetModel("model"); + ASSERT_NE(model, nullptr); + EXPECT_EQ(model->Id(), cpu.model_id); + EXPECT_EQ(catalog->GetModel("gpu-only"), nullptr); + EXPECT_EQ(FindVariant(catalog->ListModels(), gpu.model_id), nullptr); + EXPECT_EQ(catalog->ListModels().size(), 1u); + + detector.capacities["CUDAExecutionProvider"] = {{8}, {16}}; + catalog->InvalidateCache(); + ASSERT_NE(catalog->GetModel("gpu-only"), nullptr); + EXPECT_EQ(catalog->GetModel("gpu-only")->Id(), gpu_only.model_id); + + detector.capacities["CUDAExecutionProvider"] = {{8}, {std::nullopt}}; + catalog->InvalidateCache(); + EXPECT_NE(catalog->GetModel("gpu-only"), nullptr); + + detector.capacities.clear(); + catalog->InvalidateCache(); + EXPECT_NE(catalog->GetModel("gpu-only"), nullptr); + + detector.capacities["CUDAExecutionProvider"] = {{8}}; + catalog->InvalidateCache(); + EXPECT_EQ(catalog->GetModel("gpu-only"), nullptr); + EXPECT_EQ(catalog->ListModels().size(), 1u); + ASSERT_NE(catalog->GetModel("model"), nullptr); + EXPECT_EQ(catalog->GetModel("model")->Id(), cpu.model_id); +} + +TEST_F(AzureModelCatalogTest, InitialPopulationRetainsCapacityHiddenCachedVariants) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + AddBehavior(url)->all_models = {gpu}; + const auto local_path = AddLocalModel(gpu.model_id, "oversized-cuda"); + WriteSnapshot({gpu}); + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + + for (const bool cache_only : {false, true}) { + SCOPED_TRACE(cache_only); + auto catalog = CreateCatalog({{url, std::nullopt}}, cache_only, &detector); + EXPECT_TRUE(catalog->ListModels().empty()); + EXPECT_EQ(catalog->GetModel(gpu.alias), nullptr); + const auto cached = catalog->GetCachedModels(); + ASSERT_EQ(cached.size(), 1u); + EXPECT_EQ(cached.front()->Id(), gpu.model_id); + EXPECT_EQ(cached.front()->LocalPathSnapshot(), local_path.string()); + EXPECT_EQ(catalog->GetModelVariant(gpu.model_id), cached.front()); + } +} + +TEST_F(AzureModelCatalogTest, ExplicitFetchRetainsCapacityHiddenVariantWithoutChangingDefault) { + const std::string url = "https://catalog.test"; + auto cpu = MakeModelInfo("model-generic-cpu:1", "model-generic-cpu", 1, "model", "Azure"); + cpu.device_type = DeviceType::kCPU; + cpu.execution_provider = "CPUExecutionProvider"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto behavior = AddBehavior(url); + behavior->all_models = {cpu}; + behavior->models_by_id = {gpu}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + auto* model = catalog->GetModel(cpu.alias); + ASSERT_NE(model, nullptr); + + auto* fetched = catalog->GetModelVariant(gpu.model_id); + ASSERT_NE(fetched, nullptr); + EXPECT_EQ(fetched->Id(), gpu.model_id); + EXPECT_EQ(catalog->GetModelVariant(gpu.model_id), fetched); + EXPECT_EQ(behavior->fetch_by_id_calls, 1); + EXPECT_EQ(model->Id(), cpu.model_id); + EXPECT_EQ(model->Variants().size(), 1u); + EXPECT_EQ(FindVariant(catalog->ListModels(), gpu.model_id), nullptr); + + detector.capacities["CUDAExecutionProvider"] = {{16}}; + catalog->InvalidateCache(); + EXPECT_EQ(FindVariant(catalog->ListModels(), gpu.model_id), fetched); +} + +TEST_F(AzureModelCatalogTest, RefreshHidesIncompatibleVariantsWithoutInvalidatingHandles) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto cpu = MakeModelInfo("model-generic-cpu:1", "model-generic-cpu", 1, "model", "Azure"); + cpu.device_type = DeviceType::kCPU; + cpu.execution_provider = "CPUExecutionProvider"; + AddBehavior(url)->all_models = {gpu, cpu}; + AddLocalModel(gpu.model_id, "retained-cuda"); + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{16}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + auto* model = catalog->GetModel("model"); + ASSERT_NE(model, nullptr); + auto* retained_gpu = FindVariant(catalog->ListModels(), gpu.model_id); + ASSERT_NE(retained_gpu, nullptr); + ASSERT_EQ(model->Variants().size(), 2u); + + detector.capacities["CUDAExecutionProvider"] = {{8}}; + catalog->InvalidateCache(); + EXPECT_EQ(catalog->GetModel("model"), model); + EXPECT_EQ(model->Id(), cpu.model_id); + EXPECT_EQ(model->Variants().size(), 1u); + EXPECT_EQ(FindVariant(catalog->ListModels(), gpu.model_id), nullptr); + EXPECT_EQ(retained_gpu->Info().model_id, gpu.model_id); + EXPECT_EQ(catalog->GetModelVariant(gpu.model_id), retained_gpu); + EXPECT_NE(FindVariant(catalog->GetCachedModels(), gpu.model_id), nullptr); + + detector.capacities["CUDAExecutionProvider"] = {{16}}; + catalog->InvalidateCache(); + EXPECT_EQ(catalog->GetModel("model"), model); + EXPECT_EQ(model->Variants().size(), 2u); + EXPECT_EQ(FindVariant(catalog->ListModels(), gpu.model_id), retained_gpu); +} + +TEST_F(AzureModelCatalogTest, GpuCapacityReevaluatesFullSnapshotOffline) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + AddBehavior(url)->all_models = {gpu}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + auto live = CreateCatalog({{url, std::nullopt}}, false, &detector); + EXPECT_TRUE(live->ListModels().empty()); + + auto offline = CreateCatalog({{url, std::nullopt}}, true, &detector); + EXPECT_TRUE(offline->ListModels().empty()); + detector.capacities["CUDAExecutionProvider"] = {{16}}; + offline->InvalidateCache(); + auto* model = offline->GetModel("model"); + ASSERT_NE(model, nullptr); + EXPECT_EQ(model->Id(), gpu.model_id); + const auto* requirement = model->Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); + ASSERT_NE(requirement, nullptr); + EXPECT_EQ(*requirement, 16); +} + +TEST_F(AzureModelCatalogTest, GpuCapacityDoesNotUseOtherProviderOrFilterMissingRequirements) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDA"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto unknown = gpu; + unknown.model_id = "unknown:1"; + unknown.name = "unknown"; + unknown.alias = "unknown"; + unknown.int_properties.clear(); + auto npu = gpu; + npu.model_id = "npu:1"; + npu.name = "npu"; + npu.alias = "npu"; + npu.device_type = DeviceType::kNPU; + npu.execution_provider = "QNNExecutionProvider"; + AddBehavior(url)->all_models = {gpu, unknown, npu}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider", "WebGpuExecutionProvider"}; + detector.devices["NPU"] = {"QNNExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}, {8}}; + detector.capacities["WebGpuExecutionProvider"] = {{64}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + + EXPECT_EQ(catalog->GetModel("model"), nullptr); + EXPECT_NE(catalog->GetModel("unknown"), nullptr); + EXPECT_NE(catalog->GetModel("npu"), nullptr); +} + +TEST_F(AzureModelCatalogTest, GenericGpuCapacityChecksEachEligibleProvider) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-generic-gpu:1", "model-generic-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "DmlExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + AddBehavior(url)->all_models = {gpu}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider", "WebGpuExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + detector.capacities["WebGpuExecutionProvider"] = {{8}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + EXPECT_EQ(catalog->GetModel("model"), nullptr); + + detector.capacities["WebGpuExecutionProvider"] = {{16}}; + catalog->InvalidateCache(); + EXPECT_NE(catalog->GetModel("model"), nullptr); +} + +TEST_F(AzureModelCatalogTest, HostedCapacityUsesOnlyHostTargetsAndNamesOnlyHostRemainsUnknown) { + const std::string url = "https://catalog.test"; + auto gpu = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + auto behavior = AddBehavior(url); + behavior->all_models = {gpu}; + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{64}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + const std::map> names{{"GPU", {"CUDAExecutionProvider"}}}; + const std::vector small{{"CUDAExecutionProvider", 7, "GPU", {8}}}; + const std::vector large{{"CUDAExecutionProvider", 7, "GPU", {24}}}; + catalog->SetRuntimeDevices(names, small); + EXPECT_EQ(catalog->GetModel("model"), nullptr); + catalog->SetRuntimeDevices(names, large); + EXPECT_NE(catalog->GetModel("model"), nullptr); + const auto fetches = behavior->fetch_all_calls; + catalog->SetRuntimeDevices(names, large); + EXPECT_NE(catalog->GetModel("model"), nullptr); + EXPECT_EQ(behavior->fetch_all_calls, fetches); + + detector.capacities["CUDAExecutionProvider"] = {{8}}; + catalog->SetRuntimeDevices(names, large); + EXPECT_NE(catalog->GetModel("model"), nullptr); + catalog->SetRuntimeDevices(names, small); + EXPECT_EQ(catalog->GetModel("model"), nullptr); + catalog->SetRuntimeDevices(names); + EXPECT_NE(catalog->GetModel("model"), nullptr); + EXPECT_EQ(detector.calls, 0); + EXPECT_EQ(detector.capacity_calls, 0); +} + TEST_F(AzureModelCatalogTest, UnavailableCachedEpRemainsVisibleWithoutOverridingCompatibleDefault) { const std::string url = "https://catalog.test"; auto cuda = MakeModelInfo("model-cuda-gpu:1", "model-cuda-gpu", 1, "model", "Azure"); @@ -699,6 +1101,38 @@ TEST_F(AzureModelCatalogTest, ProviderRefreshPreservesLoadedArtifactUntilItIsUnl } } +TEST_F(AzureModelCatalogTest, VisibilitySharesDeviceSnapshotsAcrossAliasesPerPass) { + const std::string url = "https://catalog.test"; + auto behavior = AddBehavior(url); + for (const auto& alias : {"first", "second", "third"}) { + auto gpu = MakeModelInfo(std::string(alias) + "-cuda-gpu:1", alias, 1, alias, "Azure"); + gpu.device_type = DeviceType::kGPU; + gpu.execution_provider = "CUDAExecutionProvider"; + gpu.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + behavior->all_models.push_back(std::move(gpu)); + } + CountingCatalogEpDetector detector; + detector.devices["GPU"] = {"CUDAExecutionProvider"}; + detector.capacities["CUDAExecutionProvider"] = {{8}}; + auto catalog = CreateCatalog({{url, std::nullopt}}, false, &detector); + + EXPECT_TRUE(catalog->ListModels().empty()); + EXPECT_EQ(detector.calls, 2); + EXPECT_EQ(detector.capacity_calls, 2); + EXPECT_EQ(catalog->GetModel("first"), nullptr); + EXPECT_EQ(detector.calls, 2); + EXPECT_EQ(detector.capacity_calls, 2); + + detector.capacities["CUDAExecutionProvider"] = {{16}}; + catalog->InvalidateCache(); + EXPECT_NE(catalog->GetModel("first"), nullptr); + EXPECT_EQ(detector.calls, 4); + EXPECT_EQ(detector.capacity_calls, 4); + EXPECT_EQ(catalog->ListModels().size(), 3u); + EXPECT_EQ(detector.calls, 5); + EXPECT_EQ(detector.capacity_calls, 5); +} + TEST_F(AzureModelCatalogTest, EligibilityUsesOneProviderSnapshotAndLookupReusesTheResult) { auto cpu = MakeModelInfo("model-generic-cpu:1", "model-generic-cpu", 1, "model", "Azure"); cpu.execution_provider = "cpu"; diff --git a/sdk_v2/cpp/test/internal_api/capacity_compatibility_test.cc b/sdk_v2/cpp/test/internal_api/capacity_compatibility_test.cc new file mode 100644 index 000000000..35a7886e1 --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/capacity_compatibility_test.cc @@ -0,0 +1,54 @@ +#include "catalog/capacity_compatibility.h" + +#include + +#include + +using namespace fl; + +TEST(CapacityCompatibilityTest, TotalVramBoundaryAndUnknowns) { + const std::array small{{{8}}}, exact{{{16}}}, unknown{{{std::nullopt}}}, zero{{{0}}}; + EXPECT_FALSE(MeetsCapacityRequirements(true, {16}, small)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {16}, exact)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {16}, unknown)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {}, zero)); + EXPECT_FALSE(MeetsCapacityRequirements(true, {16}, zero)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {0}, zero)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {16}, {})); + EXPECT_TRUE(MeetsCapacityRequirements(false, {16}, small)); +} + +TEST(CapacityCompatibilityTest, RequiresOneFittingDeviceNotAggregatedMemory) { + const std::array small{{{8}, {8}}}, fitting{{{8}, {16}}}, unknown{{{8}, {std::nullopt}}}; + EXPECT_FALSE(MeetsCapacityRequirements(true, {16}, small)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {16}, fitting)); + EXPECT_TRUE(MeetsCapacityRequirements(true, {16}, unknown)); +} + +TEST(CapacityCompatibilityTest, DistinguishesUnknownAndRetainsFittingDeviceIdentity) { + const std::array targets{{ + {"CUDAExecutionProvider", 0, "GPU", {8}}, + {"CUDAExecutionProvider", 1, "GPU", {24}}, + {"CUDAExecutionProvider", 2, "GPU", {}}, + }}; + const auto result = EvaluateCapacity(true, {16}, targets); + EXPECT_EQ(result.state, CompatibilityState::kFits); + EXPECT_EQ(result.reason, CompatibilityReason::kSufficientCapacity); + ASSERT_EQ(result.eligible_targets.size(), 2u); + EXPECT_EQ(result.eligible_targets.front().device_id, 1u); + EXPECT_EQ(EvaluateCapacity(true, {32}, targets).state, CompatibilityState::kUnknown); + EXPECT_EQ(EvaluateCapacity(true, {32}, std::span(targets).first(2)).state, CompatibilityState::kDoesNotFit); + EXPECT_EQ(EvaluateCapacity(true, {}, targets).reason, CompatibilityReason::kMissingRequirement); +} + +TEST(CapacityCompatibilityTest, GenericGpuEvaluatesProvidersTogetherAndPrefersKnownFit) { + const std::array targets{{ + {"CUDAExecutionProvider", 0, "GPU", {8}}, + {"WebGpuExecutionProvider", 7, "GPU", {24}}, + }}; + const auto result = EvaluateProviderCapacity(true, {16}, targets, {}, true); + ASSERT_EQ(result.state, CompatibilityState::kFits); + ASSERT_EQ(result.eligible_targets.size(), 1u); + EXPECT_EQ(result.eligible_targets.front().provider, "WebGpuExecutionProvider"); + EXPECT_EQ(EvaluateProviderCapacity(true, {16}, targets, "cuda").state, CompatibilityState::kDoesNotFit); +} \ No newline at end of file diff --git a/sdk_v2/cpp/test/internal_api/ep_detector_test.cc b/sdk_v2/cpp/test/internal_api/ep_detector_test.cc index f4a617557..9652019b4 100644 --- a/sdk_v2/cpp/test/internal_api/ep_detector_test.cc +++ b/sdk_v2/cpp/test/internal_api/ep_detector_test.cc @@ -143,6 +143,156 @@ class EpDetectorTest : public ::testing::Test { // can't be called safely in integration tests (would download real binaries). // ======================================================================== +TEST_F(EpDetectorTest, ExecutionTargetsEnumerateAllDevicesWithHardwareIdentityAndGpuTotalOnly) { + struct FakeDevice { + OrtHardwareDeviceType type; + uint32_t id; + const char* provider; + bool supported; + }; + static const FakeDevice fake_devices[] = { + {OrtHardwareDeviceType_GPU, 7, "CUDAExecutionProvider", true}, + {OrtHardwareDeviceType_GPU, 2, "CUDAExecutionProvider", false}, + {OrtHardwareDeviceType_CPU, 11, "CPUExecutionProvider", true}, + {OrtHardwareDeviceType_NPU, 23, "QNNExecutionProvider", true}, + }; + static const OrtEpDevice* devices[] = { + reinterpret_cast(&fake_devices[0]), + reinterpret_cast(&fake_devices[1]), + reinterpret_cast(&fake_devices[2]), + reinterpret_cast(&fake_devices[3]), + }; + OrtApi api = *ort_api_; + api.GetEpDevices = [](const OrtEnv*, const OrtEpDevice* const** output, size_t* count) noexcept -> OrtStatus* { + *output = devices; + *count = 4; + return nullptr; + }; + api.EpDevice_Device = [](const OrtEpDevice* device) -> const OrtHardwareDevice* { + return reinterpret_cast(device); + }; + api.HardwareDevice_Type = [](const OrtHardwareDevice* device) { + return reinterpret_cast(device)->type; + }; + api.HardwareDevice_DeviceId = [](const OrtHardwareDevice* device) { + return reinterpret_cast(device)->id; + }; + api.EpDevice_EpName = [](const OrtEpDevice* device) { + return reinterpret_cast(device)->provider; + }; + api.EpDevice_EpOptions = [](const OrtEpDevice* device) -> const OrtKeyValuePairs* { + const auto* fake = reinterpret_cast(device); + return fake->type == OrtHardwareDeviceType_GPU ? reinterpret_cast(device) : nullptr; + }; + api.GetKeyValuePairs = [](const OrtKeyValuePairs* options, const char* const** keys, + const char* const** values, size_t* count) { + static const char* option_keys[] = {"device_id"}; + static const char* first_values[] = {"0"}; + static const char* second_values[] = {"1"}; + *keys = option_keys; + *values = reinterpret_cast(options)->id == 7 ? first_values : second_values; + *count = 1; + }; +#if defined(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) + static size_t capacity_queries; + capacity_queries = 0; + api.EpDevice_GetMemoryCapacity = [](const OrtEpDevice* device, OrtDeviceMemoryType memory_type, + uint64_t* total, uint64_t* available) noexcept -> OrtStatus* { + ++capacity_queries; + EXPECT_EQ(memory_type, OrtDeviceMemoryType_DEFAULT); + EXPECT_NE(total, nullptr); + EXPECT_EQ(available, nullptr); + const auto* fake = reinterpret_cast(device); + if (!fake->supported) { + return OrtGetApiBase()->GetApi(ORT_API_VERSION)->CreateStatus(ORT_NOT_IMPLEMENTED, "unsupported"); + } + *total = 16; + return nullptr; + }; +#endif + EpDetector detector(api, *ort_env_, {}, logger_, fl::test::TestTelemetrySink()); + const auto targets = detector.GetExecutionTargets(); + ASSERT_EQ(targets.size(), 4u); + EXPECT_EQ(targets[0].provider, "CUDAExecutionProvider"); + EXPECT_EQ(targets[0].device_id, 7u); + EXPECT_EQ(targets[0].options.at("device_id"), "0"); + EXPECT_EQ(targets[0].device_type, "GPU"); + EXPECT_EQ(targets[1].provider, "CUDAExecutionProvider"); + EXPECT_EQ(targets[1].device_id, 2u); + EXPECT_EQ(targets[1].options.at("device_id"), "1"); + EXPECT_EQ(targets[1].device_type, "GPU"); + EXPECT_EQ(targets[2].provider, "CPUExecutionProvider"); + EXPECT_EQ(targets[2].device_id, 11u); + EXPECT_EQ(targets[2].device_type, "CPU"); + EXPECT_EQ(targets[3].provider, "QNNExecutionProvider"); + EXPECT_EQ(targets[3].device_id, 23u); + EXPECT_EQ(targets[3].device_type, "NPU"); + EXPECT_FALSE(targets[1].capacity.total_vram_bytes.has_value()); + EXPECT_FALSE(targets[2].capacity.total_vram_bytes.has_value()); + EXPECT_FALSE(targets[3].capacity.total_vram_bytes.has_value()); +#if defined(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY) + EXPECT_EQ(capacity_queries, 4u); + EXPECT_EQ(targets[0].capacity.total_vram_bytes, 16u); +#else + EXPECT_FALSE(targets[0].capacity.total_vram_bytes.has_value()); +#endif + + const auto capacities = detector.GetDeviceCapacities(); + ASSERT_EQ(capacities.size(), 1u); + const auto& cuda = capacities.at("CUDAExecutionProvider"); + ASSERT_EQ(cuda.size(), 2u); + EXPECT_EQ(cuda[0], targets[0].capacity); + EXPECT_EQ(cuda[1], targets[1].capacity); +} + +TEST_F(EpDetectorTest, ExecutionTargetsReturnEmptyOnEnumerationFailure) { + OrtApi api = *ort_api_; + api.GetEpDevices = [](const OrtEnv*, const OrtEpDevice* const**, size_t*) noexcept -> OrtStatus* { + return OrtGetApiBase()->GetApi(ORT_API_VERSION)->CreateStatus(ORT_FAIL, "enumeration failed"); + }; + EpDetector detector(api, *ort_env_, {}, logger_, fl::test::TestTelemetrySink()); + EXPECT_TRUE(detector.GetExecutionTargets().empty()); + EXPECT_TRUE(detector.GetDeviceCapacities().empty()); +} + +TEST(EpDetectorLegacyAdapterTest, ExecutionTargetsAdaptCapacitiesWithIndexedIdsAndProviderTypes) { + class LegacyDetector : public IEpDetector { + public: + std::map> GetAvailableDevicesToEPs() const override { + return {{"CPU", {"CPUExecutionProvider"}}, + {"GPU", {"CUDAExecutionProvider", "DmlExecutionProvider"}}, + {"NPU", {"QNNExecutionProvider"}}}; + } + std::map> GetDeviceCapacities() const override { + return {{"CUDAExecutionProvider", {{16}, {}}}, {"QNNExecutionProvider", {}}}; + } + } detector; + + const std::vector expected = { + {"CUDAExecutionProvider", 0, "GPU", {16}}, + {"CUDAExecutionProvider", 1, "GPU", {}}, + {"CPUExecutionProvider", 0, "CPU", {}}, + {"DmlExecutionProvider", 0, "GPU", {}}, + {"QNNExecutionProvider", 0, "NPU", {}}, + }; + EXPECT_EQ(detector.GetExecutionTargets(), expected); +} + +TEST(EpDetectorLegacyAdapterTest, ExecutionTargetsAdaptProvidersWithoutCapacitySupport) { + class LegacyDetector : public IEpDetector { + public: + std::map> GetAvailableDevicesToEPs() const override { + return {{"CPU", {"CPUExecutionProvider"}}, {"GPU", {"CUDAExecutionProvider"}}}; + } + } detector; + + const std::vector expected = { + {"CPUExecutionProvider", 0, "CPU", {}}, + {"CUDAExecutionProvider", 0, "GPU", {}}, + }; + EXPECT_EQ(detector.GetExecutionTargets(), expected); +} + TEST_F(EpDetectorTest, GetAvailableDevices_AlwaysIncludesCpu) { std::vector mocks; auto detector = MakeDetector(mocks, {{"CUDAExecutionProvider", true}}); diff --git a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc index 2384099bc..4a798398e 100644 --- a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc @@ -9,6 +9,7 @@ #include "internal_api/test_helpers.h" #include "internal_api/test_model_cache.h" #include "logger.h" +#include "model_info.h" #include "util/path_safety.h" #include "utils/temp_path.h" @@ -341,6 +342,67 @@ TEST(ModelLoadManagerTest, GenericGpuAutoSelectionStillPrefersCudaThenWebGpuWith } } +TEST(ModelLoadManagerTest, CapacityAwareGenericGpuLoadPreparesFittingProvider) { + class TargetDetector final : public fl::IEpDetector { + public: + std::map> GetAvailableDevicesToEPs() const override { + return {{"GPU", {"CUDAExecutionProvider", "WebGpuExecutionProvider"}}}; + } + std::vector GetExecutionTargets() const override { + return {{"CUDAExecutionProvider", 99, "GPU", {8}}, + {"WebGpuExecutionProvider", 77, "GPU", {24}}}; + } + bool PrepareForModelLoad(std::string_view provider) override { + prepared = provider; + return false; + } + std::string prepared; + } detector; + fl::ModelInfo info; + info.model_id = "model-generic-gpu:1"; + info.device_type = fl::DeviceType::kGPU; + info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + TempModelDir directory("capacity-generic", "dml"); + fl::StderrLogger logger; + fl::ModelLoadManager manager(detector, logger); + EXPECT_THROW(manager.LoadModel(directory.path(), info.model_id, fl::ExecutionProvider::kDefault, &info), + fl::Exception); + EXPECT_EQ(detector.prepared, "WebGpuExecutionProvider"); + detector.prepared.clear(); + EXPECT_THROW(manager.LoadModel(directory.path(), info.model_id, fl::ExecutionProvider::kCUDA, &info), + fl::Exception); + EXPECT_TRUE(detector.prepared.empty()); +} + +TEST(ModelLoadManagerTest, CapacityAwareLoadSelectsSecondCudaOrdinalNotHardwareId) { + class TargetDetector final : public fl::IEpDetector { + public: + std::map> GetAvailableDevicesToEPs() const override { + return {{"GPU", {"CUDAExecutionProvider"}}}; + } + std::vector GetExecutionTargets() const override { + return {{"CUDAExecutionProvider", 99, "GPU", {8}, {{"device_id", "0"}}}, + {"CUDAExecutionProvider", 99, "GPU", {24}, {{"device_id", "1"}}}}; + } + bool PrepareForModelLoad(std::string_view) override { return false; } + } detector; + class RecordingLogger final : public fl::ILogger { + public: + void Log(fl::LogLevel, std::string_view message) override { messages.append(message).append("\n"); } + std::string messages; + } logger; + fl::ModelInfo info; + info.model_id = "model-cuda-gpu:1"; + info.device_type = fl::DeviceType::kGPU; + info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = 16; + TempModelDir directory("capacity-second-gpu", "cuda"); + fl::ModelLoadManager manager(detector, logger); + EXPECT_THROW(manager.LoadModel(directory.path(), info.model_id, fl::ExecutionProvider::kDefault, &info), + fl::Exception); + EXPECT_NE(logger.messages.find(R"(hardware_id=99 options={"device_id":"1"})"), std::string::npos); + EXPECT_EQ(logger.messages.find(R"(options={"device_id":"0"})"), std::string::npos); +} + TEST(ModelLoadManagerTest, LoadRuntimeIdWithFullDmlProviderName_DoesNotRequireDownloadableEp) { CpuOnlyDetector ep; fl::StderrLogger logger; diff --git a/sdk_v2/cpp/test/internal_api/sdk_session_test.cc b/sdk_v2/cpp/test/internal_api/sdk_session_test.cc index 29aafb062..10713095d 100644 --- a/sdk_v2/cpp/test/internal_api/sdk_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/sdk_session_test.cc @@ -56,9 +56,13 @@ void CollectLocal(flStreamingCallbackData data, std::string& text) { class ProviderBackend final : public api::ManagerBackend { public: - ProviderBackend() : ManagerBackend(api::BackendKind::Embedded) {} + explicit ProviderBackend(std::shared_ptr metadata = {}) + : ManagerBackend(api::BackendKind::Embedded), metadata_(std::move(metadata)) {} - api::CatalogBackend& GetCatalog(CatalogType) override { + api::CatalogBackend& GetCatalog(CatalogType type) override { + if (metadata_) { + return metadata_->GetCatalog(type); + } FL_THROW(FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED, "provider fixture has no catalog"); } std::unique_ptr CaptureModel( @@ -75,7 +79,10 @@ class ProviderBackend final : public api::ManagerBackend { api::DeviceProviders GetRuntimeDevices() const override { return {{"CPU", {"CPUExecutionProvider"}}, {"GPU", {"wire-ep"}}}; } - void SetCatalogRuntimeDevices(api::DeviceProviders) override {} + api::RuntimeDeviceSnapshot GetRuntimeDeviceSnapshot() const override { + return {GetRuntimeDevices(), {{"wire-ep", 7, "GPU", {total_vram_bytes}}}}; + } + void SetCatalogRuntimeDevices(api::DeviceProviders, std::vector) override {} EpDownloadResult DownloadAndRegisterEps(const std::vector* names, const api::EpProgressCallback& progress) override { if (!names || *names != std::vector{"wire-ep"}) { @@ -96,8 +103,10 @@ class ProviderBackend final : public api::ManagerBackend { bool IsShutdownRequested() const override { return false; } std::atomic reporting_download{false}; + uint64_t total_vram_bytes = 8; private: + std::shared_ptr metadata_; std::vector urls_; std::vector eps_{{FOUNDRY_LOCAL_DATA_VERSION, "wire-ep", false}}; }; @@ -856,6 +865,39 @@ TEST_F(SdkSessionTest, HostedRuntimeDeviceObservationDoesNotAcquireOrRegisterPro EXPECT_FALSE(hosted->IsEpDownloadInProgress()); } +TEST_F(SdkSessionTest, HostedRuntimeSnapshotUsesHostCapacityInsteadOfClientCapacity) { + auto provider = std::make_shared(); + manager_dispatcher_ = std::make_unique(provider, *lifetime_); + auto metadata = std::make_shared(manager_); + metadata->total_vram_bytes = 64; + auto context = ClientContext(); + int acquired = 0; + auto hosted = api::CreateHostedBackend(metadata, [&](bool acquire) { + acquired += acquire; + return context; + }); + const auto snapshot = hosted->GetRuntimeDeviceSnapshot(); + EXPECT_EQ(snapshot.devices, provider->GetRuntimeDevices()); + EXPECT_EQ(snapshot.execution_targets, provider->GetRuntimeDeviceSnapshot().execution_targets); + ASSERT_EQ(snapshot.execution_targets.size(), 1u); + EXPECT_EQ(snapshot.execution_targets.front().capacity.total_vram_bytes, 8u); + EXPECT_EQ(metadata->GetRuntimeDeviceSnapshot().execution_targets.front().capacity.total_vram_bytes, 64u); + EXPECT_EQ(acquired, 0); + + const auto legacy = context->Call(Operation::RuntimeDevices, Json::object()); + EXPECT_EQ(legacy.get(), provider->GetRuntimeDevices()); +} + +TEST_F(SdkSessionTest, HostedRuntimeSnapshotWithoutHostNeverUsesClientHardware) { + auto metadata = std::make_shared(manager_); + metadata->total_vram_bytes = 64; + auto hosted = api::CreateHostedBackend(metadata, [](bool) { return std::shared_ptr{}; }); + EXPECT_TRUE(hosted->GetRuntimeDevices().empty()); + const auto snapshot = hosted->GetRuntimeDeviceSnapshot(); + EXPECT_TRUE(snapshot.devices.empty()); + EXPECT_TRUE(snapshot.execution_targets.empty()); +} + TEST_F(SdkSessionTest, HostedSessionAndCapturedPreflightKeepTheClientDependencyGraphAlive) { auto owner = std::make_shared(0); const std::weak_ptr observed = owner; diff --git a/sdk_v2/cpp/test/internal_api/sdk_values_test.cc b/sdk_v2/cpp/test/internal_api/sdk_values_test.cc index 5dabb51cf..8cac19324 100644 --- a/sdk_v2/cpp/test/internal_api/sdk_values_test.cc +++ b/sdk_v2/cpp/test/internal_api/sdk_values_test.cc @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "api/backend.h" #include "exception.h" #include "hosting/sdk_values.h" #include "inferencing/generative/chat/chat_generator.h" @@ -18,6 +19,43 @@ using namespace fl::hosting; json ThroughCbor(const json& value) { return json::from_cbor(json::to_cbor(value)); } +TEST(SdkValuesTest, RuntimeDeviceSnapshotPreservesHostTargetIdentityAndOptionalCapacity) { + const fl::api::RuntimeDeviceSnapshot snapshot{ + {{"GPU", {"CUDAExecutionProvider"}}}, + {{"CUDAExecutionProvider", 7, "GPU", {uint64_t{24} << 30}, {{"device_id", "1"}}}, + {"CUDAExecutionProvider", 2, "GPU", {}}, + {"CPUExecutionProvider", 0, "CPU", {0}}}}; + const auto decoded = DecodeRuntimeDeviceSnapshot(ThroughCbor(EncodeRuntimeDeviceSnapshot(snapshot))); + EXPECT_EQ(decoded.devices, snapshot.devices); + EXPECT_EQ(decoded.execution_targets, snapshot.execution_targets); +} + +TEST(SdkValuesTest, RuntimeDeviceSnapshotFromNamesOnlyHostHasUnknownCapacity) { + const fl::api::DeviceProviders names{{"GPU", {"CUDAExecutionProvider"}}}; + const auto legacy = DecodeRuntimeDeviceSnapshot(ThroughCbor(json(names))); + EXPECT_EQ(legacy.devices, names); + EXPECT_TRUE(legacy.execution_targets.empty()); + const auto missing = DecodeRuntimeDeviceSnapshot({{"devices", names}}); + EXPECT_EQ(missing.devices, names); + EXPECT_TRUE(missing.execution_targets.empty()); +} + +TEST(SdkValuesTest, RuntimeDeviceSnapshotRejectsLossyIdentityAndCapacityIntegers) { + const fl::api::RuntimeDeviceSnapshot snapshot{ + {{"GPU", {"CUDAExecutionProvider"}}}, {{"CUDAExecutionProvider", 0, "GPU", {8}}}}; + for (const json& invalid : {json(-1), json(uint64_t{1} << 32), json(true)}) { + auto encoded = EncodeRuntimeDeviceSnapshot(snapshot); + encoded["execution_targets"][0]["device_id"] = invalid; + EXPECT_THROW(DecodeRuntimeDeviceSnapshot(encoded), fl::Exception); + } + + for (const json& invalid : {json(-1), json(1.5), json(true)}) { + auto encoded = EncodeRuntimeDeviceSnapshot(snapshot); + encoded["execution_targets"][0]["capacity"]["total_vram_bytes"] = invalid; + EXPECT_THROW(DecodeRuntimeDeviceSnapshot(encoded), fl::Exception); + } +} + TEST(SdkValuesTest, CapturesReplayAndPreparedToolContextWithoutChangingPayloadText) { fl::Request request; request.options.Add("max_output_tokens", "17"); diff --git a/sdk_v2/js/test/chat-client.test.ts b/sdk_v2/js/test/chat-client.test.ts index 1da01405c..192b1f82c 100644 --- a/sdk_v2/js/test/chat-client.test.ts +++ b/sdk_v2/js/test/chat-client.test.ts @@ -83,7 +83,7 @@ describe.skipIf(!haveTestModelCache)("ChatClient (real model, V1 OpenAI-JSON pas (error: unknown) => { expect(error).toMatchObject({ name: "AbortError" }); }, - ); + ); } finally { if (timer !== undefined) clearTimeout(timer); } diff --git a/sdk_v2/js/test/request-cancellation.test.ts b/sdk_v2/js/test/request-cancellation.test.ts index adb15a9dd..7265244ff 100644 --- a/sdk_v2/js/test/request-cancellation.test.ts +++ b/sdk_v2/js/test/request-cancellation.test.ts @@ -97,6 +97,20 @@ describe("non-streaming request cancellation", () => { test.reject(failure); await expect(pending).rejects.toBe(failure); }); + + it.each([false, true])("preserves successful completion when racing an abort (aborted=%s)", async (aborted) => { + vi.useFakeTimers(); + const test = fixture(); + const controller = new AbortController(); + const removeListener = vi.spyOn(controller.signal, "removeEventListener"); + const pending = test.session.processRequest(test.request, { signal: controller.signal }); + if (aborted) controller.abort(); + const response = {} as NativeResponse; + test.resolve(response); + await expect(pending).resolves.toBe(response); + expect(vi.getTimerCount()).toBe(0); + expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)); + }); }); describe("streaming request cancellation", () => {