Add GPU total-VRAM compatibility filtering to the native model catalog

.pipelines/templates/stages-build-native.yml

@@ -66,6 +66,7 @@ stages:
     - compute_version
     jobs:
     - job: build
+      timeoutInMinutes: 120
       pool:
         name: onnxruntime-medium-arm64-scus
         os: windows

.pipelines/templates/steps-build-windows.yml

@@ -104,6 +104,7 @@ steps:
     env:
       VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg
       VCPKG_DOWNLOADS: $(Build.BinariesDirectory)\vcpkg-downloads
+      VCPKG_MAX_CONCURRENCY: '1'
       PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat
 
 - ${{ if eq(parameters.arch, 'arm64') }}:
@@ -119,6 +120,7 @@ steps:
     env:
       VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg
       VCPKG_DOWNLOADS: $(Build.BinariesDirectory)\vcpkg-downloads
+      VCPKG_MAX_CONCURRENCY: '1'
       PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat
 
 - ${{ if eq(parameters.arch, 'x64') }}:

sdk_v2/cpp/README.md

@@ -115,6 +115,64 @@ required runtime directories when using it. It is not needed solely for the x64
 Foundry Local leaves runtime dependency loading to ORT and GenAI. On Linux, use the
 system loader configuration or `LD_LIBRARY_PATH`.
 
+## Catalog GPU Memory Compatibility
+
+Native catalog filtering interprets an explicit `variantMetadata.vRamFootprintBytes` as the variant's minimum
+total GPU memory requirement, in bytes. It is retained as the optional integer property `min_vram_bytes`.
+Only variants explicitly marked as GPU are checked. CPU and NPU compatibility rules are unchanged.
+Model file size is not used as a memory requirement. Missing, null, malformed, negative, or out-of-range
+requirements are unknown, not zero.
+
+A variant is excluded only when every candidate device for its execution provider has known total VRAM below
+the requirement. Multiple devices are not summed. A fitting device, an unknown capacity, an unsupported query,
+or a query failure leaves capacity compatibility undecided and does not exclude the variant. An alias remains
+visible when a compatible variant remains. Full metadata snapshots are saved before capacity filtering so
+offline catalogs can be reevaluated on different hardware. Previously returned handles remain stable across
+refreshes; memory-incompatible retained variants are hidden from discovery and cannot become automatic defaults.
+They remain accessible by ID and in cached/loaded inventory, and become visible again if capacity permits.
+
+Capacity queries use ORT's device-memory total only, never current available VRAM. This is not a guarantee
+that loading will succeed. CPU RAM and unified-memory checks are deliberately not implemented and require
+separate metadata and measurement contracts.
+
+The query requires the actual `EpDevice_GetMemoryCapacity` API and matching CUDA provider built with
+[ORT PR #33447](https://github.com/microsoft/onnxruntime/pull/33447). An API 32 version alone is insufficient:
+pre-merge 1.32 nightlies lack the member. CMake checks the member and warns if capacity filtering is inactive.
+The pinned 1.31 runtime and CUDA bundles predate the API; builds using them treat capacity as unknown.
+Updating default distribution requires both a post-merge runtime package and matching CUDA bundles.
+Until those are published, use a source-built ORT with matching headers in `<ort-install>/include` and runtime
+libraries in `<ort-install>/lib`, and require the API at configure time:
+
+```bash
+python sdk_v2/cpp/build.py --configure --build --ort_home <ort-install> \
+  --cmake_extra_defines FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY=ON
+```
+
+Register the matching CUDA provider before refreshing the catalog. Do not mix provider and runtime builds.
+Hosted device metadata currently carries only provider names, so hosted clients also treat capacity as unknown
+rather than using the client's local GPU.
+
+### Real-GPU catalog end-to-end test
+
+`AzureModelCatalogCapacityEndToEndTest.RealCudaCapacityFiltersParsedCatalogAndOfflineSnapshot` uses the real
+ORT environment, registered CUDA provider, EP detector, Azure catalog parser, selection logic, and disk snapshot.
+Only the catalog HTTP response is replaced with controlled metadata; no model downloads or inference are required.
+Requirements are derived from the detected devices, covering exact total-capacity fit, oversized GPU rejection,
+CPU fallback, aliases with no fitting variants, missing requirements, explicit zero, and offline snapshot reuse.
+
+Build with the capacity-enabled ORT described above and run from the repository root on Linux:
+
+```bash
+FOUNDRY_LOCAL_TEST_CUDA_EP_LIBRARY=/absolute/path/to/libonnxruntime_providers_cuda.so \
+  sdk_v2/cpp/build/Linux/Release/bin/foundry_local_tests \
+  --gtest_filter='AzureModelCatalogCapacityEndToEndTest.*'
+```
+
+The provider must match the ORT runtime used to build the tests. Configure CUDA and cuDNN library search paths
+as for normal CUDA usage. The test skips when built without the capacity API or when the provider path is unset.
+Once the provider path is set, registration failures, missing visible GPUs, and failed capacity queries fail
+the test. Set `CUDA_VISIBLE_DEVICES=7,2` (or another valid subset) to verify reordered device visibility.
+
 ## Clearing a Model Cache
 
 Use `Model::RemoveFromCache()` to clear a completed, interrupted, or corrupted download.

sdk_v2/cpp/cmake/FindOnnxRuntime.cmake

@@ -134,6 +134,34 @@ if(NOT EXISTS "${_ORT_HEADER_DIR}/onnxruntime_c_api.h")
     message(FATAL_ERROR "OnnxRuntime header not found at ${_ORT_HEADER_DIR}/onnxruntime_c_api.h")
 endif()
 
+option(FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY
+    "Require an ORT build supporting EP device total-memory queries" OFF)
+include(CheckCXXSourceCompiles)
+include(CMakePushCheckState)
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_INCLUDES "${_ORT_HEADER_DIR}")
+unset(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY CACHE)
+check_cxx_source_compiles("
+    #include <onnxruntime_c_api.h>
+    int main() {
+        using CapacityQuery = decltype(&OrtApi::EpDevice_GetMemoryCapacity);
+        return sizeof(CapacityQuery) == 0;
+    }" FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY)
+cmake_pop_check_state()
+if(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY)
+    add_compile_definitions(FOUNDRY_LOCAL_HAS_EP_DEVICE_MEMORY_CAPACITY=1)
+    message(STATUS "ORT EP device-memory capacity API enabled; provider support is checked at runtime")
+elseif(FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY)
+    message(FATAL_ERROR
+        "ORT lacks EpDevice_GetMemoryCapacity. Use --ort_home with a source build containing ORT PR #33447, "
+        "including matching runtime headers/libraries and CUDA provider. An API 32 version alone is insufficient.")
+else()
+    message(WARNING
+        "ORT lacks EpDevice_GetMemoryCapacity: catalog VRAM capacity filtering is inactive. "
+        "Use a post-PR #33447 ORT build via --ort_home; enable FOUNDRY_LOCAL_REQUIRE_DEVICE_MEMORY_CAPACITY "
+        "to reject this dependency at configure time.")
+endif()
+
 # Create imported target
 add_library(OnnxRuntime::OnnxRuntime SHARED IMPORTED)
 set_target_properties(OnnxRuntime::OnnxRuntime PROPERTIES

sdk_v2/cpp/include/foundry_local/foundry_local_c.h

@@ -280,6 +280,7 @@ typedef enum flTensorDataType {
 #define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT "supports_tool_calling"          ///< optional bool (not set or -1=unknown, 0=false, 1=true)
 #define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT "supports_reasoning"                ///< optional bool (not set or -1=unknown, 0=false, 1=true)
 #define FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT "filesize_mb"                              ///< optional int32_t
+#define FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT "min_vram_bytes"                        ///< optional int64_t
 #define FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT "max_output_tokens"                  ///< optional int32_t
 #define FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT "created_at_unix"                      ///< Unix timestamp. default=0
 #define FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT "is_test_model"                          ///< bool (0=false, 1=true)

sdk_v2/cpp/src/catalog/azure_catalog_models.cc

@@ -212,6 +212,14 @@ void from_json(const nlohmann::json& j, VariantMetadata& v) {
   opt_str(j, "device", v.device);
   opt_str(j, "executionProvider", v.execution_provider);
   opt_int64(j, "fileSizeBytes", v.file_size_bytes);
+  const auto footprint = j.find("vRamFootprintBytes");
+  if (footprint != j.end() && footprint->is_number_integer() &&
+      (!footprint->is_number_unsigned() || footprint->get<uint64_t>() <= uint64_t{INT64_MAX})) {
+    const auto bytes = footprint->get<int64_t>();
+    if (bytes >= 0) {
+      v.vram_footprint_bytes = bytes;
+    }
+  }
 }
 
 void from_json(const nlohmann::json& j, VariantParent& v) {
@@ -447,6 +455,9 @@ std::optional<ModelInfo> CatalogModelToModelInfo(const CatalogLocalModel& cm) {
     if (vm.file_size_bytes) {
       info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT] = *vm.file_size_bytes / (1024 * 1024);
     }
+    if (vm.vram_footprint_bytes) {
+      info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT] = *vm.vram_footprint_bytes;
+    }
   } else {
     info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX";
   }

sdk_v2/cpp/src/catalog/azure_catalog_models.h

@@ -56,6 +56,7 @@ struct VariantMetadata {
   std::optional<std::string> device;
   std::optional<std::string> execution_provider;
   std::optional<int64_t> file_size_bytes;
+  std::optional<int64_t> vram_footprint_bytes;
 };
 
 struct VariantParent {

sdk_v2/cpp/src/catalog/azure_model_catalog.cc

@@ -158,6 +158,11 @@ class AzureModelCatalog::CatalogDevices final : public IEpDetector {
     return hosted_ ? *hosted_ : local_.GetAvailableDevicesToEPs();
   }
 
+  std::map<std::string, std::vector<DeviceCapacity>> GetDeviceCapacities() const override {
+    std::lock_guard lock(mutex_);
+    return hosted_ ? std::map<std::string, std::vector<DeviceCapacity>>{} : local_.GetDeviceCapacities();
+  }
+
   bool Set(Devices devices) {
     std::lock_guard lock(mutex_);
     if (hosted_ && *hosted_ == devices) {
@@ -207,21 +212,57 @@ void AzureModelCatalog::SetRuntimeDevices(std::map<std::string, std::vector<std:
 }
 
 BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility() const {
+  return MakeCompatibilityPredicate(false);
+}
+
+BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantVisibility() const {
+  return [this, compatible = VariantEligibility{}](const Model& variant) mutable {
+    if (variant.Info().device_type != DeviceType::kGPU ||
+        !variant.Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT)) {
+      return true;
+    }
+    if (!compatible) {
+      compatible = MakeCompatibilityPredicate(true);
+    }
+    return compatible(variant);
+  };
+}
+
+BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeCompatibilityPredicate(bool capacity_only) const {
   std::unordered_set<std::string> providers;
   for (const auto& [device, eps] : catalog_devices_->GetAvailableDevicesToEPs()) {
     for (const auto& ep : eps) {
       providers.insert(ToLower(EPUtils::NormalizeRegistrationName(ep)));
     }
   }
 
+  std::unordered_map<std::string, std::vector<DeviceCapacity>> capacities;
+  for (auto& [provider, devices] : catalog_devices_->GetDeviceCapacities()) {
+    auto& target = capacities[ToLower(EPUtils::NormalizeRegistrationName(provider))];
+    target.insert(target.end(), devices.begin(), devices.end());
+  }
+
   // Cache config requirements only within this pass so artifact edits and provider changes are seen on refresh.
-  return [this, providers = std::move(providers),
+  return [this, capacity_only, providers = std::move(providers), capacities = std::move(capacities),
           configured_providers = std::unordered_map<std::string, std::optional<std::string>>{}](
              const Model& variant) mutable {
     const auto& info = variant.Info();
+    ModelCapacityRequirements requirements;
+    if (const auto* bytes = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT); bytes && *bytes >= 0) {
+      requirements.minimum_vram_bytes = static_cast<uint64_t>(*bytes);
+    }
+    if (capacity_only && (info.device_type != DeviceType::kGPU || !requirements.minimum_vram_bytes)) {
+      return true;
+    }
+    const auto meets_capacity = [&](const std::string& provider) {
+      const auto found = capacities.find(provider);
+      return MeetsCapacityRequirements(info.device_type == DeviceType::kGPU, requirements,
+                                       found == capacities.end() ? std::span<const DeviceCapacity>{} : found->second);
+    };
     if (info.model_id.find("generic-gpu") != std::string::npos &&
         (providers.contains("cudaexecutionprovider") || providers.contains("webgpuexecutionprovider"))) {
-      return true;
+      return (providers.contains("cudaexecutionprovider") && meets_capacity("cudaexecutionprovider")) ||
+             (providers.contains("webgpuexecutionprovider") && meets_capacity("webgpuexecutionprovider"));
     }
 
     auto provider = info.execution_provider;
@@ -240,7 +281,7 @@ BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility()
         }
 
         if (!entry->second) {
-          return false;
+          return capacity_only;
         }
 
         provider = *entry->second;
@@ -252,10 +293,31 @@ BaseModelCatalog::VariantEligibility AzureModelCatalog::MakeVariantEligibility()
     }
 
     provider = ToLower(EPUtils::NormalizeRegistrationName(provider));
-    return provider.empty() || provider == "cpuexecutionprovider" || providers.contains(provider);
+    return (capacity_only || provider.empty() || provider == "cpuexecutionprovider" || providers.contains(provider)) &&
+           meets_capacity(provider);
   };
 }
 
+void AzureModelCatalog::FilterCapacityIncompatibleModels(std::vector<Model>& models) const {
+  const bool has_requirements = std::any_of(models.begin(), models.end(), [](const Model& model) {
+    return model.Info().device_type == DeviceType::kGPU &&
+           model.Info().GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_MIN_VRAM_BYTES_INT) != nullptr;
+  });
+  if (!has_requirements) {
+    return;
+  }
+  auto compatible = MakeCompatibilityPredicate(true);
+  std::erase_if(models, [&](const Model& model) {
+    if (compatible(model)) {
+      return false;
+    }
+    logger_.Log(LogLevel::Information,
+                fmt::format("Catalog: filtering variant '{}' because its minimum total GPU VRAM exceeds capacity.",
+                            model.Info().model_id));
+    return true;
+  });
+}
+
 std::unique_ptr<ICatalogClient> AzureModelCatalog::CreateCatalogClient(const std::string& url,
                                                                        const std::string& filter) const {
   return MakeCatalogClient(url, filter, *catalog_devices_, logger_, cache_dir_);
@@ -384,6 +446,7 @@ std::vector<Model> AzureModelCatalog::FetchModelVersions(
     out.push_back(model_factory_(std::move(info), /*local_path=*/""));
   }
 
+  FilterCapacityIncompatibleModels(out);
   logger_.Log(LogLevel::Information,
               fmt::format("FetchModelVersions('{}') returned {} variant(s).",
                           model_alias, out.size()));

sdk_v2/cpp/src/catalog/azure_model_catalog.h

@@ -48,6 +48,7 @@ class AzureModelCatalog : public BaseModelCatalog {
   virtual std::unique_ptr<ICatalogClient> CreateCatalogClient(const std::string& url,
                                                               const std::string& filter) const;
   VariantEligibility MakeVariantEligibility() const override;
+  VariantEligibility MakeVariantVisibility() const override;
   void RefreshRetainedVariantCache(Model& model) const override;
 
  private:
@@ -66,6 +67,9 @@ class AzureModelCatalog : public BaseModelCatalog {
   static constexpr const char* kDefaultCatalogUrl = Configuration::kDefaultCatalogUrl;
   static constexpr const char* kDefaultCatalogFilter = Configuration::kDefaultCatalogFilter;
 
+  VariantEligibility MakeCompatibilityPredicate(bool capacity_only) const;
+  void FilterCapacityIncompatibleModels(std::vector<Model>& models) const;
+
   CatalogResult GetLiveCatalogOrLocalSnapshot(const std::vector<std::string>& cached_model_ids) const;
   std::vector<Model> CreateModelsWithLocalPaths(const std::vector<ModelInfo>& model_infos,
                                                 const LocalModels& local_models) const;