feat(podvm): opt-in NVIDIA 595.71.05 driver for B200 multi-GPU CC

.github/workflows/build-podvm-cohere.yaml

@@ -22,10 +22,20 @@ on:
         type: string
         default: "https://github.com/cohere-ai/guest-components.git"
       guest_components_ref:
-        description: "guest-components ref (default: cohere)"
+        description: |
+          guest-components ref (branch, tag, or SHA).
+
+          Default: alhassankhedr/sync-main-to-cohere (head of PR #9).
+          That branch carries upstream main's nvidia-attester rewrite
+          (NVAT SDK based, no `count == 1` guard) and is required for
+          multi-GPU evidence to work end-to-end on 8x B200 hosts. The
+          plain `cohere` branch still has the old NVML-based attester
+          which silently produces empty evidence on 2+ GPU systems
+          (mod a sed `s/count == 1/count >= 1/` patch the podvm-mkosi
+          Dockerfile applies). Switch back to `cohere` after PR #9 merges.
         required: false
         type: string
-        default: "cohere"
+        default: "alhassankhedr/sync-main-to-cohere"
       custom_gc_binaries:
         description: "guest-components binaries to build from source"
         required: false
@@ -46,6 +56,11 @@ on:
         required: false
         type: boolean
         default: false
+      b200_cc_drivers:
+        description: "Install NVIDIA 595.71.05 open driver (enables Confidential Computing on multi-GPU B200). EXPERIMENTAL"
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   id-token: write      # OIDC token for build provenance attestation
@@ -70,11 +85,13 @@ jobs:
       image_name_debug: ${{ steps.compute.outputs.image_name_debug }}
       image_tag_release: ${{ steps.compute.outputs.image_tag_release }}
       image_tag_debug: ${{ steps.compute.outputs.image_tag_debug }}
+      b200_cc_drivers: ${{ steps.compute.outputs.b200_cc_drivers }}
     steps:
       - name: Compute tags and image names
         id: compute
         env:
           DISTRO: ${{ inputs.distro || 'ubuntu' }}
+          B200_CC_DRIVERS: ${{ inputs.b200_cc_drivers && 'true' || 'false' }}
         run: |
           if [[ "$GITHUB_REF" == refs/tags/podvm-v* ]]; then
             TAG="${GITHUB_REF#refs/tags/podvm-}"
@@ -84,10 +101,15 @@ jobs:
             REPLACE_IMAGE="true"
           fi
           TAG="${TAG//./-}"
+          # Suffix CC-driver builds so they never collide with standard images
+          if [ "$B200_CC_DRIVERS" = "true" ]; then
+            TAG="${TAG}-cc595"
+          fi
           {
             echo "tag=$TAG"
             echo "distro=$DISTRO"
             echo "replace_image=$REPLACE_IMAGE"
+            echo "b200_cc_drivers=$B200_CC_DRIVERS"
             echo "image_name_release=podvm-${DISTRO}-${TEE_PLATFORM}-release-${TAG}"
             echo "image_name_debug=podvm-${DISTRO}-${TEE_PLATFORM}-debug-${TAG}"
             echo "image_tag_release=${TAG}-${DISTRO}-release"
@@ -176,7 +198,7 @@ jobs:
           PODVM_DISTRO: ${{ needs.meta.outputs.distro }}
           AA_FEATURES: ${{ inputs.aa_features || 'bin,ttrpc,kbs,coco_as,rust-crypto,tdx-attester,nvidia-attester' }}
           GC_REPO: ${{ inputs.guest_components_repo || 'https://github.com/cohere-ai/guest-components.git' }}
-          GC_REF: ${{ inputs.guest_components_ref || 'cohere' }}
+          GC_REF: ${{ inputs.guest_components_ref || 'alhassankhedr/sync-main-to-cohere' }}
           GC_CUSTOM_BINARIES: ${{ inputs.custom_gc_binaries || 'attestation-agent,api-server-rest' }}
         run: |
           MAKE_ARGS=(
@@ -195,6 +217,47 @@ jobs:
           echo "Disk after binaries build:"
           df -h /
 
+      - name: Override NVIDIA driver to 595.71.05 (B200 multi-GPU CC)
+        if: needs.meta.outputs.b200_cc_drivers == 'true'
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          set -euo pipefail
+          CONF=mkosi.presets/system/mkosi.conf.d/ubuntu.conf
+          # The 595 branch only ships the unversioned `nvidia-driver-open`
+          # metapackage in NVIDIA's CUDA repo (no `nvidia-driver-595-open`).
+          # Match by package name only so this survives future 580.x.y bumps.
+          sed -i -E \
+            -e 's|^([[:space:]]*)nvidia-driver-580-open=.*|\1nvidia-driver-open=595.71.05-1ubuntu1|' \
+            -e 's|^([[:space:]]*)nvidia-persistenced=.*|\1nvidia-persistenced=595.71.05-1ubuntu1|' \
+            -e 's|^([[:space:]]*)nvidia-fabricmanager=.*|\1nvidia-fabricmanager=595.71.05-1ubuntu1|' \
+            -e 's|^([[:space:]]*)libnvidia-nscq=.*|\1libnvidia-nscq=595.71.05-1ubuntu1|' \
+            "$CONF"
+          echo "----- Updated NVIDIA package pins -----"
+          grep -E '^[[:space:]]*(nvidia|libnvidia)' "$CONF"
+
+      - name: Increase debug root partition for CC595 drivers
+        if: needs.meta.outputs.b200_cc_drivers == 'true' && matrix.profile == 'debug'
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          set -euo pipefail
+          CONF=mkosi.presets/system/mkosi.repart-debug/10-root.conf
+          # NVIDIA 595 drivers make the root filesystem too large for
+          # systemd-repart's Minimize=guess estimation, causing mkfs.ext4
+          # "No space left on device" during the build.
+          printf '[Partition]\nType=root\nFormat=ext4\nCopyFiles=/\nMinimize=off\nSizeMinBytes=12G\nSizeMaxBytes=12G\n' > "$CONF"
+          echo "----- Updated repart config -----"
+          cat "$CONF"
+
+      - name: Resolve installed NVIDIA driver version
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          set -euo pipefail
+          CONF=mkosi.presets/system/mkosi.conf.d/ubuntu.conf
+          DRIVER_LINE=$(grep -E '^[[:space:]]*nvidia-driver(-580)?-open=' "$CONF" | head -n1)
+          DRIVER_VER=$(printf '%s' "$DRIVER_LINE" | sed -E 's|.*=([0-9]+\.[0-9]+\.[0-9]+).*|\1|')
+          echo "Resolved NVIDIA driver version: $DRIVER_VER"
+          echo "NVIDIA_DRIVER=$DRIVER_VER" >> "$GITHUB_ENV"
+
       - name: Build OS image
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         env:
@@ -265,6 +328,7 @@ jobs:
             --arg distro "$DISTRO" \
             --arg profile "$PROFILE" \
             --arg tee_platform "$TEE_PLATFORM" \
+            --arg nvidia_driver "$NVIDIA_DRIVER" \
             --arg build_date "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
             '$ARGS.named' > /tmp/measurements.json
 
@@ -307,6 +371,7 @@ jobs:
             --annotation "com.cohere.caa.commit=${CAA_COMMIT}" \
             --annotation "com.cohere.caa.version=${GITHUB_REF_NAME}" \
             --annotation "com.cohere.rtmr2=${RTMR2}" \
+            --annotation "com.cohere.nvidia.driver=${NVIDIA_DRIVER}" \
             --format json > oras-output.json
 
           cat oras-output.json

src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/ubuntu.conf

@@ -25,11 +25,77 @@ Packages=
     iptables
     e2fsprogs
     cryptsetup
-    nvidia-driver-580-open=580.126.20-1ubuntu1
-    nvidia-persistenced=580.126.20-1ubuntu1
-    nvidia-fabricmanager=580.126.20-1
-    libnvidia-nscq=580.126.20-1
+    nvidia-driver-open=595.71.05-1ubuntu1
+    nvidia-persistenced=595.71.05-1ubuntu1
+    nvidia-fabricmanager=595.71.05-1ubuntu1
+    libnvidia-nscq=595.71.05-1ubuntu1
     nvidia-container-toolkit=1.19.0-1
+    nvidia-fabricmanager-dev=595.71.05-1ubuntu1
+    nvlsm
+    # Remaining nvlink5-595 metapackage components that aren't already pulled in
+    # above. Per the NVIDIA Fabric Manager User Guide §"Installing Fabric
+    # Manager / Systems Using Fourth Generation NVSwitches", the canonical
+    # B200/B300 SVM install is `apt-get install -V nvlink5-<branch>`. That
+    # metapackage is now deprecated (no files, just dependencies; see
+    # https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/nvlink5-migration.html);
+    # NVIDIA's documented migration is to install the individual components
+    # explicitly. The complete dep list of nvlink5_595.71.05-1 is:
+    #   libnvidia-nscq (>=595)            — already above
+    #   libnvsdm (>=595)                  — added below
+    #   nvidia-fabricmanager (>=595)      — already above
+    #   nvidia-imex (>=595)               — added below
+    #   nvidia-dkms-open / nvidia-kernel-open-dkms (>=595)  — provided by nvidia-driver-open above
+    #   libnvidia-compute / nvidia-driver-cuda (>=595)      — provided by nvidia-driver-open above
+    #   collectx-bringup (>=1.22.1)       — added below
+    #   mft (>=4.35.0.159)                — added below
+    #   mft-oem (>=4.35.0.159)            — added below
+    #   mft-autocomplete (>=4.35.0.159)   — added below
+    #   nvlsm (>=2025.10.12)              — already above
+    #
+    # Why each one matters (FM Guide §Shared NVSwitch Virtualization Model
+    # and §1259 NVSwitch Errors On DGX B200/B300):
+    #   libnvsdm     — NVSwitch Device Manager telemetry library; replaces
+    #                  the SXID error path on B200/B300, used by FM/NVLSM
+    #                  and DCGM to surface NVSwitch errors. Without it,
+    #                  NVSwitch faults are invisible.
+    #   nvidia-imex  — Internode Memory Exchange daemon. Manages secure
+    #                  cross-OS-instance shared CUDA memory channels over
+    #                  NVLink. Required for multi-tenant Shared NVSwitch
+    #                  workloads where NCCL group formation registers
+    #                  memory regions that cross partition boundaries;
+    #                  without IMEX the GPU's secure subsystem fires
+    #                  Xid 170 SECURE Fatal CROSS_CONTAIN. See fortress
+    #                  scratch/oci-b200/.../docs/KNOWN-ISSUES.md §1
+    #                  "mode C" for the symptom this targets.
+    #   collectx-*   — UFM Telemetry / CX bringup utilities; part of the
+    #                  CX bridge management plane on B200.
+    #   mft / mft-oem — Mellanox Firmware Tools (mst, flint, mlxconfig,
+    #                  mlxlink); needed for CX firmware diagnostics when
+    #                  triaging an LPF that fails to come up.
+    libnvsdm=595.71.05-1ubuntu1
+    nvidia-imex=595.71.05-1ubuntu1
+    collectx-bringup
+    mft
+    mft-oem
+    mft-autocomplete
+    # Userspace OFED bits that the FM User Guide "NVIDIA Software Packages"
+    # section requires on B200/B300 ("OFED or MOFED package is required").
+    # rdma-core pulls in libibverbs1, librdmacm1, libibmad5, libibumad3,
+    # libibnetdisc5, etc. ibverbs-utils gives ibv_devices/ibv_devinfo for
+    # triage. infiniband-diags below already provides ibstatus/ibstat.
+    rdma-core
+    ibverbs-utils
+    infiniband-diags
+    docker.io
+    python3-minimal
+    python3-pip
+    curl
+    iputils-ping
+    pciutils
+    libcurl4t64
+    libxml2
+    libxmlsec1-openssl
+    pciutils
 
 RemoveFiles=/etc/issue
 RemoveFiles=/etc/issue.net

src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.repart-debug/10-root.conf

@@ -2,4 +2,12 @@
 Type=root
 Format=ext4
 CopyFiles=/
-Minimize=guess
+# `Minimize=guess` chronically under-sizes the debug rootfs once the
+# NVIDIA 595 stack (driver-open + fabricmanager + nscq + persistenced +
+# nv-attestation-sdk) lands in /usr — the build either fails with
+# `no space left on device` mid-mkosi or produces a qcow2 whose
+# rootfs runs out of space at first boot. Pin to a fixed 12 GiB so
+# the debug variant has headroom for the full B200 CC userspace.
+Minimize=off
+SizeMinBytes=12G
+SizeMaxBytes=12G

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/modules-load.d/nvlink-fabric.conf

@@ -0,0 +1,3 @@
+# Required by nv-fabricmanager NVL5+ subnet management path
+# (nvidia-fabricmanager-start.sh checks lsmod for ib_umad and exits if missing)
+ib_umad

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/nvidia-persistenced.service.d/override.conf

@@ -3,6 +3,10 @@ After=nvidia-fabricmanager.service
 
 [Service]
 ExecCondition=/usr/local/bin/check-nvidia-gpu
+# Block daemon startup until every visible GPU has fabric.state=Completed.
+# See /usr/local/bin/wait-nvlink-fabric.sh for the rationale (B200 CC race).
+ExecStartPre=/usr/local/bin/wait-nvlink-fabric.sh
+
 ExecStart=
 ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --uvm-persistence-mode --verbose
 ExecStartPost=/usr/bin/nvidia-smi conf-compute -srs 1

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/local/bin/wait-nvlink-fabric.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# wait-nvlink-fabric.sh — block until NVML reports the NVLink fabric
+# registration is COMPLETE on every visible GPU.
+#
+# Why this exists (HGX B200 + CC/NVLE):
+#   On B200 in confidential-compute mode, the Service VM Fabric Manager
+#   programs NVSwitch routing for a partition asynchronously. The handshake
+#   with the guest GPU happens over in-band NVLink MAD (Probe Request ->
+#   Probe Response) AFTER fmActivateFabricPartition() returns success.
+#   nvidia-persistenced therefore races the handshake on guest boot. If it
+#   registers a GPU before that GPU finishes its handshake, NVML returns
+#   0x81 (NVLINK_FABRIC_NOT_READY) and the daemon silently falls back to
+#   non-UVM persistence — leaving that GPU permanently unable to do NVLink
+#   P2P. Per the NVIDIA Secure AI Operations Guide ("Ensure that
+#   Persistence Mode is On"), the only way to recover from a missed
+#   SPDM/UVM session in CC mode is an FLR, i.e. a full VM restart. So we
+#   gate persistenced startup on fabric readiness rather than rely on the
+#   daemons internal retry path (it does not retry on 0x81).
+#
+# This script is wired in via the nvidia-persistenced service drop-in
+#   /usr/lib/systemd/system/nvidia-persistenced.service.d/override.conf
+# as ExecStartPre. It exits non-zero on timeout so the daemon fails loud
+# rather than silently degrading.
+
+set -euo pipefail
+
+TIMEOUT_SEC="${WAIT_FABRIC_TIMEOUT:-180}"
+POLL_SEC="${WAIT_FABRIC_POLL:-2}"
+
+log() { printf "[wait-nvlink-fabric] %s\n" "$*" >&2; }
+
+deadline=$(( $(date +%s) + TIMEOUT_SEC ))
+attempt=0
+
+while :; do
+    attempt=$((attempt+1))
+    states=$(nvidia-smi --query-gpu=index,uuid,fabric.state,fabric.status \
+                --format=csv,noheader 2>/dev/null || true)
+
+    if [[ -z "$states" ]]; then
+        log "attempt $attempt: nvidia-smi returned empty (driver not ready yet)"
+    else
+        not_ready=$(echo "$states" | awk -F"," "
+            BEGIN { n = 0 }
+            {
+                gsub(/^ +| +\$/, \"\", \$3)
+                gsub(/^ +| +\$/, \"\", \$4)
+                if (\$3 != \"Completed\" || \$4 != \"Success\") n++
+            }
+            END { print n+0 }")
+
+        if (( not_ready == 0 )); then
+            log "all GPUs fabric ready (attempt $attempt)"
+            echo "$states" | sed "s/^/[wait-nvlink-fabric]   /" >&2
+            exit 0
+        fi
+        log "attempt $attempt: $not_ready GPU(s) fabric not yet Completed/Success"
+    fi
+
+    if (( $(date +%s) >= deadline )); then
+        log "FAIL: NVLink fabric did not become ready within ${TIMEOUT_SEC}s"
+        log "current per-GPU state:"
+        echo "${states:-<empty>}" | sed "s/^/[wait-nvlink-fabric]   /" >&2
+        exit 1
+    fi
+
+    sleep "$POLL_SEC"
+done

src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.ubuntu

@@ -17,6 +17,8 @@ ARG CUSTOM_GC_BINARIES=""
 ARG AA_FEATURES=""
 ARG GUEST_COMPONENTS_REF=""
 ARG GUEST_COMPONENTS_REPO="https://github.com/confidential-containers/guest-components.git"
+ARG NVAT_REPO="https://github.com/NVIDIA/attestation-sdk.git"
+ARG NVAT_TAG="2026.03.02"
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -e; \
     if [ -n "${CUSTOM_GC_BINARIES}" ] && [ -z "${GUEST_COMPONENTS_REF}" ]; then \
@@ -26,15 +28,34 @@ RUN set -e; \
     if [ -n "${CUSTOM_GC_BINARIES}" ] && [ -n "${GUEST_COMPONENTS_REF}" ]; then \
       apt-get update && \
       apt-get install -y --no-install-recommends \
-        protobuf-compiler pkg-config clang libssl-dev libtss2-dev && \
+        protobuf-compiler pkg-config clang libclang-dev libssl-dev libtss2-dev \
+        cmake libcurl4-openssl-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl && \
       apt-get clean && rm -rf /var/lib/apt/lists/* && \
       mkdir -p /build/gc && cd /build/gc && \
       git init && \
       git remote add origin "${GUEST_COMPONENTS_REPO}" && \
       git fetch --depth=1 origin "${GUEST_COMPONENTS_REF}" && \
       git reset --hard FETCH_HEAD; \
     fi
+# Build NVIDIA Attestation SDK (libnvat) from source so the nvidia-attester
+# feature can link against it. The AA binary will dynamically link libnvat.so,
+# which must also be present in the final PodVM image at runtime.
+# Installs its own build deps so this works even without CUSTOM_GC_BINARIES.
+RUN set -e; \
+    if echo "${AA_FEATURES}" | grep -q "nvidia-attester"; then \
+      apt-get update && \
+      apt-get install -y --no-install-recommends cmake libssl-dev libcurl4-openssl-dev \
+        libxml2-dev libxmlsec1-dev libxmlsec1-openssl && \
+      apt-get clean && rm -rf /var/lib/apt/lists/* && \
+      git clone --depth 1 --branch "${NVAT_TAG}" "${NVAT_REPO}" /build/nvat && \
+      cd /build/nvat/nv-attestation-sdk-cpp && \
+      cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib && \
+      cmake --build build && \
+      cmake --install build && \
+      ldconfig; \
+    fi
 COPY cloud-api-adaptor/podvm/build-guest-components.sh /build/
+ENV NVAT_USE_SYSTEM_LIB=1
 RUN /build/build-guest-components.sh "${CUSTOM_GC_BINARIES}" "${AA_FEATURES}"
 
 # ubuntu:24.04
@@ -146,5 +167,14 @@ RUN for bin in /tmp/gc-overrides/*; do \
       install -m0755 "$bin" /src/cloud-api-adaptor/podvm/files/usr/local/bin/"$(basename "$bin")"; \
     done; true
 
+# Copy libnvat shared library if it was built (needed at runtime by attestation-agent
+# when compiled with nvidia-attester feature). Uses a mount instead of COPY so
+# builds without nvidia-attester don't fail on an empty glob.
+RUN --mount=from=gc_builder,src=/usr/lib/,dst=/tmp/gc-lib/,readonly \
+    if ls /tmp/gc-lib/libnvat* 1>/dev/null 2>&1; then \
+      mkdir -p /src/cloud-api-adaptor/podvm/files/usr/lib/ && \
+      cp /tmp/gc-lib/libnvat* /src/cloud-api-adaptor/podvm/files/usr/lib/; \
+    fi; true
+
 FROM scratch
 COPY --from=podvm_binaries_builder /src/cloud-api-adaptor/podvm/files /

src/cloud-api-adaptor/podvm/build-guest-components.sh

@@ -30,6 +30,10 @@ for bin in "${BINS[@]}"; do
         exit 1
       fi
       cd /build/gc/attestation-agent/attestation-agent
+      # Refresh lockfile so optional feature deps (e.g. nv-attestation-sdk
+      # for nvidia-attester) are resolved even if the checked-in Cargo.lock
+      # was generated without them.
+      cargo update --workspace
       cargo build --release --locked --no-default-features \
         --features "$AA_FEATURES" --bin ttrpc-aa
       cp /build/gc/target/release/ttrpc-aa "$OUTDIR/attestation-agent"