diff --git a/.github/workflows/build-kata-uvm-cohere.yaml b/.github/workflows/build-kata-uvm-cohere.yaml new file mode 100644 index 0000000000..f7a16ab177 --- /dev/null +++ b/.github/workflows/build-kata-uvm-cohere.yaml @@ -0,0 +1,709 @@ +name: Build Kata UVM Image (Cohere NVIDIA GPU Confidential) + +# Build the Kata Containers NVIDIA-GPU-confidential UVM image with our +# attestation-agent + api-server-rest baked in *from source*, instead of +# post-hoc patching the stock NVIDIA image (which is what the legacy +# fortress/scratch/oci-b200/k8s/08-patch-uvm.sh does). +# +# How: +# 1. Check out kata-containers @ ${kata_ref}. +# 2. Rewrite versions.yaml: point externals.coco-guest-components.url / +# .version at our cohere-ai/guest-components fork. The kata build +# driver clones that and statically builds AA + api-server-rest + +# CDH. nvidia_rootfs.sh's coco_guest_components() step then copies +# those binaries into the final UVM rootfs at /usr/local/bin/. +# 3. Run `make rootfs-image-nvidia-gpu-confidential-tarball` (which also +# builds agent, busybox, pause-image, coco-guest-components, and +# kernel-nvidia-gpu under the hood — every dep is containerised by +# kata-deploy-binaries-in-docker.sh, so the runner just needs Docker). +# 4. Extract the .image + root_hash file from the tarball. +# 5. Push to GHCR as an OCI artifact with the dm-verity params surfaced +# as annotations so the host install script can wire kata config +# without re-running `veritysetup format`. +# +# Output OCI ref: +# ghcr.io/${{ github.repository }}/kata-uvm-nvidia-gpu-confidential: +# +# Companion install script (consumes this artifact on a B200 host): +# fortress/scratch/oci-b200/k8s/05-install-uvm.sh + +on: + push: + tags: ["kata-uvm-v*"] + branches: + - "cohere" + # TEMPORARY: enable end-to-end validation of the workflow on the + # feature branch before merge. Remove this entry as part of the + # final review; only `cohere` should remain. + - "alhassankhedr/build-kata-uvm-cohere" + paths: + - ".github/workflows/build-kata-uvm-cohere.yaml" + workflow_dispatch: + inputs: + kata_ref: + description: "kata-containers ref to build from (tag, branch, or SHA)" + required: false + type: string + default: "3.30.0" + kata_repo: + description: "kata-containers repo URL" + required: false + type: string + default: "https://github.com/kata-containers/kata-containers.git" + gc_repo: + description: "guest-components repo URL" + required: false + type: string + default: "https://github.com/cohere-ai/guest-components.git" + gc_ref: + description: | + guest-components ref (branch, tag, or SHA). + + Default: alhassankhedr/sync-main-to-cohere (head of PR #9). + That branch carries upstream main's nvidia-attester rewrite + (NVAT SDK based, no `count == 1` guard) and is required for + multi-GPU evidence to work end-to-end on 8x B200 hosts. The + plain `cohere` branch still has the old NVML-based attester + which silently produces empty evidence on 2+ GPU systems + (mod a sed `s/count == 1/count >= 1/` patch the podvm-mkosi + Dockerfile applies). Switch back to `cohere` after PR #9 merges. + required: false + type: string + default: "alhassankhedr/sync-main-to-cohere" + kata_nvidia_driver_ver: + description: | + Override .externals.nvidia.driver.version in kata's versions.yaml + (e.g. 595.71.05). Leave empty to use kata's default. The tag must + exist at https://github.com/NVIDIA/open-gpu-kernel-modules and a + matching nvidia-driver-pinning- package must exist in the + NVIDIA CUDA apt repo. Required to fix the 8x B200 fabric-probe + race in kata 3.30.0's default 595.58.03 driver pin. + required: false + type: string + default: "" + kata_nvat_ver: + description: | + Pin .externals.nvidia.nvat.version in kata's versions.yaml. Default + 2026.03.02 (a real tag in https://github.com/NVIDIA/attestation-sdk). + + Without this pin, kata's coco-guest-components builder Dockerfile + skips the libnvat build (its `if [ -n "${NVAT_VERSION}" ]` guard + short-circuits), which makes the second AA build pass — the one + that links the nvidia-attester cargo feature against + /usr/local/lib/libnvat.so and installs the result as + /usr/local/bin/attestation-agent-nv — silently no-op. Net effect: + the AA baked into the rootfs has no GPU evidence support + regardless of gc_ref, and /aa/additional_evidence on multi-GPU + pods returns empty. Set "" to leave nvat unpinned (matches + upstream kata 3.30.0 behaviour). + required: false + type: string + default: "2026.03.02" + nvidia_gpu_stack: + description: "NVIDIA GPU stack components (driver= is added from versions.yaml)" + required: false + type: string + default: "compute,dcgm,nvswitch" + tag_suffix: + description: "Optional suffix appended to the OCI tag (e.g. for ad-hoc test builds)" + required: false + type: string + default: "" + +permissions: + id-token: write + attestations: write + contents: read + packages: write + +env: + OCI_IMAGE: ghcr.io/${{ github.repository }}/kata-uvm-nvidia-gpu-confidential + +jobs: + meta: + name: Compute metadata + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.compute.outputs.tag }} + kata_ref: ${{ steps.compute.outputs.kata_ref }} + gc_repo: ${{ steps.compute.outputs.gc_repo }} + gc_ref: ${{ steps.compute.outputs.gc_ref }} + kata_nvidia_driver_ver: ${{ steps.compute.outputs.kata_nvidia_driver_ver }} + kata_nvat_ver: ${{ steps.compute.outputs.kata_nvat_ver }} + nvidia_gpu_stack: ${{ steps.compute.outputs.nvidia_gpu_stack }} + steps: + - name: Compute tag and inputs + id: compute + env: + KATA_REF: ${{ inputs.kata_ref || '3.30.0' }} + GC_REPO: ${{ inputs.gc_repo || 'https://github.com/cohere-ai/guest-components.git' }} + GC_REF: ${{ inputs.gc_ref || 'alhassankhedr/sync-main-to-cohere' }} + DRIVER_VER: ${{ inputs.kata_nvidia_driver_ver || '' }} + NVAT_VER: ${{ inputs.kata_nvat_ver || '2026.03.02' }} + STACK: ${{ inputs.nvidia_gpu_stack || 'compute,dcgm,nvswitch' }} + SUFFIX: ${{ inputs.tag_suffix || '' }} + run: | + # Tag pattern: + # kata-uvm-v* push -> use the tag literal (after stripping `kata-uvm-`) + # workflow_dispatch -> kata-${KATA_REF}-gc-${GC_REF_SHORT}[-drv-][-nvat-][suffix] + # branch push -> cohere-latest + if [[ "$GITHUB_REF" == refs/tags/kata-uvm-v* ]]; then + TAG="${GITHUB_REF#refs/tags/kata-uvm-}" + elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + GC_SHORT="${GC_REF//\//-}" + GC_SHORT="${GC_SHORT:0:12}" + TAG="kata-${KATA_REF//\//-}-gc-${GC_SHORT}" + # If the caller overrode the NVIDIA driver pin, surface it in the + # OCI tag so the artifact name unambiguously identifies which + # driver is baked in. + [ -n "$DRIVER_VER" ] && TAG="${TAG}-drv-${DRIVER_VER}" + # Same for the NVAT SDK pin: the artifact ABI changes meaningfully + # between NVAT releases (libnvat soname + GpuEvidenceSource API), + # so make this visible too. + [ -n "$NVAT_VER" ] && TAG="${TAG}-nvat-${NVAT_VER}" + else + TAG="cohere-latest" + fi + [ -n "$SUFFIX" ] && TAG="${TAG}-${SUFFIX}" + # OCI tags can't have '+' or unbounded length; sanitize. + TAG="${TAG//+/-}" + { + echo "tag=$TAG" + echo "kata_ref=$KATA_REF" + echo "gc_repo=$GC_REPO" + echo "gc_ref=$GC_REF" + echo "kata_nvidia_driver_ver=$DRIVER_VER" + echo "kata_nvat_ver=$NVAT_VER" + echo "nvidia_gpu_stack=$STACK" + } >> "$GITHUB_OUTPUT" + + build: + name: Build kata UVM (nvidia-gpu-confidential) + needs: meta + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - name: Free up runner disk space + # The kata build pulls a CUDA repo + NVIDIA drivers into a chroot + # and a kernel build alongside. Default ubuntu-latest leaves ~14G; + # we need ~40G or the rootfs build OOMs the disk. + run: | + set -eux + df -h / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ + /usr/local/share/boost /opt/hostedtoolcache/CodeQL \ + /usr/local/share/powershell /usr/local/share/chromium + sudo apt-get purge -y google-cloud-cli azure-cli microsoft-edge-stable \ + dotnet-* aspnetcore-* mongodb-* mysql-* 2>/dev/null || true + sudo apt-get autoremove -y + sudo apt-get clean + docker system prune -af --volumes 2>/dev/null || true + df -h / + + - name: Install host build dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + git make curl ca-certificates jq python3 python3-pip + # Ensure yq is present (kata's build scripts rely on it). + if ! command -v yq >/dev/null 2>&1; then + sudo curl -fsSL -o /usr/local/bin/yq \ + https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + fi + yq --version + + - name: Install ORAS + # Pin via fortress/CAA convention: read from caa's versions.yaml so + # we stay in lockstep. Fallback to a known-good version if the file + # is unavailable for some reason. + run: | + ORAS_VERSION=1.2.0 + curl -fsSLO "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" + tar -xzf "oras_${ORAS_VERSION}_linux_amd64.tar.gz" oras + sudo mv oras /usr/local/bin/ + rm -f "oras_${ORAS_VERSION}_linux_amd64.tar.gz" + oras version + + - name: Checkout kata-containers @ ${{ needs.meta.outputs.kata_ref }} + run: | + set -eux + git clone --depth 1 --branch "${{ needs.meta.outputs.kata_ref }}" \ + "${{ inputs.kata_repo || 'https://github.com/kata-containers/kata-containers.git' }}" \ + /tmp/kata + ( cd /tmp/kata && git rev-parse HEAD ) + + - name: Override coco-guest-components in versions.yaml + # This is the key step: tell kata's coco-guest-components builder + # to clone our cohere-ai fork at our chosen ref. Everything + # downstream (rootfs assembly, dm-verity, root_hash) is unchanged + # and uses these binaries as if they had come from upstream. + env: + GC_REPO: ${{ needs.meta.outputs.gc_repo }} + GC_REF: ${{ needs.meta.outputs.gc_ref }} + run: | + set -eux + cd /tmp/kata + # Resolve gc_ref to a SHA so the build is reproducible. We do + # this with `git ls-remote` rather than cloning the whole tree. + GC_SHA=$(git ls-remote "${GC_REPO}" "${GC_REF}" | awk '{print $1}' | head -n1) + if [[ -z "$GC_SHA" ]]; then + # Maybe gc_ref already IS a SHA; let downstream fail loudly if not. + GC_SHA="${GC_REF}" + fi + echo "Resolved guest-components ref ${GC_REF} -> ${GC_SHA}" + + yq -i \ + ".externals.\"coco-guest-components\".url = \"${GC_REPO}\" | + .externals.\"coco-guest-components\".version = \"${GC_SHA}\"" \ + versions.yaml + + echo "----- updated versions.yaml (coco-guest-components) -----" + yq '.externals."coco-guest-components"' versions.yaml + + - name: Override NVIDIA driver pin in versions.yaml + # kata 3.30.0's versions.yaml pins driver=595.58.03, but on 8x B200 + # OCI hosts that driver hits a fabric-probe race (kernel timeout in + # RmGpuFabricProbe -> fail-stop). The fix landed in 595.71.05. The + # build pulls open kernel modules from + # https://github.com/NVIDIA/open-gpu-kernel-modules tags, and the + # userspace via nvidia-driver-pinning- from the NVIDIA CUDA + # apt repo, so any version that exists in both places is a valid + # override. Skipped when input is empty. + if: needs.meta.outputs.kata_nvidia_driver_ver != '' + env: + DRIVER_VER: ${{ needs.meta.outputs.kata_nvidia_driver_ver }} + run: | + set -eux + cd /tmp/kata + OLD_VER=$(yq '.externals.nvidia.driver.version' versions.yaml | tr -d '"') + yq -i ".externals.nvidia.driver.version = \"${DRIVER_VER}\"" versions.yaml + echo "NVIDIA driver pin: ${OLD_VER} -> ${DRIVER_VER}" + echo "----- updated versions.yaml (nvidia.driver) -----" + yq '.externals.nvidia.driver' versions.yaml + + - name: Pin NVIDIA Attestation SDK (libnvat) in versions.yaml + # kata's tools/packaging/static-build/coco-guest-components/build.sh + # reads `.externals.nvidia.nvat.version` and forwards it to the GC + # builder Dockerfile as NVAT_VERSION. The Dockerfile gates the entire + # libnvat clone+cmake+install behind `if [ -n "${NVAT_VERSION}" ]`, + # so an unset key (the default in upstream kata 3.30.0) means no + # libnvat in the builder image. Without libnvat, + # build-static-coco-guest-components.sh's second build pass — the one + # that compiles AA with `nvidia-attester` against + # /usr/local/lib/libnvat.so and installs the result as + # /usr/local/bin/attestation-agent-nv — silently no-ops, and the + # rootfs ends up with only the no-NVIDIA AA. /aa/additional_evidence + # then returns empty regardless of which guest-components we baked. + # Pin a real attestation-sdk tag here so libnvat actually gets built. + if: needs.meta.outputs.kata_nvat_ver != '' + env: + NVAT_VER: ${{ needs.meta.outputs.kata_nvat_ver }} + run: | + set -eux + cd /tmp/kata + OLD_VER=$(yq '.externals.nvidia.nvat.version // ""' versions.yaml | tr -d '"') + yq -i \ + ".externals.nvidia.nvat.version = \"${NVAT_VER}\" | + .externals.nvidia.nvat.url = \"https://github.com/NVIDIA/attestation-sdk\" | + .externals.nvidia.nvat.desc = \"NVIDIA Attestation SDK (libnvat); enables attestation-agent-nv\"" \ + versions.yaml + if [[ -z "$OLD_VER" || "$OLD_VER" == "null" ]]; then + echo "NVAT SDK pin: (unset) -> ${NVAT_VER}" + else + echo "NVAT SDK pin: ${OLD_VER} -> ${NVAT_VER}" + fi + echo "----- updated versions.yaml (nvidia.nvat) -----" + yq '.externals.nvidia.nvat' versions.yaml + + - name: Force a clean kernel + rootfs rebuild when overriding the driver + # Mirrors KATA_NVIDIA_FORCE_REBUILD in + # fortress/scratch/oci-b200/k8s/04-build-uvm-locally.sh. Each kata + # kernel build generates a fresh random `certs/signing_key.pem`, + # and the NVIDIA modules in + # `kata-static-kernel-nvidia-gpu-modules.tar.zst` are signed + # against THAT key. If kata's make reuses any cached kernel- + # nvidia-gpu/ artifacts while we're trying to bump the driver + # version, we end up with userspace<->kernel ABI skew or, worse, + # NVIDIA `.ko` files signed against a different key than the + # one the kernel binary embeds (README "Bug F"). On a fresh CI + # runner this is a no-op, but if we ever start caching the + # kata checkout between runs (or someone re-runs a job with a + # different driver_ver) the wipe makes the build deterministic. + if: needs.meta.outputs.kata_nvidia_driver_ver != '' + run: | + set -eux + BUILD=/tmp/kata/tools/packaging/kata-deploy/local-build/build + rm -rf "$BUILD/kernel-nvidia-gpu" \ + "$BUILD/kata-static-kernel-nvidia-gpu-modules.tar.zst" \ + "$BUILD/kata-static-kernel-nvidia-gpu.tar.zst" \ + "$BUILD/rootfs-image-nvidia-gpu-confidential" \ + "$BUILD/rootfs-nvidia-gpu-confidential-stage-one" \ + "$BUILD/kata-static-rootfs-image-nvidia-gpu-confidential.tar.zst" \ + 2>/dev/null || true + echo "wiped kernel-nvidia-gpu/, modules tarball, and rootfs build dirs" + + - name: Build rootfs-image-nvidia-gpu-confidential + env: + NVIDIA_GPU_STACK: ${{ needs.meta.outputs.nvidia_gpu_stack }} + run: | + set -eux + cd /tmp/kata/tools/packaging/kata-deploy/local-build + # kata 3.30+ nvidia_chroot.sh runs with `set -u` and only assigns + # driver_version when NVIDIA_GPU_STACK contains a literal + # `driver=` (tools/osbuilder/rootfs-builder/nvidia/ + # nvidia_chroot.sh::install_userspace_components). Without it, + # the rootfs-assembly stage dies at the very end with + # `driver_version: unbound variable`. Derive the canonical pin + # from kata's own versions.yaml so this auto-tracks KATA_REF. + if [[ ",${NVIDIA_GPU_STACK}," != *",driver="* ]]; then + DRIVER_VER=$(yq '.externals.nvidia.driver.version' /tmp/kata/versions.yaml 2>/dev/null | tr -d '"') + if [[ -n "$DRIVER_VER" && "$DRIVER_VER" != "null" ]]; then + NVIDIA_GPU_STACK="driver=${DRIVER_VER},${NVIDIA_GPU_STACK}" + echo "Prepended driver=${DRIVER_VER} (from versions.yaml) -> ${NVIDIA_GPU_STACK}" + else + echo "WARN: could not resolve .externals.nvidia.driver.version from /tmp/kata/versions.yaml" >&2 + fi + fi + # `make -tarball` chains all the Docker-isolated builds + # (agent, busybox, pause-image, coco-guest-components, + # kernel-nvidia-gpu) before running the rootfs assembly. Each + # sub-build runs in its own ephemeral container, so we don't + # need to install rust/go/etc on the host. + NVIDIA_GPU_STACK="$NVIDIA_GPU_STACK" \ + make rootfs-image-nvidia-gpu-confidential-tarball + + ls -lh build/ + + - name: Extract .image and root_hash from the tarball + run: | + set -euxo pipefail + cd /tmp/kata/tools/packaging/kata-deploy/local-build/build + TARBALL=kata-static-rootfs-image-nvidia-gpu-confidential.tar.zst + [[ -f "$TARBALL" ]] || { echo "FATAL: $TARBALL missing"; exit 1; } + + mkdir -p /tmp/uvm-out + # Tarball layout (the .img entry is a symlink to the + # versioned .image alongside it): + # ./opt/kata/share/kata-containers/kata-containers-nvidia-gpu-confidential.img + # -> kata-ubuntu-noble-nvidia-gpu-confidential-.image + # ./opt/kata/share/kata-containers/kata-ubuntu-noble-nvidia-gpu-confidential-.image + # ./opt/kata/share/kata-containers/root_hash_nvidia-gpu-confidential.txt + tar --zstd -xvf "$TARBALL" -C /tmp/uvm-out + + # Resolve the symlink to the real image and copy it (not move), + # so the underlying file survives the `rm -rf opt/` below. + KATA_IMG_LINK=/tmp/uvm-out/opt/kata/share/kata-containers/kata-containers-nvidia-gpu-confidential.img + KATA_IMG_REAL=$(readlink -f "$KATA_IMG_LINK") + [[ -f "$KATA_IMG_REAL" ]] || { echo "FATAL: $KATA_IMG_LINK -> $KATA_IMG_REAL missing"; exit 1; } + cp --reflink=auto "$KATA_IMG_REAL" /tmp/uvm-out/kata-containers-nvidia-gpu-confidential.img + + mv /tmp/uvm-out/opt/kata/share/kata-containers/root_hash_nvidia-gpu-confidential.txt \ + /tmp/uvm-out/root_hash.txt + rm -rf /tmp/uvm-out/opt + ls -lh /tmp/uvm-out/ + echo "----- root_hash.txt -----" + cat /tmp/uvm-out/root_hash.txt + file /tmp/uvm-out/kata-containers-nvidia-gpu-confidential.img + + - name: Stage paired kernel binary alongside the rootfs + # WHY: kata's kernel-nvidia-gpu build emits a fresh random + # `certs/signing_key.pem` per invocation and uses it to sign both + # the embedded NVIDIA modules and (transitively) the modules + # tarball that nvidia_rootfs.sh extracts into the rootfs. The + # rootfs we produced in the previous step therefore carries + # NVIDIA `.ko` files signed against THIS build's key. If the + # host running the resulting UVM uses a kernel from a different + # build (e.g. the kata-deploy-bundled one), the modules are + # rejected at first modprobe, NVRC panics, the guest powers + # down, and pods sit in Pending. Verified end-to-end on + # 2026-05-15 (README "Bug F"). The fix on the install side + # (fortress/scratch/oci-b200/k8s/05-install-uvm.sh) is to + # atomically install both the kernel and the rootfs from the + # same build. For that to work, the OCI artifact has to ship + # the kernel binary alongside the rootfs. + run: | + set -euxo pipefail + KBUILD_DESTDIR=/tmp/kata/tools/packaging/kata-deploy/local-build/build/kernel-nvidia-gpu/destdir/opt/kata/share/kata-containers + KVER_FILE=$(ls "${KBUILD_DESTDIR}"/vmlinuz-*-nvidia-gpu 2>/dev/null | head -n1 || true) + if [[ -z "$KVER_FILE" ]]; then + echo "FATAL: no locally-built kernel at ${KBUILD_DESTDIR}/vmlinuz-*-nvidia-gpu" >&2 + exit 1 + fi + KVER_BASENAME=$(basename "$KVER_FILE") + KVER_VERSION="${KVER_BASENAME#vmlinuz-}" + cp -p "$KVER_FILE" "/tmp/uvm-out/${KVER_BASENAME}" + for sib in "vmlinux-${KVER_VERSION}" "System.map-${KVER_VERSION}" "config-${KVER_VERSION}"; do + [[ -f "${KBUILD_DESTDIR}/${sib}" ]] && cp -p "${KBUILD_DESTDIR}/${sib}" /tmp/uvm-out/ + done + # Single source of truth for which kernel pairs with this rootfs; + # 05-install-uvm.sh reads this on the install side. + echo "$KVER_BASENAME" > /tmp/uvm-out/kernel.basename + ls -lh /tmp/uvm-out/ + + - name: Verify NVIDIA modules signing key matches the kernel + # Defensive check that the SKID embedded in the kernel's + # `certs/signing_key.x509` appears (raw-bytes hex-encoded) + # somewhere in the trailing PKCS#7 signature of the NVIDIA + # modules tarball's nvidia.ko. Same gate fortress's + # 04-build-uvm-locally.sh applies. On a clean CI runner this + # should always pass; if it ever fails we catch it here, in + # CI, instead of via guest serial capture in production. + run: | + set -euxo pipefail + KBASENAME=$(cat /tmp/uvm-out/kernel.basename) + KVER_VERSION="${KBASENAME#vmlinuz-}" + SIGNING_X509=/tmp/kata/tools/packaging/kata-deploy/local-build/build/kernel-nvidia-gpu/builddir/kata-linux-${KVER_VERSION}/certs/signing_key.x509 + MODULES_TARBALL=/tmp/kata/tools/packaging/kata-deploy/local-build/build/kata-static-kernel-nvidia-gpu-modules.tar.zst + if [[ ! -f "$SIGNING_X509" || ! -f "$MODULES_TARBALL" ]]; then + echo "WARN: skipping signing-key check (missing $SIGNING_X509 or $MODULES_TARBALL)" + exit 0 + fi + KEY_SKID=$(openssl x509 -in "$SIGNING_X509" -noout -text \ + | awk '/X509v3 Subject Key Identifier/{getline; gsub(/[: ]/,""); print tolower($0); exit}') + if [[ -z "$KEY_SKID" ]]; then + echo "WARN: could not extract SKID from $SIGNING_X509"; exit 0 + fi + TMP=$(mktemp -d) + tar --zstd -xf "$MODULES_TARBALL" -C "$TMP" --wildcards '*/kernel/drivers/video/nvidia.ko' + SAMPLE_KO=$(find "$TMP" -name nvidia.ko -print -quit) + if [[ -z "$SAMPLE_KO" ]]; then + echo "WARN: no nvidia.ko in $MODULES_TARBALL"; exit 0 + fi + if xxd -p -c 999999 "$SAMPLE_KO" | grep -qi "$KEY_SKID"; then + echo "OK: nvidia.ko signed by this build's signing key (SKID=$KEY_SKID)" + else + echo "FATAL: nvidia.ko in modules tarball is NOT signed by the kernel's signing_key.x509 (SKID=$KEY_SKID)" + echo " guest will reject NVIDIA modules at first modprobe; pod will sit in Pending" + exit 1 + fi + + - name: Surface verity params as JSON metadata + id: measure + # The root_hash.txt file is the source of truth for kata's + # `kernel_verity_params` (root_hash, salt, data_blocks, etc). + # We re-emit those values as a flat JSON file so the host install + # script can parse them without invoking veritysetup. + # + # NOTE: kata's osbuilder writes root_hash.txt as a single + # comma-separated line, e.g. + # root_hash=,salt=,data_blocks=N,data_block_size=4096,hash_block_size=4096 + # so we split on commas first, then on '=' to populate each var. + run: | + set -euxo pipefail + + ROOT_HASH=""; SALT=""; DATA_BLOCKS="" + DATA_BLOCK_SIZE=""; HASH_BLOCK_SIZE="" + while IFS='=' read -r k v; do + case "$k" in + root_hash) ROOT_HASH=$v ;; + salt) SALT=$v ;; + data_blocks) DATA_BLOCKS=$v ;; + data_block_size) DATA_BLOCK_SIZE=$v ;; + hash_block_size) HASH_BLOCK_SIZE=$v ;; + esac + done < <(tr ',' '\n' < /tmp/uvm-out/root_hash.txt) + + # Fail loudly on parse regressions instead of producing a junk + # measurements.json with empty fields. + : "${ROOT_HASH:?root_hash missing from root_hash.txt}" + : "${SALT:?salt missing from root_hash.txt}" + : "${DATA_BLOCKS:?data_blocks missing from root_hash.txt}" + : "${DATA_BLOCK_SIZE:?data_block_size missing from root_hash.txt}" + : "${HASH_BLOCK_SIZE:?hash_block_size missing from root_hash.txt}" + [[ "$ROOT_HASH" =~ ^[0-9a-f]{64}$ ]] || { echo "bad root_hash: $ROOT_HASH"; exit 1; } + [[ "$SALT" =~ ^[0-9a-f]{64}$ ]] || { echo "bad salt: $SALT"; exit 1; } + + IMG_SHA256=$(sha256sum /tmp/uvm-out/kata-containers-nvidia-gpu-confidential.img | awk '{print $1}') + IMG_BYTES=$(stat -c %s /tmp/uvm-out/kata-containers-nvidia-gpu-confidential.img) + [[ "$IMG_SHA256" =~ ^[0-9a-f]{64}$ ]] || { echo "bad image sha256: $IMG_SHA256"; exit 1; } + # GPU UVM is always hundreds of MB; anything tiny means we measured + # a dangling symlink or an empty file. + [[ "$IMG_BYTES" -gt 104857600 ]] || { echo "image suspiciously small: $IMG_BYTES bytes"; exit 1; } + + # Paired kernel binary surfaced by the "Stage paired kernel" step. + # The install side (fortress/scratch/oci-b200/k8s/05-install-uvm.sh) + # uses .kernel.{filename,sha256} from measurements.json to validate + # and atomically install the kernel alongside the rootfs. + KERNEL_BASENAME="" + KERNEL_SHA="" + if [[ -f /tmp/uvm-out/kernel.basename ]]; then + KERNEL_BASENAME=$(cat /tmp/uvm-out/kernel.basename) + KERNEL_SHA=$(sha256sum "/tmp/uvm-out/${KERNEL_BASENAME}" | awk '{print $1}') + fi + + # Resolve the *actual* baked-in driver / nvat pins from versions.yaml + # so the artifact reports what is really installed, not just what + # was requested. (When the kata_* inputs are empty we want kata's + # default to be reflected here.) + DRIVER_VER=$(yq '.externals.nvidia.driver.version' /tmp/kata/versions.yaml | tr -d '"') + NVAT_VER=$(yq '.externals.nvidia.nvat.version // ""' /tmp/kata/versions.yaml | tr -d '"') + + jq -n \ + --arg kata_ref "${{ needs.meta.outputs.kata_ref }}" \ + --arg gc_repo "${{ needs.meta.outputs.gc_repo }}" \ + --arg gc_ref "${{ needs.meta.outputs.gc_ref }}" \ + --arg driver_ver "$DRIVER_VER" \ + --arg nvat_ver "$NVAT_VER" \ + --arg nvidia_stack "${{ needs.meta.outputs.nvidia_gpu_stack }}" \ + --arg root_hash "$ROOT_HASH" \ + --arg salt "$SALT" \ + --arg data_blocks "$DATA_BLOCKS" \ + --arg data_block_sz "$DATA_BLOCK_SIZE" \ + --arg hash_block_sz "$HASH_BLOCK_SIZE" \ + --arg img_sha256 "$IMG_SHA256" \ + --arg img_bytes "$IMG_BYTES" \ + --arg kernel_name "$KERNEL_BASENAME" \ + --arg kernel_sha "$KERNEL_SHA" \ + --arg caa_commit "$GITHUB_SHA" \ + --arg build_date "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{ + kata_ref: $kata_ref, + guest_components: {repo: $gc_repo, ref: $gc_ref}, + nvidia_driver: {version: $driver_ver}, + nvat_sdk: ( if $nvat_ver == "" or $nvat_ver == "null" then null + else {version: $nvat_ver} end ), + nvidia_gpu_stack: $nvidia_stack, + dm_verity: { + root_hash: $root_hash, + salt: $salt, + data_blocks: ($data_blocks | tonumber), + data_block_size: ($data_block_sz | tonumber), + hash_block_size: ($hash_block_sz | tonumber) + }, + image: { + filename: "kata-containers-nvidia-gpu-confidential.img", + sha256: $img_sha256, + bytes: ($img_bytes | tonumber) + }, + kernel: ( if $kernel_name == "" then null + else {filename: $kernel_name, sha256: $kernel_sha} end ), + source: {caa_commit: $caa_commit, build_date: $build_date} + }' > /tmp/uvm-out/measurements.json + + jq -e . /tmp/uvm-out/measurements.json >/dev/null + cat /tmp/uvm-out/measurements.json + echo "root_hash=$ROOT_HASH" >> "$GITHUB_OUTPUT" + echo "img_sha256=$IMG_SHA256" >> "$GITHUB_OUTPUT" + echo "driver_ver=$DRIVER_VER" >> "$GITHUB_OUTPUT" + echo "nvat_ver=$NVAT_VER" >> "$GITHUB_OUTPUT" + echo "kernel_basename=$KERNEL_BASENAME" >> "$GITHUB_OUTPUT" + echo "kernel_sha=$KERNEL_SHA" >> "$GITHUB_OUTPUT" + + - name: Compress .image for transport + run: | + set -eux + cd /tmp/uvm-out + # The raw .image is ~250 MiB; zstd brings it under 100 MiB which + # makes oras push fast on cold registries. + zstd -19 --long -T0 --rm kata-containers-nvidia-gpu-confidential.img \ + -o kata-containers-nvidia-gpu-confidential.img.zst + ls -lh + + - name: Login to GHCR + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push artifact to GHCR + id: push + env: + OCI_TAG: ${{ needs.meta.outputs.tag }} + ROOT_HASH: ${{ steps.measure.outputs.root_hash }} + IMG_SHA256: ${{ steps.measure.outputs.img_sha256 }} + DRIVER_VER: ${{ steps.measure.outputs.driver_ver }} + NVAT_VER: ${{ steps.measure.outputs.nvat_ver }} + run: | + set -eux + OCI_REF="${OCI_IMAGE}:${OCI_TAG}" + cd /tmp/uvm-out + # NVAT annotation is conditional: an empty string would push a + # value-less label which is misleading. + NVAT_ANNOTATION=() + [[ -n "$NVAT_VER" ]] && NVAT_ANNOTATION+=(--annotation "com.cohere.kata-uvm.nvat-sdk=${NVAT_VER}") + + # The paired kernel (and its sibling artifacts) MUST ride along + # with the rootfs in the same OCI artifact so 05-install-uvm.sh + # can install both atomically. See README "Bug F" for the full + # mechanism. We push the kernel uncompressed (~80 MiB raw); zstd + # would only shave ~20 MiB and complicates the install side. + KERNEL_FILES=() + if [[ -f kernel.basename ]]; then + KBASENAME=$(cat kernel.basename) + KVER_VERSION="${KBASENAME#vmlinuz-}" + for kf in "$KBASENAME" "kernel.basename" \ + "vmlinux-${KVER_VERSION}" \ + "System.map-${KVER_VERSION}" \ + "config-${KVER_VERSION}"; do + if [[ -f "$kf" ]]; then + KERNEL_FILES+=( "${kf}:application/vnd.cohere.kata-uvm.kernel+octet-stream" ) + fi + done + fi + + oras push "$OCI_REF" \ + kata-containers-nvidia-gpu-confidential.img.zst:application/vnd.cohere.kata-uvm.image+zstd \ + root_hash.txt:application/vnd.cohere.kata-uvm.verity+plain \ + measurements.json:application/vnd.cohere.kata-uvm.measurements+json \ + "${KERNEL_FILES[@]}" \ + --annotation "org.opencontainers.image.title=kata-uvm-nvidia-gpu-confidential" \ + --annotation "org.opencontainers.image.description=Kata Containers NVIDIA GPU confidential UVM image, built from source with cohere-ai/guest-components" \ + --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --annotation "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --annotation "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --annotation "com.cohere.caa.commit=${GITHUB_SHA}" \ + --annotation "com.cohere.kata.ref=${{ needs.meta.outputs.kata_ref }}" \ + --annotation "com.cohere.guest-components.repo=${{ needs.meta.outputs.gc_repo }}" \ + --annotation "com.cohere.guest-components.ref=${{ needs.meta.outputs.gc_ref }}" \ + --annotation "com.cohere.kata-uvm.nvidia-driver=${DRIVER_VER}" \ + "${NVAT_ANNOTATION[@]}" \ + --annotation "com.cohere.kata-uvm.image-sha256=${IMG_SHA256}" \ + --annotation "com.cohere.kata-uvm.root-hash=${ROOT_HASH}" \ + --annotation "com.cohere.kata-uvm.kernel-basename=${{ steps.measure.outputs.kernel_basename }}" \ + --annotation "com.cohere.kata-uvm.kernel-sha256=${{ steps.measure.outputs.kernel_sha }}" \ + --format json > oras-output.json + + cat oras-output.json + DIGEST=$(jq -r '.digest' oras-output.json) + { + echo "digest=$DIGEST" + echo "oci_ref=${OCI_REF}@${DIGEST}" + echo "oci_tag=$OCI_TAG" + } >> "$GITHUB_OUTPUT" + echo "Pushed: $OCI_REF @ $DIGEST" + + - name: Attest build provenance + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4 + with: + subject-name: ${{ env.OCI_IMAGE }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + + - name: Job summary + run: | + { + echo "### Kata UVM image built" + echo "" + echo "| Field | Value |" + echo "| --- | --- |" + echo "| OCI ref | \`${OCI_IMAGE}:${{ needs.meta.outputs.tag }}\` |" + echo "| Digest | \`${{ steps.push.outputs.digest }}\` |" + echo "| kata-containers ref | \`${{ needs.meta.outputs.kata_ref }}\` |" + echo "| guest-components | \`${{ needs.meta.outputs.gc_repo }}@${{ needs.meta.outputs.gc_ref }}\` |" + echo "| NVIDIA driver | \`${{ steps.measure.outputs.driver_ver }}\` |" + echo "| NVAT SDK | \`${{ steps.measure.outputs.nvat_ver || '(unset — attestation-agent-nv NOT built)' }}\` |" + echo "| NVIDIA stack | \`${{ needs.meta.outputs.nvidia_gpu_stack }}\` |" + echo "| root_hash | \`${{ steps.measure.outputs.root_hash }}\` |" + echo "| image sha256 | \`${{ steps.measure.outputs.img_sha256 }}\` |" + echo "| kernel | \`${{ steps.measure.outputs.kernel_basename }}\` |" + echo "| kernel sha256 | \`${{ steps.measure.outputs.kernel_sha }}\` |" + echo "" + echo "Install on a B200 host with:" + echo "" + echo '```bash' + echo "ORAS_REF=${OCI_IMAGE}:${{ needs.meta.outputs.tag }} \\" + echo " bash fortress/scratch/oci-b200/k8s/05-install-uvm.sh" + echo '```' + } >> "$GITHUB_STEP_SUMMARY"