From babe619fd7eb33927245b9242a43527d7b4b1d5f Mon Sep 17 00:00:00 2001 From: UnArbosFive Date: Mon, 20 Jul 2026 14:13:57 +0200 Subject: [PATCH] ci: cut CI time with trusted routing and Fireactions caches Reduce CI wall time while retaining fail-closed test ownership and required check reporting. - route Rust, runtime, Docker, clone, Rust SDK, and TypeScript coverage from trusted change classifiers - share production builds and snapshots across consumers - compile the Rust SDK E2E harness once and execute its verified 112-test manifest in balanced shards - shard TypeScript E2Es with explicit suite ownership and actionable validation failures - reuse immutable localnet artifacts where the PR cannot affect chain behavior - add bounded R2-backed compiler and artifact caching with idle-host proactive warming - keep assertion-bearing clone coverage required while making the reporting-only deprecated-stake histogram manual --- .../build-production-binary/action.yml | 27 +- .github/actions/run-typescript-e2e/action.yml | 24 +- .github/actions/rust-setup/action.yml | 43 +- .github/actions/sccache-setup/action.yml | 5 + .github/actions/warm-native-cache/action.yml | 81 +++ .github/rust-ci-paths.txt | 13 + .github/scripts/benchmark-artifact-cache.sh | 49 ++ .github/scripts/benchmark-sccache-paired.sh | 87 +++ .github/scripts/build-bittensor-e2e-matrix.py | 65 ++ .../scripts/classify-bittensor-e2e-changes.sh | 101 +++ .github/scripts/classify-runtime-changes.sh | 31 +- .github/scripts/classify-rust-changes.sh | 51 ++ .../classify-typescript-e2e-changes.sh | 126 ++++ .github/scripts/download-artifact.sh | 148 +++++ .github/scripts/extract-pull-file-paths.sh | 39 ++ .github/scripts/install-rust-toolchain.sh | 30 + .github/scripts/prewarm-exact-runtime.sh | 49 ++ .github/scripts/publish-artifact-mirror.sh | 41 ++ .../publish-current-run-artifact-mirror.sh | 37 ++ .github/scripts/r2-artifact-mirror.py | 221 +++++++ .github/scripts/r2-sccache-warmset.py | 366 +++++++++++ .github/scripts/rust-setup-preflight.sh | 110 ++++ .github/scripts/sccache-config.py | 602 ++++++++++++++++++ .github/scripts/sccache-configure.sh | 413 +----------- .github/scripts/sccache-report.sh | 63 ++ .../scripts/select-shared-release-artifact.sh | 106 +++ .github/scripts/snapshot-artifact.sh | 4 + .../test-classify-bittensor-e2e-changes.sh | 155 +++++ .../test-classify-typescript-e2e-changes.sh | 272 ++++++++ .../scripts/test-clone-regression-phase.sh | 100 +++ .github/scripts/test-download-artifact.sh | 111 ++++ .github/scripts/test-prewarm-exact-runtime.sh | 55 ++ .github/scripts/test-r2-artifact-mirror.py | 300 +++++++++ .github/scripts/test-r2-sccache-warmset.py | 186 ++++++ .github/scripts/test-runtime-change-filter.sh | 53 +- .github/scripts/test-rust-ci-paths.sh | 72 +++ .github/scripts/test-rust-setup-preflight.sh | 162 +++++ .github/scripts/test-sccache-configure.sh | 80 ++- .github/scripts/test-sccache-report.sh | 59 ++ .../test-select-shared-release-artifact.sh | 73 +++ .github/scripts/test-snapshot-artifact.sh | 2 + .github/scripts/validate-rust-ci-paths.sh | 81 +++ .github/workflows/cargo-audit.yml | 19 +- .../workflows/check-bittensor-e2e-tests.yml | 495 ++++++++------ .github/workflows/check-docker.yml | 85 ++- .github/workflows/check-rust.yml | 152 ++++- .github/workflows/eco-tests.yml | 84 ++- .../workflows/refresh-mainnet-snapshot.yml | 66 +- .github/workflows/runtime-checks.yml | 340 ++++++---- .github/workflows/sccache-warm.yml | 152 +++-- .github/workflows/typescript-e2e.yml | 350 ++++++++-- .github/workflows/validate-sccache.yml | 258 +++++++- .../js-tests/scripts/run-clone-regressions.ts | 9 +- clones/scripts/run-clone-regression-phase.sh | 95 +++ ts-tests/e2e-shards.json | 90 +++ ts-tests/e2e-suite-ownership.json | 40 ++ ts-tests/moonwall.config.json | 241 +++---- ts-tests/scripts/e2e-shard-plan.mjs | 268 ++++++++ ts-tests/scripts/extract-runtime-wasm.mjs | 34 + .../scripts/generate-types-from-chain-spec.sh | 8 + ts-tests/scripts/generate-types.sh | 47 +- ts-tests/scripts/test-e2e-shard-plan.mjs | 119 ++++ ts-tests/scripts/validate-e2e-config.mjs | 262 +++++++- .../zombienet_shield/00.01-basic.test.ts | 72 ++- .../zombienet_shield/01-scaling.test.ts | 36 +- .../zombienet_shield/02-edge-cases.test.ts | 21 +- .../suites/zombienet_shield/03-timing.test.ts | 52 +- .../zombienet_shield/04-mortality.test.ts | 59 +- ts-tests/utils/shield_helpers.ts | 22 +- 69 files changed, 6989 insertions(+), 1180 deletions(-) create mode 100644 .github/actions/warm-native-cache/action.yml create mode 100644 .github/rust-ci-paths.txt create mode 100755 .github/scripts/benchmark-artifact-cache.sh create mode 100755 .github/scripts/benchmark-sccache-paired.sh create mode 100755 .github/scripts/build-bittensor-e2e-matrix.py create mode 100755 .github/scripts/classify-bittensor-e2e-changes.sh create mode 100755 .github/scripts/classify-rust-changes.sh create mode 100755 .github/scripts/classify-typescript-e2e-changes.sh create mode 100755 .github/scripts/download-artifact.sh create mode 100755 .github/scripts/extract-pull-file-paths.sh create mode 100755 .github/scripts/install-rust-toolchain.sh create mode 100755 .github/scripts/prewarm-exact-runtime.sh create mode 100755 .github/scripts/publish-artifact-mirror.sh create mode 100755 .github/scripts/publish-current-run-artifact-mirror.sh create mode 100755 .github/scripts/r2-artifact-mirror.py create mode 100755 .github/scripts/r2-sccache-warmset.py create mode 100755 .github/scripts/rust-setup-preflight.sh create mode 100755 .github/scripts/sccache-config.py create mode 100755 .github/scripts/sccache-report.sh create mode 100755 .github/scripts/select-shared-release-artifact.sh create mode 100755 .github/scripts/test-classify-bittensor-e2e-changes.sh create mode 100755 .github/scripts/test-classify-typescript-e2e-changes.sh create mode 100755 .github/scripts/test-clone-regression-phase.sh create mode 100755 .github/scripts/test-download-artifact.sh create mode 100755 .github/scripts/test-prewarm-exact-runtime.sh create mode 100755 .github/scripts/test-r2-artifact-mirror.py create mode 100755 .github/scripts/test-r2-sccache-warmset.py create mode 100755 .github/scripts/test-rust-ci-paths.sh create mode 100755 .github/scripts/test-rust-setup-preflight.sh create mode 100755 .github/scripts/test-sccache-report.sh create mode 100755 .github/scripts/test-select-shared-release-artifact.sh create mode 100755 .github/scripts/validate-rust-ci-paths.sh create mode 100755 clones/scripts/run-clone-regression-phase.sh create mode 100644 ts-tests/e2e-shards.json create mode 100644 ts-tests/e2e-suite-ownership.json create mode 100755 ts-tests/scripts/e2e-shard-plan.mjs create mode 100644 ts-tests/scripts/extract-runtime-wasm.mjs create mode 100755 ts-tests/scripts/generate-types-from-chain-spec.sh create mode 100644 ts-tests/scripts/test-e2e-shard-plan.mjs diff --git a/.github/actions/build-production-binary/action.yml b/.github/actions/build-production-binary/action.yml index 78947a1a40..f49fbac3d6 100644 --- a/.github/actions/build-production-binary/action.yml +++ b/.github/actions/build-production-binary/action.yml @@ -20,6 +20,9 @@ inputs: sccache-writer-secret-access-key: description: "Protected writer secret" default: "" + sccache-local-tier: + description: "auto for host-local reads, or disabled for direct R2 maintenance" + default: "auto" runs: using: "composite" @@ -33,6 +36,7 @@ runs: sccache-credential-mode: ${{ inputs.sccache-credential-mode }} sccache-writer-access-key-id: ${{ inputs.sccache-writer-access-key-id }} sccache-writer-secret-access-key: ${{ inputs.sccache-writer-secret-access-key }} + sccache-local-tier: ${{ inputs.sccache-local-tier }} - name: Require protected writer activation if: inputs.sccache-credential-mode == 'writer' @@ -49,10 +53,20 @@ runs: if: inputs.arch == 'arm64' shell: bash run: | - cargo install cross \ - --git https://github.com/cross-rs/cross \ - --rev 64b5bb4d3d34de062552b9a2093affe77b4ad16a \ - --locked + contract=/etc/fireactions-runner-image/image-contract.env + revision=64b5bb4d3d34de062552b9a2093affe77b4ad16a + system_bin=/usr/local/bin/cross + if [[ -r "$contract" ]] && \ + grep -Fxq "CROSS_SOURCE_REVISION=$revision" "$contract" && \ + [[ -x "$system_bin" ]] && "$system_bin" --version; then + install -m 0755 "$system_bin" "$HOME/.cargo/bin/cross" + echo "Using preinstalled cross from $revision" + else + cargo install cross \ + --git https://github.com/cross-rs/cross \ + --rev "$revision" \ + --locked + fi - name: Build production binary shell: bash @@ -108,3 +122,8 @@ runs: install -Dm0755 "$binary" "build/ci_target/${ARCH}/node-subtensor" echo "Staged $ARCH binary: $(du -h "build/ci_target/${ARCH}/node-subtensor" | cut -f1)" + + - name: Report production compiler cache + if: always() + shell: bash + run: .github/scripts/sccache-report.sh "Production ${{ inputs.arch }} compiler cache" diff --git a/.github/actions/run-typescript-e2e/action.yml b/.github/actions/run-typescript-e2e/action.yml index b79a3d3854..26f8ebe28d 100644 --- a/.github/actions/run-typescript-e2e/action.yml +++ b/.github/actions/run-typescript-e2e/action.yml @@ -1,5 +1,5 @@ name: Run TypeScript E2E suite -description: Download a prebuilt node and run one Moonwall environment. +description: Download a prebuilt node and run one or two isolated Moonwall environments. inputs: binary: @@ -8,6 +8,10 @@ inputs: test: description: Moonwall environment to run. required: true + additional-test: + description: Optional second Moonwall environment to run after the first one exits. + required: false + default: "" runs: using: composite @@ -41,12 +45,24 @@ runs: if: inputs.test == 'dev' shell: bash run: | - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends lsof + if ! command -v lsof >/dev/null 2>&1; then + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends lsof + fi - name: Run tests shell: bash working-directory: ts-tests env: MOONWALL_TEST: ${{ inputs.test }} - run: pnpm moonwall test "$MOONWALL_TEST" + MOONWALL_ADDITIONAL_TEST: ${{ inputs.additional-test }} + run: | + # The checked-in Moonwall file contains only canonical environments. + # Generate shard environments from the data manifest in this disposable + # checkout so local and CI topology cannot drift. + node scripts/e2e-shard-plan.mjs materialize moonwall.config.json e2e-shards.json + pnpm moonwall test "$MOONWALL_TEST" + if [[ -n "$MOONWALL_ADDITIONAL_TEST" ]]; then + echo "Starting isolated follow-up environment: $MOONWALL_ADDITIONAL_TEST" + pnpm moonwall test "$MOONWALL_ADDITIONAL_TEST" + fi diff --git a/.github/actions/rust-setup/action.yml b/.github/actions/rust-setup/action.yml index ef5bc2b5b2..d3988bf2e2 100644 --- a/.github/actions/rust-setup/action.yml +++ b/.github/actions/rust-setup/action.yml @@ -1,6 +1,6 @@ name: "Rust setup" description: >- - Common prologue for Rust jobs: system dependencies, stable toolchain, + Common prologue for Rust jobs: system dependencies, repository toolchain, shared R2 sccache, and Cargo registry/git caching. inputs: @@ -25,11 +25,22 @@ inputs: sccache-writer-secret-access-key: description: "Protected writer secret; used only for an authorized source" default: "" + sccache-local-tier: + description: "auto to use a validated Fireactions host cache, or disabled for direct R2" + default: "auto" runs: using: "composite" steps: + - name: Detect preprovisioned Fireactions toolchain + id: runner-image + shell: bash + env: + RUST_SETUP_COMPONENTS: ${{ inputs.components }} + run: .github/scripts/rust-setup-preflight.sh "$GITHUB_OUTPUT" + - name: Install system dependencies + if: steps.runner-image.outputs.system_ready != 'true' shell: bash run: | sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update @@ -39,10 +50,27 @@ runs: python3 python3-dev \ ${{ inputs.extra-packages }} - - name: Install Rust (stable) + - name: Install extra system dependencies + if: steps.runner-image.outputs.system_ready == 'true' && inputs.extra-packages != '' + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends \ + -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ + ${{ inputs.extra-packages }} + + - name: Bootstrap rustup when absent + if: >- + steps.runner-image.outputs.toolchain_ready != 'true' && + steps.runner-image.outputs.rustup_ready != 'true' uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - with: - components: ${{ inputs.components }} + + - name: Install repository Rust toolchain on image miss + if: steps.runner-image.outputs.toolchain_ready != 'true' + shell: bash + env: + RUST_SETUP_COMPONENTS: ${{ inputs.components }} + run: .github/scripts/install-rust-toolchain.sh "$RUST_SETUP_COMPONENTS" - name: Fast linker if: inputs.fast-linker == 'true' @@ -52,7 +80,11 @@ runs: # links dozens of test binaries. Guarded so a runner image without # the mold package falls back to the default linker instead of # failing every Rust job at setup. - if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends mold; then + # The production runner image already carries mold. Avoid consulting + # apt (whose package lists are intentionally removed from the image) + # when the linker is ready, while retaining the old-runner fallback. + if command -v mold >/dev/null 2>&1 || \ + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends mold; then mkdir -p "$HOME/.cargo" cat >> "$HOME/.cargo/config.toml" <<'EOF' [target.x86_64-unknown-linux-gnu] @@ -89,5 +121,6 @@ runs: uses: ./.github/actions/sccache-setup with: credential-mode: ${{ inputs.sccache-credential-mode }} + local-tier: ${{ inputs.sccache-local-tier }} writer-access-key-id: ${{ inputs.sccache-writer-access-key-id }} writer-secret-access-key: ${{ inputs.sccache-writer-secret-access-key }} diff --git a/.github/actions/sccache-setup/action.yml b/.github/actions/sccache-setup/action.yml index 19913fa95f..afa103ab4f 100644 --- a/.github/actions/sccache-setup/action.yml +++ b/.github/actions/sccache-setup/action.yml @@ -11,6 +11,9 @@ inputs: writer-secret-access-key: description: "Protected Environment writer secret; used only for an authorized source" default: "" + local-tier: + description: "auto to use validated Fireactions host cache metadata, or disabled for direct R2" + default: "auto" outputs: enabled: @@ -25,6 +28,7 @@ runs: shell: bash env: SCCACHE_CREDENTIAL_MODE: ${{ inputs.credential-mode }} + SCCACHE_LOCAL_TIER_MODE: ${{ inputs.local-tier }} AWS_ACCESS_KEY_ID: ${{ inputs.writer-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ inputs.writer-secret-access-key }} run: | @@ -49,6 +53,7 @@ runs: env: SCCACHE_CONFIG_FILE: ${{ steps.prepare.outputs.config-file }} SCCACHE_INSTALL_OUTCOME: ${{ steps.install.outcome }} + SCCACHE_LOCAL_TIER_MODE: ${{ inputs.local-tier }} run: | "$GITHUB_ACTION_PATH/../../scripts/sccache-configure.sh" activate \ "$SCCACHE_CONFIG_FILE" \ diff --git a/.github/actions/warm-native-cache/action.yml b/.github/actions/warm-native-cache/action.yml new file mode 100644 index 0000000000..f0e4535830 --- /dev/null +++ b/.github/actions/warm-native-cache/action.yml @@ -0,0 +1,81 @@ +name: "Warm native Rust compiler cache" +description: "Exercise the native Rust workloads whose compiler objects are reused by CI" + +runs: + using: "composite" + steps: + - name: Warm default workspace check artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: cargo check --workspace --locked + + - name: Warm all-feature workspace check artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: cargo check --workspace --all-features --locked + + - name: Warm default clippy artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: cargo clippy --workspace --all-targets --locked -- -D warnings + + - name: Warm all-feature clippy artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + + - name: Warm deny-warnings check artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: RUSTFLAGS="-D warnings" cargo check --locked + + - name: Warm workspace test artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: cargo test --workspace --all-features --locked --no-run + + - name: Warm eco-tests artifacts + shell: bash + working-directory: eco-tests + run: cargo test --no-run + + - name: Install runtime wasm toolchain components + shell: bash + run: | + rustup target add wasm32-unknown-unknown + rustup component add rust-src + + - name: Warm try-runtime wasm artifacts + shell: bash + run: | + unset SKIP_WASM_BUILD + cargo build --profile production -p node-subtensor-runtime --features try-runtime -q --locked + + - name: Warm release node artifacts + shell: bash + run: | + unset SKIP_WASM_BUILD + cargo build --release --locked -p node-subtensor + + - name: Warm fast-runtime release node artifacts + shell: bash + run: | + unset SKIP_WASM_BUILD + cargo build --release --locked -p node-subtensor --features fast-runtime + + # Package selection changes Cargo feature unification and therefore the + # rustc/sccache keys. Clean first so Cargo cannot satisfy this variant from + # the job-local target directory without exercising sccache. + - name: Warm runtime-only check artifacts + shell: bash + env: + SKIP_WASM_BUILD: "1" + run: | + cargo clean + cargo check --locked -p node-subtensor-runtime diff --git a/.github/rust-ci-paths.txt b/.github/rust-ci-paths.txt new file mode 100644 index 0000000000..68927aceb6 --- /dev/null +++ b/.github/rust-ci-paths.txt @@ -0,0 +1,13 @@ +chain-extensions +common +node +pallets +precompiles +primitives +runtime +sdk/bittensor-core +sdk/bittensor-core-py +sdk/bittensor-core-wasm +src +support +vendor diff --git a/.github/scripts/benchmark-artifact-cache.sh b/.github/scripts/benchmark-artifact-cache.sh new file mode 100755 index 0000000000..d0b9dcbf80 --- /dev/null +++ b/.github/scripts/benchmark-artifact-cache.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${RUNNER_TEMP:?RUNNER_TEMP must be set}" +: "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY must be set}" +: "${ARTIFACT_ID:?ARTIFACT_ID must be set}" +: "${ARTIFACT_DIGEST:?ARTIFACT_DIGEST must be set}" +: "${ARTIFACT_SIZE:?ARTIFACT_SIZE must be set}" + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +benchmark_dir="$RUNNER_TEMP/artifact-cache-benchmark" +direct_output="$benchmark_dir/direct-output" +first_output="$benchmark_dir/first-output" +second_output="$benchmark_dir/second-output" +rm -rf "$benchmark_dir" +mkdir -p "$benchmark_dir" +: > "$direct_output" +: > "$first_output" +: > "$second_output" + +FIREACTIONS_ARTIFACT_CACHE_DISABLE=true \ + "$script_dir/download-artifact.sh" "$ARTIFACT_ID" mainnet-snapshot \ + "$ARTIFACT_DIGEST" "$ARTIFACT_SIZE" "$benchmark_dir/direct" "$direct_output" +"$script_dir/download-artifact.sh" "$ARTIFACT_ID" mainnet-snapshot \ + "$ARTIFACT_DIGEST" "$ARTIFACT_SIZE" "$benchmark_dir/local-first" "$first_output" +"$script_dir/download-artifact.sh" "$ARTIFACT_ID" mainnet-snapshot \ + "$ARTIFACT_DIGEST" "$ARTIFACT_SIZE" "$benchmark_dir/local-second" "$second_output" + +direct_sha=$(sha256sum "$benchmark_dir/direct/mainnet-snapshot.tar.gz" | awk '{print $1}') +first_sha=$(sha256sum "$benchmark_dir/local-first/mainnet-snapshot.tar.gz" | awk '{print $1}') +second_sha=$(sha256sum "$benchmark_dir/local-second/mainnet-snapshot.tar.gz" | awk '{print $1}') +[[ "$direct_sha" == "$first_sha" && "$direct_sha" == "$second_sha" ]] + +direct_seconds=$(sed -n 's/^seconds=//p' "$direct_output") +first_seconds=$(sed -n 's/^seconds=//p' "$first_output") +second_seconds=$(sed -n 's/^seconds=//p' "$second_output") +first_source=$(sed -n 's/^source=//p' "$first_output") +second_source=$(sed -n 's/^source=//p' "$second_output") +[[ "$second_source" == local-hit ]] + +{ + echo "### Mainnet snapshot artifact cache" + echo "- Artifact: $ARTIFACT_ID ($ARTIFACT_SIZE bytes)" + echo "- Direct GitHub: ${direct_seconds}s" + echo "- Cache-only probe 1: ${first_seconds}s ($first_source)" + echo "- Cache-only probe 2: ${second_seconds}s ($second_source)" + echo "- Extracted payload SHA-256: $second_sha" +} >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/scripts/benchmark-sccache-paired.sh b/.github/scripts/benchmark-sccache-paired.sh new file mode 100755 index 0000000000..7d19189626 --- /dev/null +++ b/.github/scripts/benchmark-sccache-paired.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${RUNNER_TEMP:?RUNNER_TEMP must be set}" +: "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY must be set}" +: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID must be set}" +: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY must be set}" + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +config="$RUNNER_TEMP/sccache-local-contract.json" +output="$RUNNER_TEMP/sccache-local-contract.output" +times="$RUNNER_TEMP/sccache-paired-times" +trap 'rm -f "$config" "$output"' EXIT +: > "$output" + +SCCACHE_GHA_FALLBACK=false SCCACHE_LOCAL_TIER_MODE=auto \ + "$script_dir/sccache-configure.sh" prepare reader "$config" "$output" +python3 -c ' +import json, sys +data = json.load(open(sys.argv[1], encoding="utf-8")) +local = data.get("local") +if not isinstance(local, dict) or local.get("endpoint") != "http://192.168.128.1:8092": + raise SystemExit("validated host-local tier is unavailable") +' "$config" + +start_backend() { + local mode="$1" + sccache --stop-server >/dev/null 2>&1 || true + if [[ "$mode" == local ]]; then + export SCCACHE_MULTILEVEL_CHAIN=webdav,s3 + export SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=ignore + export SCCACHE_WEBDAV_ENDPOINT=http://192.168.128.1:8092 + export SCCACHE_WEBDAV_KEY_PREFIX="" + export SCCACHE_WEBDAV_USERNAME="$AWS_ACCESS_KEY_ID" + export SCCACHE_WEBDAV_PASSWORD="$AWS_SECRET_ACCESS_KEY" + else + unset SCCACHE_MULTILEVEL_CHAIN SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY + unset SCCACHE_WEBDAV_ENDPOINT SCCACHE_WEBDAV_KEY_PREFIX + unset SCCACHE_WEBDAV_USERNAME SCCACHE_WEBDAV_PASSWORD + fi + sccache --start-server >/dev/null +} + +measure() { + local label="$1" + local mode="$2" + local started seconds + + cargo clean + start_backend "$mode" + sccache --zero-stats >/dev/null + started=$(date -u +%s) + cargo check --locked -p node-subtensor-runtime + seconds=$(($(date -u +%s) - started)) + if ! sccache --show-adv-stats 2>&1 | tee "$RUNNER_TEMP/sccache-$label.txt"; then + sccache --show-stats | tee "$RUNNER_TEMP/sccache-$label.txt" + fi + echo "$label=$seconds" >> "$times" +} + +: > "$times" +# Discard the first compile so registry extraction, filesystem page cache, and +# daemon startup do not get attributed to either backend. +measure warmup origin +measure local_1 local +measure origin_1 origin +measure origin_2 origin +measure local_2 local + +source "$times" +origin_average=$(( (origin_1 + origin_2) / 2 )) +local_average=$(( (local_1 + local_2) / 2 )) +{ + echo "### Paired sccache benchmark (same VM)" + echo "- Discarded warmup: origin" + echo "- Measured order: local, origin, origin, local" + echo "- Direct R2: ${origin_1}s, ${origin_2}s (mean ${origin_average}s)" + echo "- Warm host-local: ${local_1}s, ${local_2}s (mean ${local_average}s)" + for label in warmup local_1 origin_1 origin_2 local_2; do + echo + echo "#### $label" + echo '```text' + cat "$RUNNER_TEMP/sccache-$label.txt" + echo '```' + done +} >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/scripts/build-bittensor-e2e-matrix.py b/.github/scripts/build-bittensor-e2e-matrix.py new file mode 100755 index 0000000000..19156f35b9 --- /dev/null +++ b/.github/scripts/build-bittensor-e2e-matrix.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Validate the Rust SDK E2E manifest and split every test into balanced shards.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +EXPECTED_TESTS = 112 +DEFAULT_SHARDS = 32 +TEST_NAME = re.compile(r"^(?:intent|test)_[A-Za-z0-9_]+$") + + +def main() -> int: + if len(sys.argv) not in (3, 4): + print(f"usage: {sys.argv[0]} MANIFEST OUTPUT_FILE [SHARDS]", file=sys.stderr) + return 2 + + manifest_path = Path(sys.argv[1]) + output_path = Path(sys.argv[2]) + shard_count = int(sys.argv[3]) if len(sys.argv) == 4 else DEFAULT_SHARDS + if shard_count < 1 or shard_count > EXPECTED_TESTS: + raise SystemExit(f"invalid shard count: {shard_count}") + + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(payload, list) or len(payload) != EXPECTED_TESTS: + raise SystemExit( + f"expected {EXPECTED_TESTS} Rust E2E manifest entries, got " + f"{len(payload) if isinstance(payload, list) else 'non-list'}" + ) + + names: list[str] = [] + for index, entry in enumerate(payload): + if not isinstance(entry, dict) or not isinstance(entry.get("test"), str): + raise SystemExit(f"manifest entry {index} has no string test name") + name = entry["test"] + if TEST_NAME.fullmatch(name) is None: + raise SystemExit(f"unsafe Rust E2E test name: {name!r}") + names.append(name) + + if len(set(names)) != len(names): + raise SystemExit("Rust E2E manifest contains duplicate test names") + + shards: list[list[str]] = [[] for _ in range(shard_count)] + for index, name in enumerate(names): + shards[index % shard_count].append(name) + + matrix = { + "include": [ + {"shard": index + 1, "tests": tests} + for index, tests in enumerate(shards) + if tests + ] + } + with output_path.open("a", encoding="utf-8") as output: + output.write(f"test_count={len(names)}\n") + output.write(f"shard_count={len(matrix['include'])}\n") + output.write(f"test_matrix={json.dumps(matrix, separators=(',', ':'))}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/classify-bittensor-e2e-changes.sh b/.github/scripts/classify-bittensor-e2e-changes.sh new file mode 100755 index 0000000000..e3c10157a5 --- /dev/null +++ b/.github/scripts/classify-bittensor-e2e-changes.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 [--all] OUTPUT_FILE" >&2 + exit 2 +} + +all=false +if [[ "${1:-}" == --all ]]; then + all=true + shift +fi +[[ $# -eq 1 ]] || usage +output_file=$1 + +e2e=false +build_image=false + +enable_all() { + e2e=true + build_image=true +} + +if [[ "$all" == true ]]; then + enable_all +else + while IFS= read -r path; do + [[ -n "$path" ]] || continue + case "$path" in + # Documentation and generated interface declarations are not loaded by + # the native Rust SDK harness or its localnet node. + *.md|LICENSE|docs/*|website/*|ts-tests/*|eco-tests/*|ink-contract/*|clones/*|.maintain/*|.vscode/*|.agents/*|.claude/*|precompiles/src/solidity/*) + ;; + + # These files cannot alter the node or SDK binary exercised here. Their + # owning unit/wasm/offline jobs still run, so starting 112 chain-facing + # tests adds latency without adding coverage. + pallets/*/src/tests/*|pallets/*/src/tests.rs|pallets/*/src/mock.rs|pallets/*/src/pallet/tests.rs|chain-extensions/src/tests.rs|chain-extensions/src/mock.rs|precompiles/src/mock.rs|node/tests/*|runtime/tests/*|support/*/tests/*|support/procedural-fork/src/pallet/parse/tests/*) + ;; + pallets/*/src/migrations/*) + # Fresh localnet genesis does not execute on-runtime-upgrade hooks. + # Cached try-runtime replays and clone-upgrade own migration coverage. + ;; + + # Benchmark-only modules are compiled by the all-features Rust checks, + # but the localnet deliberately does not enable runtime-benchmarks. + pallets/*/src/benchmarking.rs|pallets/subtensor/src/benchmarks.rs|pallets/subtensor/src/benchmarks/*|node/src/benchmarking.rs) + ;; + + # Repository maintenance binaries and lints are workspace-tested but do + # not participate in either the native SDK harness or node build. + support/linting/*|support/tools/*|support/weight-tools/*) + ;; + sdk/python/*|sdk/bittensor-core-py/*|sdk/bittensor-core-wasm/*) + # This workflow executes the native bittensor-core Rust harness. The + # Python and wasm bindings have dedicated compile/behavioral checks. + ;; + + # The harness/Dockerfile itself needs both newly compiled tests and a + # freshly built localnet image. + sdk/bittensor-core/tests/Dockerfile.localnet-fast) + enable_all + ;; + + # Native core changes need the full 112-test harness, but can safely run + # against the immutable image built from main when chain code is unchanged. + sdk/bittensor-core/*) + e2e=true + ;; + + # Production chain inputs require a localnet image built from the PR. + common/*|node/*|pallets/*|precompiles/*|primitives/*|runtime/*|support/*|chain-extensions/*|src/*|vendor/*|Cargo.toml|Cargo.lock|build.rs|rust-toolchain.toml|Dockerfile-localnet|snapshot.json|scripts/localnet.sh|scripts/localnet_patch.sh) + enable_all + ;; + + # CI plumbing changes are exercised fail-closed with the complete path. + .github/workflows/check-bittensor-e2e-tests.yml|.github/workflows/docker-localnet.yml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/rust-setup-preflight.sh|.github/scripts/install-rust-toolchain.sh|.github/scripts/sccache-configure.sh|.github/scripts/sccache-config.py|.github/scripts/sccache-report.sh|.github/scripts/classify-bittensor-e2e-changes.sh|.github/scripts/build-bittensor-e2e-matrix.py|.github/scripts/test-classify-bittensor-e2e-changes.sh|.github/scripts/extract-pull-file-paths.sh) + enable_all + ;; + + # A new SDK subtree may contain another chain-facing client. Prefer a + # complete run until its ownership is explicitly classified. + sdk/*) + enable_all + ;; + + # The workflow deliberately invokes this classifier for every PR. New + # top-level files and build inputs must get coverage until they are + # reviewed and placed in a known-safe exemption above. + *) + enable_all + ;; + esac + done +fi + +{ + echo "e2e=$e2e" + echo "build_image=$build_image" +} >> "$output_file" diff --git a/.github/scripts/classify-runtime-changes.sh b/.github/scripts/classify-runtime-changes.sh index 521142e17f..e383974927 100755 --- a/.github/scripts/classify-runtime-changes.sh +++ b/.github/scripts/classify-runtime-changes.sh @@ -15,28 +15,45 @@ python_sdk=false sdk_drift=false snapshot_ci=false -# Keep path ownership explicit. SDK-only changes are covered by sdk-checks and -# the Rust SDK e2e workflow; they should not force clone-upgrade or SDK drift. +# Known-safe surfaces stay explicitly exempt. Everything else fails closed to +# runtime + SDK-drift coverage so future production roots and Cargo inputs do +# not silently bypass try-runtime or clone-upgrade. while IFS= read -r path; do case "$path" in - common/*|node/*|pallets/*|precompiles/*|primitives/*|runtime/*|support/*|chain-extensions/*|src/*|vendor/*|Cargo.toml|build.rs|rust-toolchain.toml) + # Rust test modules are compiled and executed by Check Rust. They cannot + # change the production node or runtime wasm consumed by this workflow, so + # rebuilding and sudo-upgrading a mainnet clone adds no coverage. + pallets/*/src/tests/*|pallets/*/src/tests.rs|pallets/*/src/mock.rs|chain-extensions/src/tests.rs|chain-extensions/src/mock.rs|precompiles/src/mock.rs|node/tests/*|runtime/tests/*|support/*/tests/*|support/procedural-fork/src/pallet/parse/tests/*) + ;; + common/*|node/*|pallets/*|precompiles/*|primitives/*|runtime/*|support/*|chain-extensions/*|src/*|vendor/*|Cargo.toml|Cargo.lock|build.rs|rust-toolchain.toml|.cargo/*) runtime=true sdk_drift=true ;; clones/*|website/apps/bittensor-website/scripts/*) runtime=true ;; - .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/sccache-configure.sh) + .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/rust-setup-preflight.sh|.github/scripts/install-rust-toolchain.sh|.github/scripts/sccache-configure.sh|.github/scripts/sccache-config.py|.github/scripts/sccache-report.sh) runtime=true ;; - .github/scripts/classify-runtime-changes.sh|.github/scripts/test-runtime-change-filter.sh|.github/scripts/snapshot-artifact.sh|.github/scripts/test-snapshot-artifact.sh) + .github/scripts/classify-runtime-changes.sh|.github/scripts/test-runtime-change-filter.sh|.github/scripts/snapshot-artifact.sh|.github/scripts/test-snapshot-artifact.sh|.github/scripts/download-artifact.sh|.github/scripts/test-download-artifact.sh|.github/scripts/select-shared-release-artifact.sh|.github/scripts/test-select-shared-release-artifact.sh|.github/scripts/r2-artifact-mirror.py|.github/scripts/test-r2-artifact-mirror.py|.github/scripts/publish-artifact-mirror.sh|.github/scripts/publish-current-run-artifact-mirror.sh|.github/scripts/prewarm-exact-runtime.sh|.github/scripts/test-prewarm-exact-runtime.sh|.github/scripts/benchmark-sccache-paired.sh|.github/scripts/benchmark-artifact-cache.sh|.github/scripts/test-clone-regression-phase.sh|clones/scripts/run-clone-regression-phase.sh) runtime=true snapshot_ci=true ;; + *.md|LICENSE|docs/*|website/*|sdk/*|ts-tests/*|eco-tests/*|ink-contract/*|.maintain/*|.vscode/*|.agents/*|.claude/*) + ;; + .github/*) + runtime=true + sdk_drift=true + snapshot_ci=true + ;; + *) + runtime=true + sdk_drift=true + ;; esac case "$path" in - website/*|sdk/python/*|.github/workflows/runtime-checks.yml) docs=true ;; + docs/*|website/*|sdk/python/*|.github/workflows/runtime-checks.yml) docs=true ;; esac case "$path" in @@ -46,7 +63,7 @@ while IFS= read -r path; do esac case "$path" in - .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml) + .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|clones/scripts/run-clone-regression-phase.sh) snapshot_ci=true ;; esac diff --git a/.github/scripts/classify-rust-changes.sh b/.github/scripts/classify-rust-changes.sh new file mode 100755 index 0000000000..358cbe8e2a --- /dev/null +++ b/.github/scripts/classify-rust-changes.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 [--all] OUTPUT_FILE" >&2 + exit 2 +} + +all=false +if [[ "${1:-}" == --all ]]; then + all=true + shift +fi +[[ $# -eq 1 ]] || usage +output_file=$1 + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +registry=${RUST_CI_PATH_REGISTRY:-$script_dir/../rust-ci-paths.txt} +if [[ ! -s "$registry" ]] || grep -qEv '^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*$' "$registry"; then + echo "Rust CI path registry is missing or invalid: $registry" >&2 + exit 1 +fi +prefixes=() +while IFS= read -r prefix; do + prefixes+=("$prefix") +done < "$registry" + +rust=$all +if [[ "$all" != true ]]; then + while IFS= read -r path; do + [[ -n "$path" ]] || continue + for prefix in "${prefixes[@]}"; do + if [[ "$path" == "$prefix/"* ]]; then + rust=true + break + fi + done + [[ "$rust" != true ]] || continue + + case "$path" in + Cargo.toml|*/Cargo.toml|Cargo.lock|build.rs|rust-toolchain.toml|zepter.yaml|rustfmt.toml|clippy.toml|.cargo/*) + rust=true + ;; + .github/rust-ci-paths.txt|.github/workflows/check-rust.yml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/classify-rust-changes.sh|.github/scripts/test-rust-ci-paths.sh|.github/scripts/validate-rust-ci-paths.sh|.github/scripts/extract-pull-file-paths.sh|.github/scripts/rust-setup-preflight.sh|.github/scripts/install-rust-toolchain.sh|.github/scripts/sccache-configure.sh|.github/scripts/sccache-config.py|.github/scripts/sccache-report.sh) + rust=true + ;; + esac + done +fi + +echo "rust=$rust" >> "$output_file" diff --git a/.github/scripts/classify-typescript-e2e-changes.sh b/.github/scripts/classify-typescript-e2e-changes.sh new file mode 100755 index 0000000000..21e3b07c16 --- /dev/null +++ b/.github/scripts/classify-typescript-e2e-changes.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 [--all] OUTPUT_FILE" >&2 + exit 2 +} + +all=false +if [[ "${1:-}" == "--all" ]]; then + all=true + shift +fi +[[ $# -eq 1 ]] || usage +output_file=$1 + +evm=false +staking=false +coldkey_swap=false +dev=false +subnets=false +shield=false +topology_audit=false + +enable_all() { + evm=true + staking=true + coldkey_swap=true + dev=true + subnets=true + shield=true +} + +if [[ "$all" == true ]]; then + enable_all +else + while IFS= read -r path; do + [[ -n "$path" ]] || continue + case "$path" in + # These surfaces cannot alter the node binaries or TypeScript harness. + # Keep them explicit so the default below remains fail-closed. + *.md|LICENSE|docs/*|website/*|sdk/*|eco-tests/*|ink-contract/*|clones/*|.maintain/*|.vscode/*|.agents/*|.claude/*) + ;; + + # Tests that only change one suite do not need unrelated networks. + ts-tests/suites/zombienet_evm/*) + evm=true + ;; + ts-tests/suites/zombienet_staking/*) + staking=true + ;; + ts-tests/suites/zombienet_coldkey_swap/*) + coldkey_swap=true + ;; + ts-tests/suites/zombienet_subnets/*) + subnets=true + ;; + ts-tests/suites/zombienet_shield/*) + shield=true + ;; + ts-tests/suites/dev/*) + dev=true + ;; + + # Shard topology is proposed as data and expanded by a trusted generic + # builder. When that contract changes, run the canonical unsharded + # environments as an equivalence audit in addition to the fast shards. + ts-tests/e2e-shards.json|ts-tests/e2e-suite-ownership.json|ts-tests/moonwall.config.json|ts-tests/configs/zombie_single_node.json|ts-tests/configs/zombie_extended.json|ts-tests/scripts/e2e-shard-plan.mjs|ts-tests/scripts/test-e2e-shard-plan.mjs|ts-tests/scripts/validate-e2e-config.mjs|.github/workflows/typescript-e2e.yml|.github/actions/run-typescript-e2e/*|.github/scripts/classify-typescript-e2e-changes.sh|.github/scripts/test-classify-typescript-e2e-changes.sh) + enable_all + topology_audit=true + ;; + + # Rust unit-test-only edits cannot change the node exercised by E2E. + pallets/*/src/tests/*|pallets/*/src/tests.rs|pallets/*/src/mock.rs|chain-extensions/src/tests.rs|chain-extensions/src/mock.rs|precompiles/src/mock.rs|node/tests/*|runtime/tests/*|support/*/tests/*|support/procedural-fork/src/pallet/parse/tests/*) + ;; + + # Fresh-genesis E2E networks never execute on-runtime-upgrade hooks. + # Migration changes are covered by the cached try-runtime replays and + # the sudo-upgraded mainnet clone instead. + pallets/subtensor/src/migrations/*) + ;; + + # These isolated production areas have tightly owned E2E coverage. + precompiles/*) + evm=true + ;; + pallets/shield/*|pallets/limit-orders/*) + shield=true + dev=true + ;; + + # Shared test infrastructure and chain/runtime inputs can affect any + # suite. Unknown paths inside an E2E-relevant tree deliberately fall + # back to the complete matrix rather than guessing. In particular, + # subtensor staking, subnet, and swap code is exercised across several + # nominally separate suites, so all production changes there stay full. + ts-tests/*|common/*|node/*|pallets/*|primitives/*|runtime/*|support/*|chain-extensions/*|src/*|vendor/*|Cargo.toml|Cargo.lock|build.rs|rust-toolchain.toml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/rust-setup-preflight.sh|.github/scripts/install-rust-toolchain.sh|.github/scripts/sccache-configure.sh|.github/scripts/sccache-config.py|.github/scripts/sccache-report.sh|.github/scripts/extract-pull-file-paths.sh) + enable_all + ;; + + # Unknown files may be future build inputs. Full coverage is cheaper + # than silently teaching a required check to pass without exercising + # them; known-safe surfaces belong in the explicit exemption above. + *) + enable_all + ;; + esac + done +fi + +e2e=false +if [[ "$evm" == true || "$staking" == true || "$coldkey_swap" == true || + "$dev" == true || "$subnets" == true || "$shield" == true ]]; then + e2e=true +fi + +{ + echo "e2e=$e2e" + echo "evm=$evm" + echo "staking=$staking" + echo "coldkey_swap=$coldkey_swap" + echo "dev=$dev" + echo "subnets=$subnets" + echo "shield=$shield" + echo "topology_audit=$topology_audit" +} >> "$output_file" diff --git a/.github/scripts/download-artifact.sh b/.github/scripts/download-artifact.sh new file mode 100755 index 0000000000..4c250bb5e5 --- /dev/null +++ b/.github/scripts/download-artifact.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "usage: $0 ARTIFACT_ID ARTIFACT_NAME SHA256_DIGEST SIZE_BYTES DESTINATION OUTPUT_FILE" >&2 + exit 2 +} + +[[ $# -eq 6 ]] || usage +artifact_id="$1" +artifact_name="$2" +expected_digest="$3" +expected_size="$4" +destination="$5" +output_file="$6" + +: "${GH_TOKEN:?GH_TOKEN must contain a short-lived Actions token}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GITHUB_REPOSITORY_ID:?GITHUB_REPOSITORY_ID must be set}" + +[[ "$artifact_id" =~ ^[1-9][0-9]*$ ]] || usage +[[ "$expected_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || usage +[[ "$expected_size" =~ ^[1-9][0-9]*$ ]] || usage +[[ -n "$destination" && "$destination" != / ]] || usage +case "$artifact_name" in + mainnet-snapshot|try-runtime-snap-v0.10.1-mainnet|try-runtime-snap-v0.10.1-testnet|try-runtime-snap-v0.10.1-devnet) ;; + "node-subtensor-release-${GITHUB_SHA:-invalid}") + [[ "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || usage + ;; + *) echo "artifact is outside the host-cache allowlist: $artifact_name" >&2; exit 2 ;; +esac + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +archive="$tmp/artifact.zip" +headers="$tmp/headers" +metadata="$tmp/artifact-cache.json" +extract_dir="$tmp/extract" +mkdir -p "$extract_dir" +started=$(date -u +%s) +source=github + +set_output() { + printf '%s=%s\n' "$1" "$2" >> "$output_file" +} + +discover_cache_endpoint() { + [[ "${FIREACTIONS_ARTIFACT_CACHE_DISABLE:-false}" != true ]] || return 1 + local token_url="${MMDS_TOKEN_URL:-http://169.254.169.254/latest/api/token}" + local metadata_url="${ARTIFACT_CACHE_METADATA_URL:-http://169.254.169.254/latest/meta-data/artifact-cache}" + local token + token=$(curl --fail --silent --show-error --connect-timeout 1 --max-time 2 \ + --request PUT --header 'X-Metadata-Token-TTL-Seconds: 60' "$token_url" 2>/dev/null) || return 1 + [[ -n "$token" ]] || return 1 + curl --fail --silent --show-error --connect-timeout 1 --max-time 3 \ + --header "X-Metadata-Token: $token" --header 'Accept: application/json' \ + --output "$metadata" "$metadata_url" 2>/dev/null || return 1 + GITHUB_REPOSITORY_ID="$GITHUB_REPOSITORY_ID" python3 -c ' +import json, os, sys +with open(sys.argv[1], encoding="utf-8") as handle: + data = json.load(handle) +if set(data) != {"schema_version", "endpoint", "repository_id"}: + raise SystemExit(1) +if data.get("schema_version") != 1: + raise SystemExit(1) +if data.get("endpoint") != "http://192.168.128.1:8093": + raise SystemExit(1) +if str(data.get("repository_id")) != os.environ["GITHUB_REPOSITORY_ID"]: + raise SystemExit(1) +print(data["endpoint"], end="") +' "$metadata" 2>/dev/null +} + +cache_endpoint=$(discover_cache_endpoint || true) +if [[ -n "$cache_endpoint" ]]; then + # Artifact IDs are immutable and scheduled producers prefill the fleet. + # A compliant cache returns a stored response or a fast 504 instead of + # synchronously fetching a multi-gigabyte miss while the job waits. + if curl --fail --silent --show-error \ + --connect-timeout 5 --max-time 1800 \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header 'Cache-Control: only-if-cached' \ + --dump-header "$headers" --output "$archive" \ + "$cache_endpoint/v1/artifacts/$artifact_id"; then + cache_result=$(awk -F': *' 'tolower($1)=="x-fireactions-cache" {gsub("\\r", "", $2); print tolower($2)}' "$headers" | tail -1) + source="local-${cache_result:-unknown}" + else + rm -f "$archive" "$headers" + fi +fi + +if [[ ! -f "$archive" ]]; then + curl --fail --silent --show-error --location --retry 3 --retry-all-errors \ + --connect-timeout 10 --max-time 1800 \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + --output "$archive" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" + source=github +fi + +actual_size=$(stat -c '%s' "$archive" 2>/dev/null || stat -f '%z' "$archive") +actual_digest="sha256:$(sha256sum "$archive" | awk '{print $1}')" +if [[ "$actual_size" != "$expected_size" || "$actual_digest" != "$expected_digest" ]]; then + echo "artifact archive integrity check failed for $artifact_id" >&2 + exit 1 +fi + +python3 - "$archive" "$extract_dir" <<'PY' +import pathlib +import stat +import sys +import zipfile + +archive, destination = sys.argv[1:] +with zipfile.ZipFile(archive) as payload: + total = 0 + for member in payload.infolist(): + path = pathlib.PurePosixPath(member.filename) + mode = member.external_attr >> 16 + if ( + not member.filename + or member.filename.startswith(("/", "\\")) + or ".." in path.parts + or (path.parts and ":" in path.parts[0]) + or stat.S_ISLNK(mode) + ): + raise SystemExit("unsafe artifact archive path") + total += member.file_size + if total > 16 * 1024 * 1024 * 1024: + raise SystemExit("artifact archive exceeds extraction limit") + payload.extractall(destination) +PY + +if [[ -d "$destination" && -n "$(find "$destination" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + echo "artifact destination is not empty: $destination" >&2 + exit 1 +fi +mkdir -p "$destination" +cp -a "$extract_dir/." "$destination/" + +seconds=$(($(date -u +%s) - started)) +set_output source "$source" +set_output seconds "$seconds" +set_output artifact-id "$artifact_id" +echo "Downloaded $artifact_name artifact $artifact_id via $source in ${seconds}s." diff --git a/.github/scripts/extract-pull-file-paths.sh b/.github/scripts/extract-pull-file-paths.sh new file mode 100755 index 0000000000..e2571595b5 --- /dev/null +++ b/.github/scripts/extract-pull-file-paths.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 || ! "$1" =~ ^[0-9]+$ ]]; then + echo "usage: $0 EXPECTED_FILE_COUNT" >&2 + exit 2 +fi + +expected=$1 +payload=$(mktemp) +trap 'rm -f "$payload"' EXIT +cat > "$payload" + +observed=$(jq -es ' + if all(.[]; type == "array") + then (map(length) | add // 0) + else error("pull-file response contains a non-array page") + end +' "$payload") + +if [[ "$observed" != "$expected" ]]; then + echo "changed-file list incomplete ($observed/$expected)" >&2 + exit 1 +fi + +jq -ers ' + [ + .[] | .[] | + if type != "object" or (.filename | type) != "string" or .filename == "" + then error("pull-file entry has no filename") + elif has("previous_filename") and (.previous_filename | type) != "string" + then error("pull-file entry has an invalid previous_filename") + else .filename, (.previous_filename // empty) + end + | select(length > 0) + ] + | unique[] +' "$payload" diff --git a/.github/scripts/install-rust-toolchain.sh b/.github/scripts/install-rust-toolchain.sh new file mode 100755 index 0000000000..c7f04630f2 --- /dev/null +++ b/.github/scripts/install-rust-toolchain.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +extra_components="${1:-}" + +# With no explicit TOOLCHAIN argument rustup installs the active toolchain, +# including the channel, profile, components, and targets selected by the +# repository's rust-toolchain.toml. This is intentionally different from +# installing the moving `stable` alias: an exact version bump must be a slow +# cache miss, never a setup failure caused by installing the wrong compiler. +rustup toolchain install --no-self-update + +if [[ -n "${extra_components}" ]]; then + IFS=',' read -r -a components <<< "${extra_components}" + for component in "${components[@]}"; do + component="${component//[[:space:]]/}" + [[ -z "${component}" ]] && continue + if [[ ! "${component}" =~ ^[a-z0-9][a-z0-9._-]*$ ]]; then + echo "invalid Rust component: ${component}" >&2 + exit 2 + fi + rustup component add "${component}" + done +fi + +# Exercise the repository override now so a malformed or unavailable pinned +# toolchain fails at setup with a useful error, not halfway through a build. +cargo --version +rustc --version diff --git a/.github/scripts/prewarm-exact-runtime.sh b/.github/scripts/prewarm-exact-runtime.sh new file mode 100755 index 0000000000..e5a938cc4d --- /dev/null +++ b/.github/scripts/prewarm-exact-runtime.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${RUNNER_TEMP:?RUNNER_TEMP must be set}" +: "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY must be set}" + +[[ "${SCCACHE_ENABLED:-false}" == true ]] +[[ "${SCCACHE_BACKEND:-}" == r2 ]] +[[ "${SCCACHE_LOCAL_TIER:-false}" == false ]] + +results="$RUNNER_TEMP/exact-prewarm" +: > "$results" + +measure() { + local label="$1" + local started seconds stats + + cargo clean + sccache --zero-stats >/dev/null + started=$(date -u +%s) + cargo check --locked -p node-subtensor-runtime + seconds=$(($(date -u +%s) - started)) + stats=$(sccache --show-stats) + printf '%s\n' "$stats" | tee "$RUNNER_TEMP/sccache-$label.txt" + printf '%s=%s\n' "${label}_seconds" "$seconds" >> "$results" + printf '%s=%s\n' "${label}_rust_hits" \ + "$(awk '$1 == "Cache" && $2 == "hits" && $3 == "(Rust)" {count = $4} END {print count + 0}' <<< "$stats")" \ + >> "$results" + printf '%s=%s\n' "${label}_rust_misses" \ + "$(awk '$1 == "Cache" && $2 == "misses" && $3 == "(Rust)" {count = $4} END {print count + 0}' <<< "$stats")" \ + >> "$results" +} + +measure fill +measure verify +source "$results" + +[[ "$verify_rust_hits" =~ ^[0-9]+$ && "$verify_rust_misses" =~ ^[0-9]+$ ]] +if (( verify_rust_hits < 500 || verify_rust_misses > 10 )); then + echo "::error::exact-key verification produced ${verify_rust_hits} Rust hits and ${verify_rust_misses} misses" + exit 1 +fi + +{ + echo "### Exact runtime-only R2 prewarm" + echo "- Fill: ${fill_seconds}s (${fill_rust_hits} Rust hits, ${fill_rust_misses} misses)" + echo "- Clean verification: ${verify_seconds}s (${verify_rust_hits} Rust hits, ${verify_rust_misses} misses)" +} >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/scripts/publish-artifact-mirror.sh b/.github/scripts/publish-artifact-mirror.sh new file mode 100755 index 0000000000..b161a47504 --- /dev/null +++ b/.github/scripts/publish-artifact-mirror.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 5 || ! "$1" =~ ^[1-9][0-9]*$ ]]; then + echo "usage: $0 ARTIFACT_ID ARTIFACT_NAME SHA256_DIGEST PRODUCER_SHA WORKFLOW_PATH" >&2 + exit 2 +fi + +artifact_id="$1" +artifact_name="$2" +digest="$3" +producer_sha="$4" +workflow_path="$5" + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GH_TOKEN:?GH_TOKEN must contain an Actions token}" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +archive="$tmp/artifact.zip" + +for attempt in 1 2 3 4 5; do + if gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" \ + > "$archive"; then + break + fi + rm -f "$archive" + sleep $((attempt * 5)) +done +[[ -s "$archive" ]] + +"$(dirname "$0")/r2-artifact-mirror.py" \ + "$archive" \ + "$artifact_id" \ + "$artifact_name" \ + "$digest" \ + "$producer_sha" \ + "$workflow_path" diff --git a/.github/scripts/publish-current-run-artifact-mirror.sh b/.github/scripts/publish-current-run-artifact-mirror.sh new file mode 100755 index 0000000000..8111eeabd3 --- /dev/null +++ b/.github/scripts/publish-current-run-artifact-mirror.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 ARTIFACT_NAME" >&2 + exit 2 +fi + +artifact_name="$1" +: "${GH_TOKEN:?GH_TOKEN must contain an Actions token}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GITHUB_RUN_ID:?GITHUB_RUN_ID must be set}" +: "${GITHUB_SHA:?GITHUB_SHA must be set}" + +case "$artifact_name" in + mainnet-snapshot|try-runtime-snap-v0.10.1-mainnet|try-runtime-snap-v0.10.1-testnet|try-runtime-snap-v0.10.1-devnet) ;; + *) echo "artifact is outside the trusted mirror allowlist: $artifact_name" >&2; exit 2 ;; +esac + +metadata=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") +selection=$(jq -cer --arg name "$artifact_name" ' + [.artifacts[] | select(.name == $name and .expired == false)] + | if length == 1 then .[0] else error("expected exactly one current-run artifact") end +' <<< "$metadata") +artifact_id=$(jq -er '.id | select(type == "number" and . > 0)' <<< "$selection") +digest=$(jq -er '.digest | select(type == "string")' <<< "$selection") +[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] + +"$(dirname "$0")/publish-artifact-mirror.sh" \ + "$artifact_id" \ + "$artifact_name" \ + "$digest" \ + "$GITHUB_SHA" \ + .github/workflows/refresh-mainnet-snapshot.yml diff --git a/.github/scripts/r2-artifact-mirror.py b/.github/scripts/r2-artifact-mirror.py new file mode 100755 index 0000000000..911418ebfe --- /dev/null +++ b/.github/scripts/r2-artifact-mirror.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Validate and publish immutable Actions artifacts through the standard S3 CLI.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.parse import urlparse + + +ACCOUNT_HOST = "3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com" +SHA256_DIGEST = re.compile(r"^sha256:([0-9a-f]{64})$") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +BUCKET = "subtensor-ci-sccache" +REGION = "auto" +MIRROR_PREFIX = "artifacts/v1" +ALLOWED_ARTIFACT_NAMES = frozenset( + { + "mainnet-snapshot", + "try-runtime-snap-v0.10.1-mainnet", + "try-runtime-snap-v0.10.1-testnet", + "try-runtime-snap-v0.10.1-devnet", + } +) + + +class MirrorError(Exception): + """A safe error that never contains credentials.""" + + +def require_environment(name: str) -> str: + value = os.environ.get(name, "") + if not value or "\n" in value or "\r" in value: + raise MirrorError(f"missing or malformed {name}") + return value + + +def validate_endpoint(value: str) -> str: + parsed = urlparse(value) + try: + port = parsed.port + except ValueError: + raise MirrorError("SCCACHE_ENDPOINT is not the expected R2 origin") from None + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.hostname != ACCOUNT_HOST + or parsed.path not in ("", "/") + or parsed.params + or parsed.query + or parsed.fragment + or parsed.username + or parsed.password + or port not in (None, 443) + ): + raise MirrorError("SCCACHE_ENDPOINT is not the expected R2 origin") + return f"https://{parsed.hostname}" + + +def file_sha256(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +class R2Publisher: + def __init__(self) -> None: + if require_environment("SCCACHE_BUCKET") != BUCKET: + raise MirrorError("SCCACHE_BUCKET is not the trusted cache bucket") + if require_environment("SCCACHE_REGION") != REGION: + raise MirrorError("SCCACHE_REGION is not the expected R2 region") + self.endpoint = validate_endpoint(require_environment("SCCACHE_ENDPOINT")) + require_environment("AWS_ACCESS_KEY_ID") + require_environment("AWS_SECRET_ACCESS_KEY") + self.aws = shutil.which("aws") + if not self.aws: + raise MirrorError("the standard AWS CLI is unavailable") + + def put(self, key: str, source: Path) -> None: + environment = { + **os.environ, + "AWS_DEFAULT_REGION": REGION, + "AWS_REGION": REGION, + "AWS_RETRY_MODE": "standard", + "AWS_MAX_ATTEMPTS": "4", + "AWS_REQUEST_CHECKSUM_CALCULATION": "when_required", + "AWS_RESPONSE_CHECKSUM_VALIDATION": "when_required", + "AWS_PAGER": "", + } + try: + result = subprocess.run( + [ + self.aws, + "s3api", + "put-object", + "--endpoint-url", + self.endpoint, + "--region", + REGION, + "--bucket", + BUCKET, + "--key", + key, + "--body", + str(source), + "--content-type", + "application/octet-stream", + ], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=1800, + ) + except (OSError, subprocess.TimeoutExpired): + raise MirrorError("R2 object upload failed through the standard S3 client") from None + if result.returncode != 0: + raise MirrorError("R2 object upload failed through the standard S3 client") + + +def build_manifest(args: argparse.Namespace, object_key: str, size: int) -> bytes: + repository = require_environment("GITHUB_REPOSITORY") + repository_id = require_environment("GITHUB_REPOSITORY_ID") + if not repository_id.isdigit() or int(repository_id) <= 0: + raise MirrorError("GITHUB_REPOSITORY_ID is malformed") + return ( + json.dumps( + { + "schema_version": 1, + "repository": repository, + "repository_id": int(repository_id), + "workflow_path": args.workflow_path, + "artifact_id": args.artifact_id, + "artifact_name": args.artifact_name, + "digest": args.digest, + "size_in_bytes": size, + "object_key": object_key, + "producer_sha": args.producer_sha, + "published_at": int(time.time()), + }, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode() + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("archive", type=Path) + parser.add_argument("artifact_id", type=int) + parser.add_argument("artifact_name") + parser.add_argument("digest") + parser.add_argument("producer_sha") + parser.add_argument("workflow_path") + args = parser.parse_args(argv) + if args.artifact_id <= 0: + parser.error("artifact_id must be positive") + if args.artifact_name not in ALLOWED_ARTIFACT_NAMES: + parser.error("artifact_name is outside the trusted mirror allowlist") + if not SHA256_DIGEST.fullmatch(args.digest): + parser.error("digest must be a sha256 digest") + if not COMMIT_SHA.fullmatch(args.producer_sha): + parser.error("producer_sha must be a full commit SHA") + if args.workflow_path != ".github/workflows/refresh-mainnet-snapshot.yml": + parser.error("workflow_path is outside the trusted producer allowlist") + if not args.archive.is_file(): + parser.error("archive does not exist") + return args + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + expected_sha = SHA256_DIGEST.fullmatch(args.digest).group(1) + actual_sha, size = file_sha256(args.archive) + if actual_sha != expected_sha or size <= 0: + raise MirrorError("downloaded artifact archive failed integrity validation") + + object_key = f"{MIRROR_PREFIX}/objects/{args.artifact_id}-{actual_sha}.zip" + manifest = build_manifest(args, object_key, size) + publisher = R2Publisher() + publisher.put(object_key, args.archive) + + manifest_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + prefix="artifact-manifest-", suffix=".json", delete=False + ) as output: + output.write(manifest) + output.flush() + os.fsync(output.fileno()) + manifest_path = Path(output.name) + publisher.put(f"{MIRROR_PREFIX}/{args.artifact_name}/latest.json", manifest_path) + finally: + if manifest_path is not None: + manifest_path.unlink(missing_ok=True) + + print(f"Published immutable artifact {args.artifact_id} and latest manifest.") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except MirrorError as error: + print(f"artifact mirror failed: {error}", file=sys.stderr) + raise SystemExit(1) from None diff --git a/.github/scripts/r2-sccache-warmset.py b/.github/scripts/r2-sccache-warmset.py new file mode 100755 index 0000000000..576ac0d79f --- /dev/null +++ b/.github/scripts/r2-sccache-warmset.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Capture exact sccache keys and publish a bounded R2 host-warm manifest.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Iterable, Mapping +from urllib.parse import urlparse + + +ACCOUNT_HOST = "3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com" +BUCKET = "subtensor-ci-sccache" +REGION = "auto" +COMPILER_PREFIX = "subtensor/v1" +MANIFEST_PREFIX = "sccache-warmsets/v1" +MAX_WARM_BYTES = 4 * 1024 * 1024 * 1024 +MAX_OBJECTS = 50_000 +MAX_KEY_FILE_BYTES = 8 * 1024 * 1024 +MAX_INVENTORY_BYTES = 256 * 1024 * 1024 +MAX_MANIFEST_BYTES = 8 * 1024 * 1024 +MANIFEST_TTL_SECONDS = 36 * 60 * 60 +HASH = re.compile(r"^[0-9a-f]{64}$") +HASH_LOG = re.compile(r"Hash key: ([0-9a-f]{64})(?:\s|$)") +COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") + + +class WarmsetError(Exception): + """A safe publisher error that never contains credential material.""" + + +def require_environment(name: str) -> str: + value = os.environ.get(name, "") + if not value or "\n" in value or "\r" in value: + raise WarmsetError(f"missing or malformed {name}") + return value + + +def validate_endpoint(value: str) -> str: + parsed = urlparse(value) + try: + port = parsed.port + except ValueError: + raise WarmsetError("SCCACHE_ENDPOINT is not the expected R2 origin") from None + if ( + parsed.scheme != "https" + or parsed.hostname != ACCOUNT_HOST + or parsed.path not in ("", "/") + or parsed.params + or parsed.query + or parsed.fragment + or parsed.username + or parsed.password + or port not in (None, 443) + ): + raise WarmsetError("SCCACHE_ENDPOINT is not the expected R2 origin") + return f"https://{parsed.hostname}" + + +def normalized_path(cache_hash: str) -> str: + if not HASH.fullmatch(cache_hash): + raise WarmsetError("compiler cache key is malformed") + return f"{cache_hash[0]}/{cache_hash[1]}/{cache_hash[2]}/{cache_hash}" + + +def extract_hashes(log_path: Path, output_path: Path) -> int: + try: + if log_path.stat().st_size > 512 * 1024 * 1024: + raise WarmsetError("sccache diagnostic log exceeds the size limit") + ordered: dict[str, None] = {} + with log_path.open(encoding="utf-8", errors="replace") as source: + for line in source: + for cache_hash in HASH_LOG.findall(line): + ordered.setdefault(cache_hash, None) + except OSError: + raise WarmsetError("sccache diagnostic log is unavailable") from None + if not ordered: + raise WarmsetError("sccache diagnostic log contained no compiler cache keys") + + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary = output_path.with_name(output_path.name + ".tmp") + try: + with temporary.open("w", encoding="utf-8") as output: + for cache_hash in ordered: + output.write(cache_hash + "\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, output_path) + except OSError: + raise WarmsetError("compiler cache key output could not be written") from None + finally: + temporary.unlink(missing_ok=True) + return len(ordered) + + +def load_hashes(paths: Iterable[Path]) -> list[str]: + ordered: dict[str, None] = {} + for path in paths: + try: + if not path.is_file() or path.stat().st_size > MAX_KEY_FILE_BYTES: + raise WarmsetError( + "compiler cache key input is unavailable or oversized" + ) + with path.open(encoding="ascii") as source: + for line in source: + cache_hash = line.rstrip("\n") + if not HASH.fullmatch(cache_hash): + raise WarmsetError("compiler cache key input is malformed") + ordered.setdefault(cache_hash, None) + except (OSError, UnicodeError): + raise WarmsetError("compiler cache key input could not be read") from None + if not ordered: + raise WarmsetError("compiler cache key inputs were empty") + if len(ordered) > MAX_OBJECTS: + raise WarmsetError("compiler cache key inputs exceed the object limit") + return list(ordered) + + +class R2Client: + def __init__(self) -> None: + if require_environment("SCCACHE_BUCKET") != BUCKET: + raise WarmsetError("SCCACHE_BUCKET is not the trusted cache bucket") + if require_environment("SCCACHE_REGION") != REGION: + raise WarmsetError("SCCACHE_REGION is not the expected R2 region") + self.endpoint = validate_endpoint(require_environment("SCCACHE_ENDPOINT")) + require_environment("AWS_ACCESS_KEY_ID") + require_environment("AWS_SECRET_ACCESS_KEY") + self.aws = shutil.which("aws") + if not self.aws: + raise WarmsetError("the standard AWS CLI is unavailable") + + def environment(self) -> dict[str, str]: + return { + **os.environ, + "AWS_DEFAULT_REGION": REGION, + "AWS_REGION": REGION, + "AWS_RETRY_MODE": "standard", + "AWS_MAX_ATTEMPTS": "4", + "AWS_REQUEST_CHECKSUM_CALCULATION": "when_required", + "AWS_RESPONSE_CHECKSUM_VALIDATION": "when_required", + "AWS_PAGER": "", + } + + def inventory(self) -> dict[str, int]: + inventory_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + prefix="sccache-inventory-", suffix=".json", delete=False + ) as output: + inventory_path = Path(output.name) + try: + result = subprocess.run( + [ + self.aws, + "s3api", + "list-objects-v2", + "--endpoint-url", + self.endpoint, + "--region", + REGION, + "--bucket", + BUCKET, + "--prefix", + COMPILER_PREFIX + "/", + "--page-size", + "1000", + "--output", + "json", + ], + env=self.environment(), + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.DEVNULL, + check=False, + timeout=1800, + ) + except (OSError, subprocess.TimeoutExpired): + raise WarmsetError("R2 compiler inventory failed") from None + if result.returncode != 0: + raise WarmsetError("R2 compiler inventory failed") + if inventory_path.stat().st_size > MAX_INVENTORY_BYTES: + raise WarmsetError("R2 compiler inventory exceeds the size limit") + with inventory_path.open(encoding="utf-8") as source: + payload = json.load(source) + except (OSError, UnicodeError, json.JSONDecodeError): + raise WarmsetError("R2 compiler inventory is invalid") from None + finally: + if inventory_path is not None: + inventory_path.unlink(missing_ok=True) + + contents = payload.get("Contents") if isinstance(payload, dict) else None + if contents is None: + contents = [] + if not isinstance(contents, list): + raise WarmsetError("R2 compiler inventory is invalid") + objects: dict[str, int] = {} + for item in contents: + if not isinstance(item, dict): + raise WarmsetError("R2 compiler inventory is invalid") + key = item.get("Key") + size = item.get("Size") + # sccache's capability probe may leave a zero-byte check object. + # Captured compiler objects are required to be non-empty below. + if not isinstance(key, str) or type(size) is not int or size < 0: + raise WarmsetError("R2 compiler inventory contains invalid metadata") + previous = objects.setdefault(key, size) + if previous != size: + raise WarmsetError("R2 compiler inventory disagrees about an object") + return objects + + def put(self, key: str, source: Path) -> None: + try: + result = subprocess.run( + [ + self.aws, + "s3api", + "put-object", + "--endpoint-url", + self.endpoint, + "--region", + REGION, + "--bucket", + BUCKET, + "--key", + key, + "--body", + str(source), + "--content-type", + "application/json", + ], + env=self.environment(), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=1800, + ) + except (OSError, subprocess.TimeoutExpired): + raise WarmsetError("R2 warm-set publication failed") from None + if result.returncode != 0: + raise WarmsetError("R2 warm-set publication failed") + + +def build_manifest( + hashes: list[str], + inventory: Mapping[str, int], + producer_sha: str, + published_at: int, +) -> dict[str, object]: + if not COMMIT_SHA.fullmatch(producer_sha): + raise WarmsetError("GITHUB_SHA is malformed") + captured: list[tuple[str, int]] = [] + missing = 0 + for cache_hash in hashes: + path = normalized_path(cache_hash) + size = inventory.get(f"{COMPILER_PREFIX}/{path}") + if type(size) is not int or size <= 0: + missing += 1 + continue + captured.append((path, size)) + if missing: + raise WarmsetError( + f"{missing} compiler object(s) were absent from the durable R2 cache" + ) + + selected: list[dict[str, object]] = [] + selected_size = 0 + for path, size in captured: + if len(selected) >= MAX_OBJECTS: + break + if size > MAX_WARM_BYTES or selected_size + size > MAX_WARM_BYTES: + continue + selected.append({"path": path, "size": size}) + selected_size += size + if not selected: + raise WarmsetError("no compiler objects fit the host warm-set budget") + + generation = f"{producer_sha}-{published_at}" + return { + "schema_version": 1, + "bucket": BUCKET, + "key_prefix": COMPILER_PREFIX, + "generation": generation, + "producer_sha": producer_sha, + "published_at": published_at, + "expires_at": published_at + MANIFEST_TTL_SECONDS, + "max_bytes": MAX_WARM_BYTES, + "captured_object_count": len(captured), + "captured_size_bytes": sum(size for _, size in captured), + "selected_object_count": len(selected), + "selected_size_bytes": selected_size, + "objects": selected, + } + + +def publish(key_files: list[Path]) -> tuple[str, int, int]: + hashes = load_hashes(key_files) + producer_sha = require_environment("GITHUB_SHA") + client = R2Client() + manifest = build_manifest( + hashes, client.inventory(), producer_sha, int(time.time()) + ) + encoded = ( + json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n" + ).encode() + if len(encoded) > MAX_MANIFEST_BYTES: + raise WarmsetError("compiler warm-set manifest exceeds the size limit") + + manifest_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + prefix="sccache-warmset-", suffix=".json", delete=False + ) as output: + output.write(encoded) + output.flush() + os.fsync(output.fileno()) + manifest_path = Path(output.name) + client.put(f"{MANIFEST_PREFIX}/latest.json", manifest_path) + finally: + if manifest_path is not None: + manifest_path.unlink(missing_ok=True) + return ( + str(manifest["generation"]), + int(manifest["selected_object_count"]), + int(manifest["selected_size_bytes"]), + ) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + extract_parser = subparsers.add_parser("extract") + extract_parser.add_argument("log", type=Path) + extract_parser.add_argument("output", type=Path) + publish_parser = subparsers.add_parser("publish") + publish_parser.add_argument("key_files", type=Path, nargs="+") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if args.command == "extract": + count = extract_hashes(args.log, args.output) + print(f"Captured {count} exact compiler cache keys.") + return 0 + generation, count, size = publish(args.key_files) + print( + f"Published compiler warm set {generation}: " f"{count} objects, {size} bytes." + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except WarmsetError as error: + print(f"compiler warm-set publication failed: {error}", file=sys.stderr) + raise SystemExit(1) from None diff --git a/.github/scripts/rust-setup-preflight.sh b/.github/scripts/rust-setup-preflight.sh new file mode 100755 index 0000000000..c19ed33269 --- /dev/null +++ b/.github/scripts/rust-setup-preflight.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +output_file="${1:?usage: rust-setup-preflight.sh GITHUB_OUTPUT}" +contract_dir="${FIREACTIONS_RUNNER_IMAGE_CONTRACT_DIR:-/etc/fireactions-runner-image}" +contract_file="${contract_dir}/image-contract.env" +packages_file="${contract_dir}/packages.txt" + +required_packages=( + build-essential + clang + curl + git + libssl-dev + libudev-dev + llvm + make + pkg-config + protobuf-compiler + python3 + python3-dev +) + +system_ready=false +rustup_ready=false +toolchain_ready=false + +if command -v rustup >/dev/null 2>&1; then + rustup_ready=true +fi + +if [[ -r "${contract_file}" && -r "${packages_file}" ]] && command -v dpkg-query >/dev/null 2>&1; then + system_ready=true + for package in "${required_packages[@]}"; do + if ! grep -Fxq -- "${package}" "${packages_file}"; then + system_ready=false + break + fi + package_status="$(dpkg-query -W -f='${db:Status-Abbrev}' "${package}" 2>/dev/null || true)" + if [[ "${package_status}" != "ii " ]]; then + system_ready=false + break + fi + done +fi + +if [[ -r "${contract_file}" ]] \ + && [[ -r rust-toolchain.toml ]] \ + && command -v rustup >/dev/null 2>&1 \ + && command -v cargo >/dev/null 2>&1 \ + && command -v rustc >/dev/null 2>&1; then + requested_toolchain="" + requested_toolchain_count=0 + while IFS= read -r channel; do + requested_toolchain="${channel}" + requested_toolchain_count=$((requested_toolchain_count + 1)) + done < <( + sed -nE 's/^[[:space:]]*channel[[:space:]]*=[[:space:]]*"([^"[:space:]]+)".*$/\1/p' \ + rust-toolchain.toml + ) + if [[ "${requested_toolchain_count}" -ne 1 ]] \ + || [[ ! "${requested_toolchain}" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then + requested_toolchain="" + fi + + installed_toolchain="" + while IFS= read -r toolchain_line; do + candidate="${toolchain_line%% *}" + if [[ "${candidate}" == "${requested_toolchain}" \ + || "${candidate}" == "${requested_toolchain}-"* ]]; then + installed_toolchain="${candidate}" + break + fi + done < <(rustup toolchain list 2>/dev/null || true) + + if [[ -n "${requested_toolchain}" && -n "${installed_toolchain}" ]]; then + toolchain_ready=true + installed_components="$( + rustup component list --installed --toolchain "${installed_toolchain}" 2>/dev/null || true + )" + required_components=(cargo rustc rust-std) + if [[ -n "${RUST_SETUP_COMPONENTS:-}" ]]; then + IFS=',' read -r -a extra_components <<< "${RUST_SETUP_COMPONENTS}" + required_components+=("${extra_components[@]}") + fi + for component in "${required_components[@]}"; do + component="${component//[[:space:]]/}" + [[ -z "${component}" ]] && continue + component_ready=false + while IFS= read -r installed_component; do + if [[ "${installed_component}" == "${component}" || "${installed_component}" == "${component}-"* ]]; then + component_ready=true + break + fi + done <<< "${installed_components}" + if [[ "${component_ready}" != true ]]; then + toolchain_ready=false + break + fi + done + fi +fi + +{ + echo "system_ready=${system_ready}" + echo "rustup_ready=${rustup_ready}" + echo "toolchain_ready=${toolchain_ready}" +} >> "${output_file}" + +echo "runner image preflight: system_ready=${system_ready} rustup_ready=${rustup_ready} toolchain_ready=${toolchain_ready}" diff --git a/.github/scripts/sccache-config.py b/.github/scripts/sccache-config.py new file mode 100755 index 0000000000..da022e8202 --- /dev/null +++ b/.github/scripts/sccache-config.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +"""Prepare and activate the complete typed sccache backend contract.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +TRUSTED_R2 = { + "bucket": "subtensor-ci-sccache", + "endpoint": "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com", + "region": "auto", + "s3_use_ssl": True, + "key_prefix": "subtensor/v1", +} +LOCAL_ENDPOINT = "http://192.168.128.1:8092" +LOCAL_FIELDS = {"endpoint", "key_prefix", "username", "password"} +MMDS_TOKEN_URL_DEFAULT = "http://169.254.169.254/latest/api/token" +MMDS_METADATA_URL_DEFAULT = "http://169.254.169.254/latest/meta-data/sccache" + + +class ConfigError(Exception): + """A configuration error safe to report without credential material.""" + + +def load_object(path: Path) -> dict[str, object]: + try: + with path.open(encoding="utf-8") as handle: + value = json.load(handle) + except (OSError, UnicodeError, json.JSONDecodeError): + raise ConfigError("configuration is not valid JSON") from None + if not isinstance(value, dict): + raise ConfigError("configuration is not an object") + return value + + +def atomic_write(path: Path, value: dict[str, object]) -> None: + temporary = path.with_name(path.name + ".normalized") + try: + with temporary.open("w", encoding="utf-8") as handle: + json.dump(value, handle, separators=(",", ":")) + os.chmod(temporary, 0o600) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def append_values(path: Path, values: dict[str, str]) -> None: + with path.open("a", encoding="utf-8") as output: + for name, value in values.items(): + if "\n" in value or "\r" in value: + raise ConfigError(f"invalid newline in {name}") + output.write(f"{name}={value}\n") + + +def warning(message: str) -> None: + print(f"::warning::{message}") + + +def mask(value: str) -> None: + if value: + print(f"::add-mask::{value}") + + +def required_string(values: dict[str, object], key: str) -> str: + value = values.get(key) + if not isinstance(value, str) or not value or "\n" in value or "\r" in value: + raise ConfigError(f"invalid {key}") + return value + + +def string_value(values: dict[str, object], key: str) -> str: + value = values.get(key) + if not isinstance(value, str) or "\n" in value or "\r" in value: + raise ConfigError(f"invalid {key}") + return value + + +def validate_r2(values: dict[str, object], mode: str) -> None: + if mode not in {"reader", "writer"}: + raise ConfigError("invalid R2 mode") + for key, expected in TRUSTED_R2.items(): + if values.get(key) != expected: + raise ConfigError(f"invalid {key}") + expected_rw_mode = "READ_ONLY" if mode == "reader" else "READ_WRITE" + if values.get("s3_rw_mode") != expected_rw_mode: + raise ConfigError("invalid s3_rw_mode") + required_string(values, "access_key_id") + required_string(values, "secret_access_key") + + +def validate_local( + value: object, + *, + credential_source: dict[str, object] | None = None, +) -> dict[str, str]: + if not isinstance(value, dict) or set(value) != LOCAL_FIELDS: + raise ConfigError("invalid local cache contract") + local = {key: string_value(value, key) for key in LOCAL_FIELDS} + if not local["endpoint"] or not local["username"] or not local["password"]: + raise ConfigError("invalid local cache contract") + if local["endpoint"] != LOCAL_ENDPOINT or local["key_prefix"] != "": + raise ConfigError("invalid local cache endpoint") + if credential_source is not None and ( + local["username"] != required_string(credential_source, "access_key_id") + or local["password"] + != required_string(credential_source, "secret_access_key") + ): + raise ConfigError("invalid local cache credential") + return local + + +def normalize_reader(path: Path, local_mode: str) -> None: + if local_mode not in {"auto", "disabled"}: + raise ConfigError("invalid local tier mode") + values = load_object(path) + validate_r2(values, "reader") + if local_mode == "disabled": + values.pop("local", None) + elif values.get("local") is not None: + values["local"] = validate_local( + values["local"], credential_source=values + ) + values["mode"] = "reader" + atomic_write(path, values) + + +def write_writer(path: Path) -> None: + values: dict[str, object] = { + "mode": "writer", + **TRUSTED_R2, + "s3_rw_mode": "READ_WRITE", + "access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", ""), + "secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", ""), + } + validate_r2(values, "writer") + atomic_write(path, values) + + +def attach_writer_local(writer_path: Path, reader_path: Path) -> None: + writer = load_object(writer_path) + if writer.get("mode") != "writer": + raise ConfigError("invalid writer mode") + validate_r2(writer, "writer") + + reader = load_object(reader_path) + if reader.get("mode") != "reader": + raise ConfigError("invalid reader mode") + validate_r2(reader, "reader") + if reader.get("local") is None: + raise ConfigError("local cache contract is unavailable") + writer["local"] = validate_local(reader["local"], credential_source=reader) + atomic_write(writer_path, writer) + + +def credential_values(values: dict[str, object]) -> list[str]: + mode = values.get("mode") + if mode == "gha": + return [] + if mode not in {"reader", "writer"}: + raise ConfigError("invalid mode") + validate_r2(values, mode) + credentials = [ + required_string(values, "access_key_id"), + required_string(values, "secret_access_key"), + ] + if values.get("local") is not None: + local = validate_local(values["local"]) + credentials.extend((local["username"], local["password"])) + return credentials + + +def source_is_trusted(event_path: Path | None) -> bool: + event_name = os.environ.get("GITHUB_EVENT_NAME", "") + ref = os.environ.get("GITHUB_REF", "") + repository = os.environ.get("GITHUB_REPOSITORY", "") + if (event_name, ref) in { + ("push", "refs/heads/main"), + ("push", "refs/heads/devnet"), + ("push", "refs/heads/testnet"), + ("schedule", "refs/heads/main"), + }: + return True + if event_name == "workflow_dispatch" and ref == "refs/heads/main": + if event_path is None or not event_path.is_file(): + return False + inputs = load_object(event_path).get("inputs") + return isinstance(inputs, dict) and inputs.get("source_ref") in { + "main", + "devnet", + "testnet", + } + if event_name == "pull_request" and re.fullmatch( + r"refs/pull/[0-9]+/merge", ref + ): + if event_path is None or not event_path.is_file() or not repository: + return False + pull = load_object(event_path).get("pull_request") + if not isinstance(pull, dict): + return False + head = pull.get("head") + user = pull.get("user") + if not isinstance(head, dict) or not isinstance(user, dict): + return False + head_repository = head.get("repo") + return ( + isinstance(head_repository, dict) + and head_repository.get("full_name") == repository + and head_repository.get("fork") is False + and user.get("login") != "dependabot[bot]" + ) + return False + + +def trusted_source(event_path: Path | None) -> bool: + try: + return source_is_trusted(event_path) + except ConfigError: + return False + + +def curl(arguments: list[str], *, capture: bool = False) -> subprocess.CompletedProcess: + try: + return subprocess.run( + ["curl", *arguments], + check=False, + capture_output=capture, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + raise ConfigError("MMDS request failed") from None + + +def fetch_reader_contract(path: Path) -> str | None: + token_url = os.environ.get("MMDS_TOKEN_URL", MMDS_TOKEN_URL_DEFAULT) + metadata_url = os.environ.get("MMDS_METADATA_URL", MMDS_METADATA_URL_DEFAULT) + try: + token_result = curl( + [ + "--fail", + "--silent", + "--show-error", + "--connect-timeout", + "1", + "--max-time", + "2", + "--request", + "PUT", + "--header", + "X-Metadata-Token-TTL-Seconds: 60", + token_url, + ], + capture=True, + ) + if token_result.returncode != 0: + return "MMDSv2 token service is unavailable" + token = token_result.stdout.decode("utf-8").rstrip("\r\n") + if not token or "\n" in token or "\r" in token: + return "MMDSv2 returned an invalid token" + + metadata_result = curl( + [ + "--fail", + "--silent", + "--show-error", + "--connect-timeout", + "1", + "--max-time", + "3", + "--header", + f"X-Metadata-Token: {token}", + "--header", + "Accept: application/json", + "--output", + str(path), + metadata_url, + ] + ) + if metadata_result.returncode != 0: + return "MMDSv2 sccache metadata is unavailable" + os.chmod(path, 0o600) + normalize_reader(path, os.environ.get("SCCACHE_LOCAL_TIER_MODE", "auto")) + except (ConfigError, OSError, UnicodeError): + path.unlink(missing_ok=True) + return "MMDSv2 sccache metadata failed validation" + return None + + +def disable_prepare(config_path: Path, output_path: Path, reason: str) -> int: + config_path.unlink(missing_ok=True) + append_values(output_path, {"available": "false"}) + warning(f"sccache disabled: {reason}") + return 0 + + +def fallback_reader(config_path: Path, output_path: Path, reason: str) -> bool: + if os.environ.get("SCCACHE_GHA_FALLBACK", "true") == "true": + atomic_write(config_path, {"mode": "gha"}) + warning(f"R2 reader unavailable; using GitHub Actions sccache: {reason}") + return True + disable_prepare(config_path, output_path, reason) + return False + + +def prepare_reader(config_path: Path, output_path: Path) -> bool: + error = fetch_reader_contract(config_path) + return error is None or fallback_reader(config_path, output_path, error) + + +def credentials_are_well_formed() -> bool: + return all( + value and "\n" not in value and "\r" not in value + for value in ( + os.environ.get("AWS_ACCESS_KEY_ID", ""), + os.environ.get("AWS_SECRET_ACCESS_KEY", ""), + ) + ) + + +def prepare_writer(config_path: Path, output_path: Path) -> bool: + event_path_value = os.environ.get("GITHUB_EVENT_PATH", "") + event_path = Path(event_path_value) if event_path_value else None + if not trusted_source(event_path): + disable_prepare( + config_path, output_path, "writer mode is restricted to trusted cache sources" + ) + return False + if not credentials_are_well_formed(): + disable_prepare( + config_path, output_path, "protected writer credentials are unavailable or malformed" + ) + return False + local_mode = os.environ.get("SCCACHE_LOCAL_TIER_MODE", "auto") + if local_mode not in {"auto", "disabled"}: + disable_prepare(config_path, output_path, "invalid local tier mode") + return False + + write_writer(config_path) + if local_mode == "disabled": + return True + + reader_path = config_path.with_name(config_path.name + ".reader") + reader_path.unlink(missing_ok=True) + error = fetch_reader_contract(reader_path) + if error is not None: + reader_path.unlink(missing_ok=True) + warning(f"local sccache reader unavailable; using direct R2 writer: {error}") + return True + try: + attach_writer_local(config_path, reader_path) + except ConfigError: + warning("local sccache reader failed validation; using direct R2 writer") + finally: + reader_path.unlink(missing_ok=True) + return True + + +def prepare(mode: str, config_path: Path, output_path: Path) -> int: + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.unlink(missing_ok=True) + + if mode == "reader": + available = prepare_reader(config_path, output_path) + elif mode == "writer": + available = prepare_writer(config_path, output_path) + elif mode == "auto": + event_path_value = os.environ.get("GITHUB_EVENT_PATH", "") + event_path = Path(event_path_value) if event_path_value else None + if trusted_source(event_path) and credentials_are_well_formed(): + available = prepare_writer(config_path, output_path) + else: + available = prepare_reader(config_path, output_path) + else: + return disable_prepare(config_path, output_path, "unknown credential mode") + + if not available: + return 0 + values = load_object(config_path) + for credential in credential_values(values): + mask(credential) + append_values( + output_path, + {"available": "true", "config-file": str(config_path)}, + ) + print(f"sccache {mode} configuration validated") + return 0 + + +def stop_server(binary: str, environment: dict[str, str]) -> None: + try: + subprocess.run( + [binary, "--stop-server"], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired): + pass + + +def start_server(binary: str, environment: dict[str, str], log_path: Path) -> bool: + try: + with log_path.open("wb") as log: + result = subprocess.run( + [binary, "--start-server"], + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + timeout=30, + ) + return result.returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + +def disable_activate(config_path: Path, output_path: Path, reason: str) -> int: + config_path.unlink(missing_ok=True) + append_values(output_path, {"enabled": "false"}) + warning(f"sccache disabled: {reason}") + return 0 + + +def activate(config_path: Path, env_path: Path, output_path: Path) -> int: + if not config_path.is_file(): + return disable_activate( + config_path, output_path, "validated configuration is unavailable" + ) + if os.environ.get("SCCACHE_INSTALL_OUTCOME", "success") != "success": + return disable_activate(config_path, output_path, "sccache installation failed") + binary = os.environ.get("SCCACHE_PATH", "") or shutil.which("sccache") or "" + if not binary or not os.access(binary, os.X_OK): + return disable_activate(config_path, output_path, "sccache executable is unavailable") + + try: + values = load_object(config_path) + mode = values.get("mode") + environment = os.environ.copy() + exported: dict[str, str] + local_tier = False + + if mode == "gha": + if set(values) != {"mode"}: + raise ConfigError("invalid GitHub Actions cache contract") + environment.update( + { + "SCCACHE_GHA_ENABLED": "true", + "SCCACHE_IGNORE_SERVER_IO_ERROR": "1", + } + ) + exported = { + "SCCACHE_ENABLED": "true", + "SCCACHE_BACKEND": "gha", + "SCCACHE_GHA_ENABLED": "true", + "SCCACHE_IGNORE_SERVER_IO_ERROR": "1", + "RUSTC_WRAPPER": "sccache", + "CARGO_INCREMENTAL": "0", + } + elif mode in {"reader", "writer"}: + validate_r2(values, mode) + access_key = required_string(values, "access_key_id") + secret_key = required_string(values, "secret_access_key") + mask(access_key) + mask(secret_key) + environment.update( + { + "SCCACHE_BUCKET": required_string(values, "bucket"), + "SCCACHE_ENDPOINT": required_string(values, "endpoint"), + "SCCACHE_REGION": required_string(values, "region"), + "SCCACHE_S3_USE_SSL": "true", + "SCCACHE_S3_KEY_PREFIX": required_string(values, "key_prefix"), + "SCCACHE_IGNORE_SERVER_IO_ERROR": "1", + "AWS_ACCESS_KEY_ID": access_key, + "AWS_SECRET_ACCESS_KEY": secret_key, + } + ) + if values.get("local") is not None: + local = validate_local(values["local"]) + mask(local["username"]) + mask(local["password"]) + environment.update( + { + "SCCACHE_MULTILEVEL_CHAIN": "webdav,s3", + "SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY": ( + "all" if mode == "writer" else "ignore" + ), + "SCCACHE_WEBDAV_ENDPOINT": local["endpoint"], + "SCCACHE_WEBDAV_KEY_PREFIX": local["key_prefix"], + "SCCACHE_WEBDAV_USERNAME": local["username"], + "SCCACHE_WEBDAV_PASSWORD": local["password"], + } + ) + local_tier = True + exported = { + "SCCACHE_ENABLED": "true", + "SCCACHE_BACKEND": "r2", + "RUSTC_WRAPPER": "sccache", + "CARGO_INCREMENTAL": "0", + **{ + key: environment[key] + for key in ( + "SCCACHE_BUCKET", + "SCCACHE_ENDPOINT", + "SCCACHE_REGION", + "SCCACHE_S3_USE_SSL", + "SCCACHE_S3_KEY_PREFIX", + "SCCACHE_IGNORE_SERVER_IO_ERROR", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + ) + }, + "SCCACHE_LOCAL_TIER": "true" if local_tier else "false", + } + else: + raise ConfigError("invalid mode") + except ConfigError: + return disable_activate( + config_path, output_path, "validated configuration could not be parsed" + ) + + runner_temp = Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) + runner_temp.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + prefix="sccache-start-", suffix=".log", dir=runner_temp, delete=False + ) as log: + log_path = Path(log.name) + try: + stop_server(binary, environment) + if not start_server(binary, environment, log_path): + if local_tier: + warning("local sccache tier startup failed; retrying direct R2") + stop_server(binary, environment) + for key in ( + "SCCACHE_MULTILEVEL_CHAIN", + "SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY", + "SCCACHE_WEBDAV_ENDPOINT", + "SCCACHE_WEBDAV_KEY_PREFIX", + "SCCACHE_WEBDAV_USERNAME", + "SCCACHE_WEBDAV_PASSWORD", + ): + environment.pop(key, None) + local_tier = False + exported["SCCACHE_LOCAL_TIER"] = "false" + if not start_server(binary, environment, log_path): + stop_server(binary, environment) + return disable_activate( + config_path, output_path, "R2 backend startup check failed" + ) + else: + stop_server(binary, environment) + backend = "GitHub Actions" if mode == "gha" else "R2" + return disable_activate( + config_path, output_path, f"{backend} backend startup failed" + ) + + if local_tier: + for key in ( + "SCCACHE_MULTILEVEL_CHAIN", + "SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY", + "SCCACHE_WEBDAV_ENDPOINT", + "SCCACHE_WEBDAV_KEY_PREFIX", + "SCCACHE_WEBDAV_USERNAME", + "SCCACHE_WEBDAV_PASSWORD", + ): + exported[key] = environment[key] + append_values(env_path, exported) + append_values(output_path, {"enabled": "true"}) + config_path.unlink(missing_ok=True) + backend = "GitHub Actions" if mode == "gha" else "R2" + print(f"sccache {backend} backend enabled" + (f" in {mode} mode" if mode != "gha" else "")) + return 0 + finally: + log_path.unlink(missing_ok=True) + + +def main(argv: list[str]) -> int: + if not argv: + raise ConfigError("missing command") + command, *arguments = argv + if command == "prepare" and len(arguments) == 3: + return prepare(arguments[0], Path(arguments[1]), Path(arguments[2])) + if command == "activate" and len(arguments) == 3: + return activate(Path(arguments[0]), Path(arguments[1]), Path(arguments[2])) + raise ConfigError("invalid command arguments") + + +if __name__ == "__main__": + os.umask(0o077) + try: + raise SystemExit(main(sys.argv[1:])) + except ConfigError as error: + print(f"sccache configuration error: {error}", file=sys.stderr) + raise SystemExit(1) from None diff --git a/.github/scripts/sccache-configure.sh b/.github/scripts/sccache-configure.sh index 782978f621..6c485dbe0a 100755 --- a/.github/scripts/sccache-configure.sh +++ b/.github/scripts/sccache-configure.sh @@ -1,412 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail -# Secret-safe configuration boundary for the shared R2 sccache backend. -# -# prepare MODE CONFIG_FILE OUTPUT_FILE -# Fetches and validates the MMDS reader contract, or materializes the trusted -# writer contract from the protected Environment credentials. -# -# activate CONFIG_FILE ENV_FILE OUTPUT_FILE -# Starts sccache against R2 and exports the wrapper only after startup works. -# -# Writer mode is write-through and content-addressed. Each successful rustc -# invocation is independently reusable even if a later compile or test fails; -# changed inputs produce a different key instead of replacing older artifacts. - -set -u - -readonly MMDS_TOKEN_URL_DEFAULT="http://169.254.169.254/latest/api/token" -readonly MMDS_METADATA_URL_DEFAULT="http://169.254.169.254/latest/meta-data/sccache" - -warning() { - printf '::warning::%s\n' "$1" -} - -set_output() { - local output_file="$1" - local name="$2" - local value="$3" - printf '%s=%s\n' "$name" "$value" >> "$output_file" -} - -disable_prepare() { - local config_file="$1" - local output_file="$2" - local reason="$3" - if [[ -n "$config_file" ]]; then - rm -f "$config_file" - fi - set_output "$output_file" available false - warning "sccache disabled: $reason" - exit 0 -} - -disable_activate() { - local config_file="$1" - local output_file="$2" - local reason="$3" - if [[ -n "$config_file" ]]; then - rm -f "$config_file" - fi - set_output "$output_file" enabled false - warning "sccache disabled: $reason" - exit 0 -} - -mask_config_credentials() { - local config_file="$1" - local -a credentials=() - local credential - while IFS= read -r credential; do - credentials+=("$credential") - done < <( - python3 -c ' -import json, sys -data = json.load(open(sys.argv[1], encoding="utf-8")) -if data.get("mode") == "gha": - raise SystemExit(0) -for key in ("access_key_id", "secret_access_key"): - print(data[key]) -' "$config_file" - ) - if [[ ${#credentials[@]} -eq 2 ]]; then - printf '::add-mask::%s\n' "${credentials[0]}" - printf '::add-mask::%s\n' "${credentials[1]}" - fi -} - -prepare_gha() { - local config_file="$1" - printf '{"mode":"gha"}' > "$config_file" - chmod 0600 "$config_file" -} - -fallback_reader() { - local config_file="$1" - local output_file="$2" - local reason="$3" - if [[ "${SCCACHE_GHA_FALLBACK:-true}" == true ]]; then - prepare_gha "$config_file" - warning "R2 reader unavailable; using GitHub Actions sccache: $reason" - return 0 - fi - disable_prepare "$config_file" "$output_file" "$reason" -} - -prepare_reader() { - local config_file="$1" - local output_file="$2" - local token_url="${MMDS_TOKEN_URL:-$MMDS_TOKEN_URL_DEFAULT}" - local metadata_url="${MMDS_METADATA_URL:-$MMDS_METADATA_URL_DEFAULT}" - local token - - if ! token="$(curl --fail --silent --show-error --connect-timeout 1 --max-time 2 \ - --request PUT \ - --header 'X-Metadata-Token-TTL-Seconds: 60' \ - "$token_url" 2>/dev/null)"; then - fallback_reader "$config_file" "$output_file" "MMDSv2 token service is unavailable" - return - fi - - if [[ -z "$token" ]]; then - fallback_reader "$config_file" "$output_file" "MMDSv2 returned an empty token" - return - fi - - if ! curl --fail --silent --show-error --connect-timeout 1 --max-time 3 \ - --header "X-Metadata-Token: $token" \ - --header 'Accept: application/json' \ - --output "$config_file" \ - "$metadata_url" 2>/dev/null; then - fallback_reader "$config_file" "$output_file" "MMDSv2 sccache metadata is unavailable" - return - fi - chmod 0600 "$config_file" - - if ! python3 -c ' -import json, os, sys -path = sys.argv[1] -expected = { - "bucket": "subtensor-ci-sccache", - "endpoint": "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com", - "region": "auto", - "s3_use_ssl": True, - "s3_rw_mode": "READ_ONLY", - "key_prefix": "subtensor/v1", -} -with open(path, encoding="utf-8") as handle: - data = json.load(handle) -for key, value in expected.items(): - if data.get(key) != value: - raise ValueError(f"invalid {key}") -for key in ("access_key_id", "secret_access_key"): - value = data.get(key) - if not isinstance(value, str) or not value or "\n" in value or "\r" in value: - raise ValueError(f"invalid {key}") -data["mode"] = "reader" -tmp = path + ".normalized" -with open(tmp, "w", encoding="utf-8") as handle: - json.dump(data, handle, separators=(",", ":")) -os.chmod(tmp, 0o600) -os.replace(tmp, path) -' "$config_file" 2>/dev/null; then - fallback_reader "$config_file" "$output_file" "MMDSv2 sccache metadata failed validation" - return - fi -} - -prepare_writer() { - local config_file="$1" - local output_file="$2" - - if ! writer_source_is_trusted; then - disable_prepare "$config_file" "$output_file" "writer mode is restricted to trusted cache sources" - fi - - if [[ -z "${AWS_ACCESS_KEY_ID:-}" || -z "${AWS_SECRET_ACCESS_KEY:-}" ]]; then - disable_prepare "$config_file" "$output_file" "protected writer credentials are unavailable" - fi - if [[ "$AWS_ACCESS_KEY_ID" == *$'\n'* || "$AWS_SECRET_ACCESS_KEY" == *$'\n'* ]]; then - disable_prepare "$config_file" "$output_file" "protected writer credentials are malformed" - fi - - if ! CONFIG_FILE="$config_file" python3 -c ' -import json, os -path = os.environ["CONFIG_FILE"] -data = { - "mode": "writer", - "bucket": "subtensor-ci-sccache", - "endpoint": "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com", - "region": "auto", - "s3_use_ssl": True, - "s3_rw_mode": "READ_WRITE", - "key_prefix": "subtensor/v1", - "access_key_id": os.environ["AWS_ACCESS_KEY_ID"], - "secret_access_key": os.environ["AWS_SECRET_ACCESS_KEY"], -} -with open(path, "w", encoding="utf-8") as handle: - json.dump(data, handle, separators=(",", ":")) -os.chmod(path, 0o600) -' 2>/dev/null; then - disable_prepare "$config_file" "$output_file" "writer configuration could not be materialized" - fi -} - -writer_source_is_trusted() { - case "${GITHUB_EVENT_NAME:-}:${GITHUB_REF:-}" in - push:refs/heads/main|push:refs/heads/devnet|push:refs/heads/testnet|schedule:refs/heads/main) - return 0 - ;; - workflow_dispatch:refs/heads/main) - [[ -f "${GITHUB_EVENT_PATH:-}" ]] || return 1 - GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-}" python3 -c ' -import json, os, sys -with open(sys.argv[1], encoding="utf-8") as handle: - event = json.load(handle) -source_ref = event.get("inputs", {}).get("source_ref") -if source_ref not in {"main", "devnet", "testnet"}: - raise SystemExit(1) -' "$GITHUB_EVENT_PATH" >/dev/null 2>&1 - return - ;; - pull_request:refs/pull/*/merge) - [[ -f "${GITHUB_EVENT_PATH:-}" && -n "${GITHUB_REPOSITORY:-}" ]] || return 1 - GITHUB_REPOSITORY="$GITHUB_REPOSITORY" python3 -c ' -import json, os, re, sys -with open(sys.argv[1], encoding="utf-8") as handle: - event = json.load(handle) -pull = event.get("pull_request") -if not isinstance(pull, dict): - raise SystemExit(1) -head_repo = pull.get("head", {}).get("repo") -if not isinstance(head_repo, dict): - raise SystemExit(1) -trusted = ( - re.fullmatch(r"refs/pull/[0-9]+/merge", os.environ.get("GITHUB_REF", "")) - and head_repo.get("full_name") == os.environ["GITHUB_REPOSITORY"] - and head_repo.get("fork") is False - and pull.get("user", {}).get("login") != "dependabot[bot]" -) -raise SystemExit(0 if trusted else 1) -' "$GITHUB_EVENT_PATH" >/dev/null 2>&1 - return - ;; - *) - return 1 - ;; - esac -} - -prepare_auto() { - local config_file="$1" - local output_file="$2" - - if writer_source_is_trusted && - [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" ]] && - [[ "$AWS_ACCESS_KEY_ID" != *$'\n'* && "$AWS_SECRET_ACCESS_KEY" != *$'\n'* ]]; then - prepare_writer "$config_file" "$output_file" - return - fi - - prepare_reader "$config_file" "$output_file" -} - -prepare() { - local mode="$1" - local config_file="$2" - local output_file="$3" - - umask 077 - mkdir -p "$(dirname "$config_file")" - rm -f "$config_file" - - case "$mode" in - reader) prepare_reader "$config_file" "$output_file" ;; - writer) prepare_writer "$config_file" "$output_file" ;; - auto) prepare_auto "$config_file" "$output_file" ;; - *) disable_prepare "$config_file" "$output_file" "unknown credential mode" ;; - esac - - mask_config_credentials "$config_file" - set_output "$output_file" available true - set_output "$output_file" config-file "$config_file" - printf 'sccache %s configuration validated\n' "$mode" -} - -activate() { - local config_file="$1" - local env_file="$2" - local output_file="$3" - local install_outcome="${SCCACHE_INSTALL_OUTCOME:-success}" - local sccache_bin="${SCCACHE_PATH:-}" - local start_log="${RUNNER_TEMP:-/tmp}/sccache-start-${GITHUB_RUN_ID:-local}-${GITHUB_JOB:-test}.log" - local fields_file="${RUNNER_TEMP:-/tmp}/sccache-fields-${GITHUB_RUN_ID:-local}-${GITHUB_JOB:-test}" - local -a values=() - local value - - umask 077 - if [[ ! -f "$config_file" ]]; then - disable_activate "$config_file" "$output_file" "validated configuration is unavailable" - fi - if [[ "$install_outcome" != "success" ]]; then - disable_activate "$config_file" "$output_file" "sccache installation failed" - fi - if [[ -z "$sccache_bin" ]]; then - sccache_bin="$(command -v sccache || true)" - fi - if [[ -z "$sccache_bin" || ! -x "$sccache_bin" ]]; then - disable_activate "$config_file" "$output_file" "sccache executable is unavailable" - fi - - if ! python3 -c ' -import json, sys -data = json.load(open(sys.argv[1], encoding="utf-8")) -mode = data.get("mode") -if mode not in ("gha", "reader", "writer"): - raise ValueError("invalid mode") -sys.stdout.write(mode + "\0") -if mode == "gha": - raise SystemExit(0) -for key in ("bucket", "endpoint", "region", "key_prefix", "access_key_id", "secret_access_key"): - value = data.get(key) - if not isinstance(value, str) or not value or "\n" in value or "\r" in value: - raise ValueError(f"invalid {key}") - sys.stdout.write(value + "\0") -if data.get("s3_use_ssl") is not True: - raise ValueError("invalid s3_use_ssl") -' "$config_file" >"$fields_file" 2>/dev/null; then - rm -f "$fields_file" - disable_activate "$config_file" "$output_file" "validated configuration could not be parsed" - fi - while IFS= read -r -d '' value; do - values+=("$value") - done < "$fields_file" - rm -f "$fields_file" - if [[ ${#values[@]} -ne 1 && ${#values[@]} -ne 7 ]]; then - disable_activate "$config_file" "$output_file" "validated configuration is incomplete" - fi - - if [[ "${values[0]}" == gha ]]; then - export SCCACHE_GHA_ENABLED=true - export SCCACHE_IGNORE_SERVER_IO_ERROR=1 - "$sccache_bin" --stop-server >/dev/null 2>&1 || true - if ! "$sccache_bin" --start-server >"$start_log" 2>&1; then - "$sccache_bin" --stop-server >/dev/null 2>&1 || true - rm -f "$start_log" - disable_activate "$config_file" "$output_file" "GitHub Actions backend startup failed" - fi - rm -f "$start_log" "$config_file" - { - printf 'SCCACHE_ENABLED=true\n' - printf 'SCCACHE_BACKEND=gha\n' - printf 'SCCACHE_GHA_ENABLED=true\n' - printf 'SCCACHE_IGNORE_SERVER_IO_ERROR=1\n' - printf 'RUSTC_WRAPPER=sccache\n' - printf 'CARGO_INCREMENTAL=0\n' - } >> "$env_file" - set_output "$output_file" enabled true - printf 'sccache GitHub Actions backend enabled\n' - return - fi - - printf '::add-mask::%s\n' "${values[5]}" - printf '::add-mask::%s\n' "${values[6]}" - - export SCCACHE_BUCKET="${values[1]}" - export SCCACHE_ENDPOINT="${values[2]}" - export SCCACHE_REGION="${values[3]}" - export SCCACHE_S3_USE_SSL=true - export SCCACHE_S3_KEY_PREFIX="${values[4]}" - # v0.15.0 has no S3 read/write-mode environment variable. Reader mode is - # enforced by the bucket-scoped Object Read credential validated above. - # sccache otherwise fails a build when its server or remote backend becomes - # unavailable after startup. This makes every such failure compile locally. - export SCCACHE_IGNORE_SERVER_IO_ERROR=1 - export AWS_ACCESS_KEY_ID="${values[5]}" - export AWS_SECRET_ACCESS_KEY="${values[6]}" - - "$sccache_bin" --stop-server >/dev/null 2>&1 || true - if ! "$sccache_bin" --start-server >"$start_log" 2>&1; then - "$sccache_bin" --stop-server >/dev/null 2>&1 || true - rm -f "$start_log" - disable_activate "$config_file" "$output_file" "R2 backend startup check failed" - fi - rm -f "$start_log" "$config_file" - - { - printf 'SCCACHE_ENABLED=true\n' - printf 'SCCACHE_BACKEND=r2\n' - printf 'RUSTC_WRAPPER=sccache\n' - printf 'CARGO_INCREMENTAL=0\n' - printf 'SCCACHE_BUCKET=%s\n' "$SCCACHE_BUCKET" - printf 'SCCACHE_ENDPOINT=%s\n' "$SCCACHE_ENDPOINT" - printf 'SCCACHE_REGION=%s\n' "$SCCACHE_REGION" - printf 'SCCACHE_S3_USE_SSL=true\n' - printf 'SCCACHE_S3_KEY_PREFIX=%s\n' "$SCCACHE_S3_KEY_PREFIX" - printf 'SCCACHE_IGNORE_SERVER_IO_ERROR=1\n' - printf 'AWS_ACCESS_KEY_ID=%s\n' "$AWS_ACCESS_KEY_ID" - printf 'AWS_SECRET_ACCESS_KEY=%s\n' "$AWS_SECRET_ACCESS_KEY" - } >> "$env_file" - set_output "$output_file" enabled true - printf 'sccache R2 backend enabled in %s mode\n' "${values[0]}" -} - -if [[ $# -lt 1 ]]; then - printf 'usage: %s prepare|activate ...\n' "$0" >&2 - exit 2 -fi - -case "$1" in - prepare) - [[ $# -eq 4 ]] || { printf 'usage: %s prepare MODE CONFIG_FILE OUTPUT_FILE\n' "$0" >&2; exit 2; } - prepare "$2" "$3" "$4" - ;; - activate) - [[ $# -eq 4 ]] || { printf 'usage: %s activate CONFIG_FILE ENV_FILE OUTPUT_FILE\n' "$0" >&2; exit 2; } - activate "$2" "$3" "$4" - ;; - *) - printf 'unknown command: %s\n' "$1" >&2 - exit 2 - ;; -esac +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$SCRIPT_DIR/sccache-config.py" "$@" diff --git a/.github/scripts/sccache-report.sh b/.github/scripts/sccache-report.sh new file mode 100755 index 0000000000..9db2380b62 --- /dev/null +++ b/.github/scripts/sccache-report.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# Emit the most detailed sccache statistics supported by the installed client, +# while keeping observability fail-open. The optional prefix controls the two +# output files: PREFIX.txt (human-readable) and PREFIX.json (machine-readable). + +set -uo pipefail + +label="${1:-Compiler cache}" +slug="$(printf '%s' "$label" | tr -cs '[:alnum:]._-' '-' | sed 's/^-//; s/-$//')" +[[ -n "$slug" ]] || slug=sccache +prefix="${2:-${RUNNER_TEMP:-/tmp}/sccache-${slug}}" +text_path="${prefix}.txt" +json_path="${prefix}.json" +backend="${SCCACHE_BACKEND:-disabled}" +local_tier="${SCCACHE_LOCAL_TIER:-false}" +stats_command=unavailable + +mkdir -p "$(dirname "$prefix")" 2>/dev/null || true + +if [[ "${SCCACHE_ENABLED:-false}" == true ]] && command -v sccache >/dev/null 2>&1; then + if sccache --show-adv-stats >"$text_path" 2>&1; then + stats_command=advanced + elif sccache --show-stats >"$text_path" 2>&1; then + stats_command=basic + else + printf 'sccache statistics were unavailable\n' >"$text_path" 2>/dev/null || true + fi + + if ! sccache --show-stats --stats-format=json >"$json_path" 2>/dev/null; then + printf '{}\n' >"$json_path" 2>/dev/null || true + fi +else + printf 'sccache is disabled for this job\n' >"$text_path" 2>/dev/null || true + printf '{}\n' >"$json_path" 2>/dev/null || true +fi + +echo "sccache report: backend=$backend host-local=$local_tier stats=$stats_command" +[[ ! -f "$text_path" ]] || cat "$text_path" + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + if ! { + echo "### $label" + echo "- Configured backend: $backend" + echo "- Host-local tier active: $local_tier" + echo "- Statistics detail: $stats_command" + if [[ "$local_tier" == true ]]; then + echo "- Per-tier note: sccache 0.15 combines host-local and R2 hits in these totals" + fi + echo '```text' + if [[ -f "$text_path" ]]; then + cat "$text_path" + else + echo "sccache report could not be written" + fi + echo '```' + } >>"$GITHUB_STEP_SUMMARY"; then + echo "::warning::could not append sccache statistics to the job summary" + fi +fi + +# Cache reporting must never change the result of the build it observes. +exit 0 diff --git a/.github/scripts/select-shared-release-artifact.sh b/.github/scripts/select-shared-release-artifact.sh new file mode 100755 index 0000000000..6ffcb79257 --- /dev/null +++ b/.github/scripts/select-shared-release-artifact.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "usage: $0 OUTPUT_FILE [MAX_WAIT_SECONDS]" >&2 + exit 2 +} + +[[ $# -ge 1 && $# -le 2 ]] || usage +output_file="$1" +max_wait_seconds="${2:-360}" + +: "${GH_TOKEN:?GH_TOKEN must contain a short-lived Actions token}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GITHUB_REPOSITORY_ID:?GITHUB_REPOSITORY_ID must be set}" +: "${GITHUB_SHA:?GITHUB_SHA must be set}" +: "${GITHUB_PR_HEAD_SHA:?GITHUB_PR_HEAD_SHA must be set}" + +[[ "$GITHUB_REPOSITORY_ID" =~ ^[1-9][0-9]*$ ]] || usage +[[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] || usage +[[ "$GITHUB_PR_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || usage +[[ "$max_wait_seconds" =~ ^[0-9]+$ ]] || usage + +artifact_name="node-subtensor-release-$GITHUB_SHA" +workflow_path=.github/workflows/runtime-checks.yml +started=$(date -u +%s) + +write_miss() { + { + echo "found=false" + echo "waited_seconds=$(($(date -u +%s) - started))" + } >> "$output_file" +} + +while true; do + artifacts=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ + 2>/dev/null || true) + + if jq -e . >/dev/null 2>&1 <<< "$artifacts"; then + while IFS= read -r candidate; do + [[ -n "$candidate" ]] || continue + artifact_id=$(jq -er '.id' <<< "$candidate") + run_id=$(jq -er '.workflow_run.id' <<< "$candidate") + + run=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" \ + 2>/dev/null || true) + if ! jq -e \ + --arg repository_id "$GITHUB_REPOSITORY_ID" \ + --arg sha "$GITHUB_PR_HEAD_SHA" \ + --arg workflow_path "$workflow_path" \ + --arg run_id "$run_id" ' + (.id | tostring) == $run_id and + .head_sha == $sha and + (.head_repository.id | tostring) == $repository_id and + .path == $workflow_path and + .event == "pull_request" + ' >/dev/null 2>&1 <<< "$run"; then + continue + fi + + digest=$(jq -er '.digest' <<< "$candidate") + size=$(jq -er '.size_in_bytes' <<< "$candidate") + { + echo "found=true" + echo "artifact_id=$artifact_id" + echo "digest=$digest" + echo "size=$size" + echo "run_id=$run_id" + echo "waited_seconds=$(($(date -u +%s) - started))" + } >> "$output_file" + echo "Selected exact-merge release artifact $artifact_id from Runtime Checks run $run_id." + exit 0 + done < <( + jq -cer \ + --arg name "$artifact_name" \ + --arg repository_id "$GITHUB_REPOSITORY_ID" \ + --arg head_sha "$GITHUB_PR_HEAD_SHA" ' + [.artifacts[] + | select(.name == $name) + | select(.expired == false) + | select((.id | type) == "number" and .id > 0) + | select((.size_in_bytes | type) == "number" and .size_in_bytes > 0) + | select((.digest | type) == "string" and (.digest | test("^sha256:[0-9a-f]{64}$"))) + | select(.workflow_run.head_sha == $head_sha) + | select((.workflow_run.head_repository_id | tostring) == $repository_id)] + | sort_by(.created_at) + | reverse[] + ' <<< "$artifacts" 2>/dev/null || true + ) + fi + + elapsed=$(($(date -u +%s) - started)) + if (( elapsed >= max_wait_seconds )); then + write_miss + echo "No exact-commit TypeScript release artifact appeared after ${elapsed}s; using the local build fallback." + exit 0 + fi + remaining=$((max_wait_seconds - elapsed)) + (( remaining > 10 )) || sleep "$remaining" + (( remaining <= 10 )) || sleep 10 +done diff --git a/.github/scripts/snapshot-artifact.sh b/.github/scripts/snapshot-artifact.sh index d2c977ecf6..857d765d18 100755 --- a/.github/scripts/snapshot-artifact.sh +++ b/.github/scripts/snapshot-artifact.sh @@ -126,6 +126,8 @@ select_artifact() { $artifacts.artifacts[] | select(.name == $name) | select(.expired == false) + | select((.size_in_bytes | type) == "number" and .size_in_bytes > 0) + | select((.digest | type) == "string" and (.digest | test("^sha256:[0-9a-f]{64}$"))) | select(.workflow_run.head_branch == $branch) | select(.workflow_run.repository_id == $repository_id) | select(.workflow_run.head_repository_id == $repository_id) @@ -156,6 +158,8 @@ select_artifact() { set_output "$output_file" artifact-id "$(jq -er '.id' <<<"$candidate")" set_output "$output_file" run-id "$(jq -er '.workflow_run.id' <<<"$candidate")" set_output "$output_file" producer-sha "$(jq -er '.workflow_run.head_sha' <<<"$candidate")" + set_output "$output_file" artifact-size-bytes "$(jq -er '.size_in_bytes' <<<"$candidate")" + set_output "$output_file" artifact-digest "$(jq -er '.digest' <<<"$candidate")" set_output "$output_file" created-at "$(jq -er '.created_at' <<<"$candidate")" set_output "$output_file" age-hours "$age_hours" if ((age_seconds > 36 * 3600)); then diff --git a/.github/scripts/test-classify-bittensor-e2e-changes.sh b/.github/scripts/test-classify-bittensor-e2e-changes.sh new file mode 100755 index 0000000000..fecd9536b9 --- /dev/null +++ b/.github/scripts/test-classify-bittensor-e2e-changes.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +classifier="$script_dir/classify-bittensor-e2e-changes.sh" +builder="$script_dir/build-bittensor-e2e-matrix.py" +extractor="$script_dir/extract-pull-file-paths.sh" +workflow="$script_dir/../workflows/check-bittensor-e2e-tests.yml" +publisher="$script_dir/publish-localnet-manifest.sh" +output=$(mktemp) +paths=$(mktemp) +trap 'rm -f "$output" "$paths"' EXIT + +value() { + sed -n "s/^$1=//p" "$output" +} + +classify() { + : > "$output" + printf '%s\n' "$@" | "$classifier" "$output" +} + +assert_value() { + local key=$1 expected=$2 actual + actual=$(value "$key") + if [[ "$actual" != "$expected" ]]; then + echo "expected $key=$expected, got $actual" >&2 + cat "$output" >&2 + exit 1 + fi +} + +classify README.md sdk/python/bittensor/core.py +assert_value e2e false +assert_value build_image false + +classify sdk/bittensor-core-py/src/lib.rs sdk/bittensor-core-wasm/src/lib.rs +assert_value e2e false +assert_value build_image false + +classify sdk/bittensor-core/src/runtime/mod.rs +assert_value e2e true +assert_value build_image false + +classify sdk/bittensor-core/tests/e2e.rs +assert_value e2e true +assert_value build_image false + +classify sdk/bittensor-core/tests/Dockerfile.localnet-fast +assert_value e2e true +assert_value build_image true + +classify pallets/subtensor/src/staking/stake.rs +assert_value e2e true +assert_value build_image true + +classify pallets/subtensor/src/tests/staking.rs pallets/shield/src/mock.rs +assert_value e2e false +assert_value build_image false + +classify pallets/swap/src/pallet/tests.rs node/tests/chain_spec.rs runtime/tests/metadata.rs +assert_value e2e false +assert_value build_image false + +classify pallets/subtensor/src/migrations/migrate_staking.rs +assert_value e2e false +assert_value build_image false + +classify pallets/shield/src/migrations/migrate_clear_v1_storage.rs +assert_value e2e false +assert_value build_image false + +classify pallets/shield/src/benchmarking.rs node/src/benchmarking.rs support/weight-tools/src/weight_compare.rs +assert_value e2e false +assert_value build_image false + +classify pallets/subtensor/README.md precompiles/src/solidity/staking.sol +assert_value e2e false +assert_value build_image false + +classify Cargo.lock +assert_value e2e true +assert_value build_image true + +classify .cargo/config.toml +assert_value e2e true +assert_value build_image true + +classify scripts/localnet.sh snapshot.json .github/actions/rust-setup/action.yml .github/workflows/docker-localnet.yml +assert_value e2e true +assert_value build_image true + +classify .github/scripts/rust-setup-preflight.sh .github/scripts/install-rust-toolchain.sh +assert_value e2e true +assert_value build_image true + +classify sdk/new-chain-client/src/lib.rs +assert_value e2e true +assert_value build_image true + +# Preserve the previous path on renames so production code cannot be moved out +# of a covered tree and silently avoid the suite. +printf '%s\n' \ + '[{"filename":"docs/moved.rs","previous_filename":"runtime/src/lib.rs"}]' \ + | "$extractor" 1 > "$paths" +: > "$output" +"$classifier" "$output" < "$paths" +assert_value e2e true +assert_value build_image true + +: > "$output" +"$classifier" --all "$output" +assert_value e2e true +assert_value build_image true + +# The real manifest must produce 32 non-empty shards containing every test +# exactly once. With 112 tests, balanced round-robin shards contain 3 or 4. +: > "$output" +"$builder" "$repo_root/sdk/bittensor-core/tests/e2e-manifest.json" "$output" +python3 - "$output" <<'PY' +import json, sys + +values = dict(line.rstrip("\n").split("=", 1) for line in open(sys.argv[1], encoding="utf-8")) +matrix = json.loads(values["test_matrix"])["include"] +tests = [test for shard in matrix for test in shard["tests"]] +assert values["test_count"] == "112" +assert values["shard_count"] == "32" +assert len(matrix) == 32 +assert len(tests) == len(set(tests)) == 112 +assert {len(shard["tests"]) for shard in matrix} == {3, 4} +assert [shard["shard"] for shard in matrix] == list(range(1, 33)) +PY + +# Routing decisions execute trusted base-branch code and fail closed while the +# classifier is first being introduced. Required coverage stays at 112 tests. +grep -Fq "ref: \${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}" "$workflow" +grep -Fq 'trusted Rust SDK E2E classifier unavailable; running the full suite' "$workflow" +grep -Fq 'trusted Rust SDK E2E classifier failed or emitted invalid outputs; running the full suite' "$workflow" +grep -Fq 'max-parallel: 32' "$workflow" +if grep -Fq 'needs.plan.outputs.test_count == '\''112'\''' "$workflow"; then + echo "the Rust SDK E2E consumer must not duplicate the builder's exact test-count contract" >&2 + exit 1 +fi +grep -Fq 'TESTS_JSON: ${{ toJSON(matrix.tests) }}' "$workflow" +grep -Fq 'IMAGE_REF: ${{ needs.build-localnet-image.outputs.image_ref || needs.plan.outputs.base_image_ref }}' "$workflow" +grep -Fq 'base_image_tag="$LOCALNET_IMAGE_REPOSITORY:sha-$BASE_SHA"' "$workflow" +grep -Fq -- '--tag "$IMAGE:sha-$SHA"' "$publisher" +if sed -n '/^ pull_request:/,/^ workflow_dispatch:/p' "$workflow" \ + | grep -Eq '^[[:space:]]+paths:'; then + echo "Rust SDK E2E must always reach its fail-closed classifier" >&2 + exit 1 +fi + +echo "Rust SDK E2E classifier and shard tests passed" diff --git a/.github/scripts/test-classify-typescript-e2e-changes.sh b/.github/scripts/test-classify-typescript-e2e-changes.sh new file mode 100755 index 0000000000..902b65f3e2 --- /dev/null +++ b/.github/scripts/test-classify-typescript-e2e-changes.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +classifier="$script_dir/classify-typescript-e2e-changes.sh" +extractor="$script_dir/extract-pull-file-paths.sh" +workflow="$script_dir/../workflows/typescript-e2e.yml" +suite_registry="$repo_root/ts-tests/e2e-suite-ownership.json" +scheduled_smoke_workflow="$repo_root/.github/workflows/scheduled-smoke-tests.yml" +output=$(mktemp) +paths=$(mktemp) +mutated_config= +validation_error= +trap 'rm -f "$output" "$paths" "${mutated_config:-}" "${validation_error:-}"' EXIT + +value() { + sed -n "s/^$1=//p" "$output" +} + +classify() { + : > "$output" + printf '%s\n' "$@" | "$classifier" "$output" +} + +assert_value() { + local key=$1 expected=$2 actual + actual=$(value "$key") + if [[ "$actual" != "$expected" ]]; then + echo "expected $key=$expected, got $actual" >&2 + cat "$output" >&2 + exit 1 + fi +} + +classify README.md +assert_value e2e false +assert_value topology_audit false + +classify Cargo.lock +assert_value e2e true +assert_value evm true +assert_value staking true +assert_value shield true +assert_value topology_audit false + +classify .cargo/config.toml +assert_value e2e true +assert_value evm true +assert_value shield true + +classify .github/scripts/rust-setup-preflight.sh .github/scripts/install-rust-toolchain.sh +assert_value e2e true +assert_value evm true +assert_value shield true +assert_value topology_audit false + +# GitHub reports only the destination as filename for a rename. Preserve the +# previous path so moving production code out of a covered tree cannot bypass +# the E2E matrix. +printf '%s\n%s\n' \ + '[{"filename":"docs/moved.rs","previous_filename":"pallets/subtensor/src/staking.rs"}]' \ + '[{"filename":"README.md"}]' \ + | "$extractor" 2 > "$paths" +grep -qx 'docs/moved.rs' "$paths" +grep -qx 'pallets/subtensor/src/staking.rs' "$paths" +: > "$output" +"$classifier" "$output" < "$paths" +assert_value evm true +assert_value shield true + +if printf '%s\n' '[{"filename":"README.md"}]' | "$extractor" 2 >/dev/null 2>&1; then + echo "expected an incomplete pull-file response to fail closed" >&2 + exit 1 +fi + +classify ts-tests/suites/zombienet_evm/precompile.test.ts +assert_value evm true +assert_value staking false +assert_value shield false + +classify ts-tests/suites/zombienet_staking/staking.test.ts ts-tests/suites/zombienet_shield/shield.test.ts +assert_value staking true +assert_value shield true +assert_value evm false + +classify ts-tests/suites/zombienet_coldkey_swap/swap.test.ts ts-tests/suites/zombienet_subnets/register.test.ts +assert_value coldkey_swap true +assert_value subnets true +assert_value dev false + +classify ts-tests/suites/dev/staking.test.ts +assert_value dev true +assert_value evm false + +# The ownership registry is the structural completeness boundary. Every suite +# directory must be present, every PR-owned suite must route through its named +# selector, and scheduled suites stay explicitly visible instead of becoming +# an accidental omission hidden by the generic ts-tests fallback. +actual_suites=$( + for directory in "$repo_root"/ts-tests/suites/*; do + [[ -d "$directory" ]] && basename "$directory" + done | sort +) +registered_suites=$(jq -r '.suites | keys[]' "$suite_registry") +if [[ "$actual_suites" != "$registered_suites" ]]; then + echo "TypeScript E2E suite ownership is incomplete." >&2 + diff -u <(printf '%s\n' "$registered_suites") <(printf '%s\n' "$actual_suites") >&2 || true + echo "Fix: add or remove the matching entry in ts-tests/e2e-suite-ownership.json." >&2 + echo "For PR coverage, set owner=pull_request and add its selector to the classifier and e2e-shard-plan.mjs; for scheduled smoke coverage, set owner=scheduled and selector=null." >&2 + echo "Verify: node ts-tests/scripts/validate-e2e-config.mjs && node ts-tests/scripts/test-e2e-shard-plan.mjs && .github/scripts/test-classify-typescript-e2e-changes.sh" >&2 + exit 1 +fi + +while IFS=$'\t' read -r suite selector; do + classify "ts-tests/suites/$suite/future.test.ts" + assert_value e2e true + assert_value "$selector" true + if ! grep -Fq "steps.filter.outputs.$selector" "$workflow"; then + echo "PR-owned TypeScript E2E selector is not wired into the workflow: $selector" >&2 + echo "Fix: expose steps.filter.outputs.$selector to the planner in .github/workflows/typescript-e2e.yml." >&2 + echo "Verify: .github/scripts/test-classify-typescript-e2e-changes.sh" >&2 + exit 1 + fi +done < <(jq -r '.suites | to_entries[] | select(.value.owner == "pull_request") | [.key, .value.selector] | @tsv' "$suite_registry") + +while IFS= read -r environment; do + if ! grep -Fq "$environment" "$scheduled_smoke_workflow"; then + echo "Scheduled Moonwall environment has no executing workflow: $environment" >&2 + echo "Fix: add $environment to the test matrix in .github/workflows/scheduled-smoke-tests.yml, or change its ownership in ts-tests/e2e-suite-ownership.json." >&2 + echo "Verify: .github/scripts/test-classify-typescript-e2e-changes.sh" >&2 + exit 1 + fi +done < <(jq -r '.suites[] | select(.owner == "scheduled") | .environments[]' "$suite_registry") + +# Core production code is cross-suite and must retain the complete selection. +for path in \ + pallets/subtensor/src/staking/stake.rs \ + pallets/subtensor/src/subnets/registration.rs \ + pallets/subtensor/src/swap/swap_stake.rs; do + classify "$path" + for suite in evm staking coldkey_swap dev subnets shield; do + assert_value "$suite" true + done +done + +classify precompiles/subtensor/src/lib.rs +assert_value evm true +assert_value dev false +assert_value shield false + +classify pallets/shield/src/lib.rs +assert_value shield true +assert_value dev true +assert_value evm false + +for path in \ + pallets/subtensor/src/tests/staking.rs \ + pallets/shield/src/tests.rs \ + pallets/drand/src/mock.rs \ + chain-extensions/src/tests.rs \ + chain-extensions/src/mock.rs \ + precompiles/src/mock.rs \ + node/tests/chain_spec.rs \ + runtime/tests/metadata.rs \ + support/macros/tests/tests.rs \ + support/procedural-fork/src/pallet/parse/tests/tasks.rs \ + pallets/subtensor/src/migrations/migrate_staking.rs; do + classify "$path" + assert_value e2e false +done + +for path in \ + ts-tests/e2e-shards.json \ + ts-tests/e2e-suite-ownership.json \ + ts-tests/moonwall.config.json \ + ts-tests/configs/zombie_single_node.json \ + ts-tests/configs/zombie_extended.json \ + ts-tests/scripts/e2e-shard-plan.mjs \ + .github/workflows/typescript-e2e.yml \ + .github/actions/run-typescript-e2e/action.yml; do + classify "$path" + assert_value e2e true + assert_value topology_audit true + assert_value evm true + assert_value shield true +done + +: > "$output" +"$classifier" --all "$output" +assert_value e2e true +while IFS= read -r selector; do + assert_value "$selector" true +done < <(jq -r '.suites[] | select(.owner == "pull_request") | .selector' "$suite_registry") +assert_value topology_audit false + +# Canonical environments must discover their complete owned directory. A +# partial include allowlist would otherwise make Moonwall pass while silently +# omitting tests that are not represented in the sharded manifest. +mutated_config=$(mktemp) +validation_error=$(mktemp) +jq '(.environments[] | select(.name == "zombienet_coldkey_swap")).include = ["suites/zombienet_coldkey_swap/00-coldkey-swap.test.ts"]' \ + "$repo_root/ts-tests/moonwall.config.json" > "$mutated_config" +if E2E_CONFIG_PATH="$mutated_config" node "$repo_root/ts-tests/scripts/validate-e2e-config.mjs" >"$validation_error" 2>&1; then + echo "expected a partial canonical include list to fail E2E ownership validation" >&2 + exit 1 +fi +grep -Fq 'canonical environment "zombienet_coldkey_swap" may not define include' "$validation_error" +rm -f "$mutated_config" "$validation_error" +mutated_config= +validation_error= + +# Both the E2E and Runtime classifiers treat these conventional Rust paths as +# unit-test-only. Fail this routing contract if a future pallet introduces one +# without a file- or module-level cfg(test) gate. +assert_cfg_test_module() { + local lib="$1" module="$2" + MODULE="$module" perl -0777 -e ' + my $module = quotemeta($ENV{MODULE}); + my $source = <>; + my $pattern = qr/#\[cfg\(test\)\]\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+$module\s*;/s; + exit($source =~ $pattern ? 0 : 1); + ' "$lib" +} + +shopt -s nullglob +for file in "$repo_root"/pallets/*/src/mock.rs "$repo_root"/pallets/*/src/tests.rs; do + module=${file##*/} + module=${module%.rs} + lib=${file%/*}/lib.rs + if ! grep -Eq '^[[:space:]]*#!\[cfg\(test\)\][[:space:]]*$' "$file" && + ! assert_cfg_test_module "$lib" "$module"; then + echo "classifier-ignored path is not cfg(test)-gated: ${file#"$repo_root"/}" >&2 + exit 1 + fi +done +for directory in "$repo_root"/pallets/*/src/tests; do + lib=${directory%/tests}/lib.rs + if ! assert_cfg_test_module "$lib" tests; then + echo "classifier-ignored directory is not cfg(test)-gated: ${directory#"$repo_root"/}" >&2 + exit 1 + fi +done +for file in \ + "$repo_root"/chain-extensions/src/mock.rs \ + "$repo_root"/chain-extensions/src/tests.rs \ + "$repo_root"/precompiles/src/mock.rs; do + module=${file##*/} + module=${module%.rs} + lib=${file%/*}/lib.rs + if ! assert_cfg_test_module "$lib" "$module"; then + echo "classifier-ignored path is not cfg(test)-gated: ${file#"$repo_root"/}" >&2 + exit 1 + fi +done +if ! assert_cfg_test_module \ + "$repo_root/support/procedural-fork/src/pallet/parse/mod.rs" tests; then + echo "classifier-ignored path is not cfg(test)-gated: support/procedural-fork/src/pallet/parse/tests" >&2 + exit 1 +fi + +# Routing policy and matrix topology must be separate: trusted base code picks +# suites, while a trusted generic builder expands the proposed data manifest. +grep -Fq 'ref: ${{ github.event_name == '\''pull_request'\'' && github.event.pull_request.base.sha || github.sha }}' "$workflow" +grep -Fq '.trusted-e2e-filter/ts-tests/scripts/e2e-shard-plan.mjs' "$workflow" +grep -Fq '.proposed-e2e-plan/ts-tests/e2e-shards.json' "$workflow" +grep -Fq 'matrix: ${{ fromJSON(needs.changes.outputs.shield_matrix) }}' "$workflow" +grep -Fq 'name: Audit canonical unsharded ${{ matrix.test }}' "$workflow" +grep -Fq 'EVM_SELECTED: ${{ needs.changes.outputs.evm }}' "$workflow" +grep -Fq 'SHIELD_SELECTED: ${{ needs.changes.outputs.shield }}' "$workflow" + +echo "typescript E2E change classifier tests passed" diff --git a/.github/scripts/test-clone-regression-phase.sh b/.github/scripts/test-clone-regression-phase.sh new file mode 100755 index 0000000000..6c86f22fed --- /dev/null +++ b/.github/scripts/test-clone-regression-phase.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +source_script="$repo_root/clones/scripts/run-clone-regression-phase.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +mkdir -p "$tmp/repo/clones/scripts" "$tmp/repo/clones/js-tests" "$tmp/repo/sdk/python" "$tmp/bin" +cp "$source_script" "$tmp/repo/clones/scripts/" + +for helper in start-local-clone-and-wait.sh stop-local-clone.sh local-clone-checkpoint.sh; do + cat > "$tmp/repo/clones/scripts/$helper" <<'EOF' +#!/usr/bin/env bash +printf '%s %s\n' "$(basename "$0")" "$*" >> "$HARNESS_LOG" +EOF + chmod +x "$tmp/repo/clones/scripts/$helper" +done + +cat > "$tmp/bin/npm" <<'EOF' +#!/usr/bin/env bash +printf 'npm %s phase=%s cwd=%s\n' "$*" "${CLONE_REGRESSION_PHASE:-}" "$PWD" >> "$HARNESS_LOG" +if [[ -n "${MOCK_NPM_FAIL_ONCE:-}" && "$*" == *"$MOCK_NPM_FAIL_ONCE"* && ! -e "$MOCK_NPM_STATE" ]]; then + : > "$MOCK_NPM_STATE" + exit 1 +fi +if [[ -n "${MOCK_NPM_FAIL:-}" && "$*" == *"$MOCK_NPM_FAIL"* ]]; then + exit 1 +fi +EOF +cat > "$tmp/bin/sleep" <<'EOF' +#!/usr/bin/env bash +printf 'sleep %s\n' "$*" >> "$HARNESS_LOG" +EOF +cat > "$tmp/bin/uv" <<'EOF' +#!/usr/bin/env bash +printf 'uv %s cwd=%s\n' "$*" "$PWD" >> "$HARNESS_LOG" +EOF +chmod +x "$tmp/bin/npm" "$tmp/bin/sleep" "$tmp/bin/uv" "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" + +export PATH="$tmp/bin:$PATH" +export HARNESS_LOG="$tmp/harness.log" +export MOCK_NPM_STATE="$tmp/npm-state" +checkpoint="$tmp/checkpoint.tar" +: > "$checkpoint" + +run_phase() { + : > "$HARNESS_LOG" + RUN_SDK_DRIFT=false CLONE_CHECKPOINT="$checkpoint" \ + "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" "$1" +} + +run_phase pristine +grep -Fq 'start-local-clone-and-wait.sh accelerated' "$HARNESS_LOG" +grep -Fq 'npm run runtime:update:alice' "$HARNESS_LOG" +grep -Fq 'npm run test:clone-regressions phase=pristine' "$HARNESS_LOG" +grep -Fq 'stop-local-clone.sh ' "$HARNESS_LOG" +if grep -Fq 'npm test' "$HARNESS_LOG"; then + echo "pristine phase unexpectedly ran remaining smoke tests" >&2 + exit 1 +fi + +run_phase remaining +grep -Fq 'npm test phase=' "$HARNESS_LOG" +grep -Fq 'npm run test:clone-regressions phase=remaining' "$HARNESS_LOG" + +run_phase combined +[[ $(grep -Fc 'start-local-clone-and-wait.sh accelerated' "$HARNESS_LOG") -eq 2 ]] +grep -Fq "local-clone-checkpoint.sh restore $checkpoint" "$HARNESS_LOG" +grep -Fq 'npm run test:clone-regressions phase=pristine' "$HARNESS_LOG" +grep -Fq 'npm run test:clone-regressions phase=remaining' "$HARNESS_LOG" + +: > "$HARNESS_LOG" +RUN_SDK_DRIFT=true "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" remaining +grep -Fq 'uv sync --locked --all-extras --dev' "$HARNESS_LOG" +grep -Fq 'uv run python -m codegen.check --drift ws://127.0.0.1:9944' "$HARNESS_LOG" + +: > "$HARNESS_LOG" +rm -f "$MOCK_NPM_STATE" +MOCK_NPM_FAIL_ONCE=runtime:update:alice \ + RUN_SDK_DRIFT=false \ + "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" remaining +[[ $(grep -Fc 'npm run runtime:update:alice' "$HARNESS_LOG") -eq 2 ]] +grep -Fq 'sleep 15' "$HARNESS_LOG" + +: > "$HARNESS_LOG" +if MOCK_NPM_FAIL=test:clone-regressions \ + RUN_SDK_DRIFT=false \ + "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" pristine; then + echo "failed clone regression phase unexpectedly succeeded" >&2 + exit 1 +fi +grep -Fq 'stop-local-clone.sh ' "$HARNESS_LOG" + +if "$tmp/repo/clones/scripts/run-clone-regression-phase.sh" invalid >/dev/null 2>&1; then + echo "invalid clone phase was accepted" >&2 + exit 1 +fi + +echo "clone regression phase harness tests passed" diff --git a/.github/scripts/test-download-artifact.sh b/.github/scripts/test-download-artifact.sh new file mode 100755 index 0000000000..785d6f02b5 --- /dev/null +++ b/.github/scripts/test-download-artifact.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +helper="$script_dir/download-artifact.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" "$tmp/payload" +printf 'snapshot payload\n' > "$tmp/payload/mainnet-snapshot.tar.gz" +(cd "$tmp/payload" && zip -q "$tmp/artifact.zip" mainnet-snapshot.tar.gz) +size=$(stat -c '%s' "$tmp/artifact.zip" 2>/dev/null || stat -f '%z' "$tmp/artifact.zip") +digest="sha256:$(sha256sum "$tmp/artifact.zip" | awk '{print $1}')" + +cat > "$tmp/metadata.json" <<'EOF' +{"schema_version":1,"endpoint":"http://192.168.128.1:8093","repository_id":608683796} +EOF + +cat > "$tmp/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +output='' +headers='' +url='' +cache_only=false +for ((index=1; index<=$#; index++)); do + value="${!index}" + case "$value" in + --output) next=$((index + 1)); output="${!next}" ;; + --dump-header) next=$((index + 1)); headers="${!next}" ;; + --header) + next=$((index + 1)) + [[ "${!next}" != 'Cache-Control: only-if-cached' ]] || cache_only=true + ;; + http://*|https://*) url="$value" ;; + esac +done +case "$url" in + */token) printf 'mmds-token' ;; + */artifact-cache) cp "$MOCK_METADATA" "$output" ;; + http://192.168.128.1:8093/*) + [[ "$cache_only" == true ]] || { echo 'local request was not cache-only' >&2; exit 2; } + [[ "${MOCK_LOCAL_FAIL:-false}" != true ]] || exit 22 + cp "$MOCK_ARCHIVE" "$output" + printf 'HTTP/1.1 200 OK\r\nX-Fireactions-Cache: hit\r\n\r\n' > "$headers" + ;; + https://api.github.com/*) cp "$MOCK_ARCHIVE" "$output" ;; + *) echo "unexpected mock curl URL: $url" >&2; exit 2 ;; +esac +EOF +chmod +x "$tmp/bin/curl" + +export PATH="$tmp/bin:$PATH" +export MOCK_METADATA="$tmp/metadata.json" +export MOCK_ARCHIVE="$tmp/artifact.zip" +export GH_TOKEN=test-job-token +export GITHUB_REPOSITORY=RaoFoundation/subtensor +export GITHUB_REPOSITORY_ID=608683796 +export GITHUB_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export MMDS_TOKEN_URL=http://mmds/token +export ARTIFACT_CACHE_METADATA_URL=http://mmds/artifact-cache + +: > "$tmp/output" +"$helper" 123 mainnet-snapshot "$digest" "$size" "$tmp/local" "$tmp/output" >/dev/null +grep -qx 'source=local-hit' "$tmp/output" +cmp "$tmp/payload/mainnet-snapshot.tar.gz" "$tmp/local/mainnet-snapshot.tar.gz" + +: > "$tmp/output" +"$helper" 123 node-subtensor-release-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa "$digest" "$size" \ + "$tmp/release" "$tmp/output" >/dev/null +grep -qx 'source=local-hit' "$tmp/output" +cmp "$tmp/payload/mainnet-snapshot.tar.gz" "$tmp/release/mainnet-snapshot.tar.gz" + +: > "$tmp/output" +"$helper" 123 try-runtime-snap-v0.10.1-mainnet "$digest" "$size" \ + "$tmp/try-runtime" "$tmp/output" >/dev/null +grep -qx 'source=local-hit' "$tmp/output" +cmp "$tmp/payload/mainnet-snapshot.tar.gz" "$tmp/try-runtime/mainnet-snapshot.tar.gz" + +: > "$tmp/output" +export MOCK_LOCAL_FAIL=true +"$helper" 123 mainnet-snapshot "$digest" "$size" "$tmp/direct" "$tmp/output" >/dev/null 2>"$tmp/fallback.log" +grep -qx 'source=github' "$tmp/output" +cmp "$tmp/payload/mainnet-snapshot.tar.gz" "$tmp/direct/mainnet-snapshot.tar.gz" +unset MOCK_LOCAL_FAIL + +if "$helper" 123 mainnet-snapshot sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff \ + "$size" "$tmp/bad-digest" "$tmp/output" >/dev/null 2>&1; then + echo "expected digest mismatch to fail" >&2 + exit 1 +fi + +if "$helper" 123 untrusted-artifact "$digest" "$size" \ + "$tmp/unsupported" "$tmp/output" >/dev/null 2>&1; then + echo "expected unsupported artifact name to fail" >&2 + exit 1 +fi + +export RUNNER_TEMP="$tmp/runner" +export GITHUB_STEP_SUMMARY="$tmp/summary" +export ARTIFACT_ID=123 +export ARTIFACT_DIGEST="$digest" +export ARTIFACT_SIZE="$size" +mkdir -p "$RUNNER_TEMP" +: > "$GITHUB_STEP_SUMMARY" +"$script_dir/benchmark-artifact-cache.sh" >/dev/null +grep -q '^### Mainnet snapshot artifact cache$' "$GITHUB_STEP_SUMMARY" +grep -q '^- Cache-only probe 1: .* (local-hit)$' "$GITHUB_STEP_SUMMARY" +grep -q '^- Cache-only probe 2: .* (local-hit)$' "$GITHUB_STEP_SUMMARY" + +echo "artifact download helper tests passed" diff --git a/.github/scripts/test-prewarm-exact-runtime.sh b/.github/scripts/test-prewarm-exact-runtime.sh new file mode 100755 index 0000000000..3c37f687e2 --- /dev/null +++ b/.github/scripts/test-prewarm-exact-runtime.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" "$tmp/runner" + +cat > "$tmp/bin/cargo" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${1:-}" in + clean) exit 0 ;; + check) + [[ "$*" == "check --locked -p node-subtensor-runtime" ]] + ;; + *) exit 2 ;; +esac +EOF + +cat > "$tmp/bin/sccache" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${1:-}" in + --zero-stats) ;; + --show-stats) + cat < "$GITHUB_STEP_SUMMARY" +"$script_dir/prewarm-exact-runtime.sh" >/dev/null +grep -q '^### Exact runtime-only R2 prewarm$' "$GITHUB_STEP_SUMMARY" +grep -q 'Clean verification: .*600 Rust hits, 0 misses' "$GITHUB_STEP_SUMMARY" + +if MOCK_RUST_MISSES=11 "$script_dir/prewarm-exact-runtime.sh" >/dev/null 2>&1; then + echo "expected excessive exact-key misses to fail verification" >&2 + exit 1 +fi + +echo "exact runtime prewarm tests passed" diff --git a/.github/scripts/test-r2-artifact-mirror.py b/.github/scripts/test-r2-artifact-mirror.py new file mode 100755 index 0000000000..8557838f62 --- /dev/null +++ b/.github/scripts/test-r2-artifact-mirror.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +SCRIPT = Path(__file__).with_name("r2-artifact-mirror.py") +PUBLISH_HELPER = Path(__file__).with_name("publish-artifact-mirror.sh") +CURRENT_RUN_HELPER = Path(__file__).with_name( + "publish-current-run-artifact-mirror.sh" +) +SPEC = importlib.util.spec_from_file_location("r2_artifact_mirror", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +assert MODULE.ALLOWED_ARTIFACT_NAMES == { + "mainnet-snapshot", + "try-runtime-snap-v0.10.1-mainnet", + "try-runtime-snap-v0.10.1-testnet", + "try-runtime-snap-v0.10.1-devnet", +} + + +assert MODULE.validate_endpoint( + "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com" +) == "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com" + +for endpoint in ( + "http://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com", + "https://example.com", + "https://00000000000000000000000000000000.r2.cloudflarestorage.com", + "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com/escape", + "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com:444", +): + try: + MODULE.validate_endpoint(endpoint) + except MODULE.MirrorError: + pass + else: + raise AssertionError(f"unsafe endpoint was accepted: {endpoint}") + +with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "artifact.zip" + archive.write_bytes(b"artifact archive") + digest, size = MODULE.file_sha256(archive) + args = SimpleNamespace( + workflow_path=".github/workflows/refresh-mainnet-snapshot.yml", + artifact_id=123, + artifact_name="mainnet-snapshot", + digest=f"sha256:{digest}", + producer_sha="a" * 40, + ) + environment = { + "GITHUB_REPOSITORY": "RaoFoundation/subtensor", + "GITHUB_REPOSITORY_ID": "608683796", + } + with ( + mock.patch.dict(os.environ, environment, clear=True), + mock.patch.object(MODULE.time, "time", return_value=1_768_476_000), + ): + manifest = json.loads( + MODULE.build_manifest(args, f"artifacts/v1/objects/123-{digest}.zip", size) + ) + assert manifest == { + "schema_version": 1, + "repository": "RaoFoundation/subtensor", + "repository_id": 608683796, + "workflow_path": ".github/workflows/refresh-mainnet-snapshot.yml", + "artifact_id": 123, + "artifact_name": "mainnet-snapshot", + "digest": f"sha256:{digest}", + "size_in_bytes": size, + "object_key": f"artifacts/v1/objects/123-{digest}.zip", + "producer_sha": "a" * 40, + "published_at": 1_768_476_000, + } + + for artifact_name in MODULE.ALLOWED_ARTIFACT_NAMES: + parsed = MODULE.parse_args( + [ + str(archive), + "123", + artifact_name, + f"sha256:{digest}", + "a" * 40, + ".github/workflows/refresh-mainnet-snapshot.yml", + ] + ) + assert parsed.artifact_name == artifact_name + + with mock.patch.object(sys, "stderr", io.StringIO()): + try: + MODULE.parse_args( + [ + str(archive), + "123", + "untrusted-artifact", + f"sha256:{digest}", + "a" * 40, + ".github/workflows/refresh-mainnet-snapshot.yml", + ] + ) + except SystemExit as error: + assert error.code == 2 + else: + raise AssertionError("untrusted mirror artifact was accepted") + + publisher_environment = { + "SCCACHE_BUCKET": "subtensor-ci-sccache", + "SCCACHE_REGION": "auto", + "SCCACHE_ENDPOINT": "https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com", + "AWS_ACCESS_KEY_ID": "test-access-key", + "AWS_SECRET_ACCESS_KEY": "test-secret-key", + } + with ( + mock.patch.dict(os.environ, publisher_environment, clear=True), + mock.patch.object(MODULE.shutil, "which", return_value="/usr/bin/aws"), + mock.patch.object( + MODULE.subprocess, + "run", + return_value=SimpleNamespace(returncode=0, stderr=""), + ) as run, + ): + MODULE.R2Publisher().put("artifacts/v1/objects/123.zip", archive) + command = run.call_args.args[0] + assert command[:3] == ["/usr/bin/aws", "s3api", "put-object"] + assert command[command.index("--bucket") + 1] == "subtensor-ci-sccache" + assert command[command.index("--key") + 1] == "artifacts/v1/objects/123.zip" + assert "test-access-key" not in command + assert "test-secret-key" not in command + + with ( + mock.patch.dict(os.environ, publisher_environment, clear=True), + mock.patch.object(MODULE.shutil, "which", return_value=None), + ): + try: + MODULE.R2Publisher() + except MODULE.MirrorError: + pass + else: + raise AssertionError("missing AWS CLI was accepted") + + with ( + mock.patch.dict(os.environ, publisher_environment, clear=True), + mock.patch.object(MODULE.shutil, "which", return_value="/usr/bin/aws"), + mock.patch.object( + MODULE.subprocess, + "run", + return_value=SimpleNamespace(returncode=1, stderr="secret response"), + ), + ): + try: + MODULE.R2Publisher().put("artifacts/v1/objects/123.zip", archive) + except MODULE.MirrorError as error: + assert "secret response" not in str(error) + else: + raise AssertionError("failed AWS CLI upload was accepted") + +with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + bin_dir = temp / "bin" + bin_dir.mkdir() + record = temp / "publisher-arguments" + (bin_dir / "gh").write_text( + "#!/usr/bin/env bash\nprintf 'immutable artifact zip'\n", + encoding="utf-8", + ) + (bin_dir / "python3").write_text( + """#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == */r2-artifact-mirror.py ]] +[[ -s "$2" ]] +printf '%s\n' "${@:3}" > "$PUBLISH_RECORD" +""", + encoding="utf-8", + ) + (bin_dir / "gh").chmod(0o755) + (bin_dir / "python3").chmod(0o755) + result = subprocess.run( + [ + str(PUBLISH_HELPER), + "123", + "mainnet-snapshot", + f"sha256:{'a' * 64}", + "b" * 40, + ".github/workflows/refresh-mainnet-snapshot.yml", + ], + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "GH_TOKEN": "token", + "GITHUB_REPOSITORY": "example/repository", + "PUBLISH_RECORD": str(record), + }, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert record.read_text(encoding="utf-8").splitlines() == [ + "123", + "mainnet-snapshot", + f"sha256:{'a' * 64}", + "b" * 40, + ".github/workflows/refresh-mainnet-snapshot.yml", + ] + +with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + bin_dir = temp / "bin" + bin_dir.mkdir() + record = temp / "publisher-arguments" + metadata = temp / "metadata.json" + metadata.write_text( + json.dumps( + { + "artifacts": [ + { + "id": 123, + "name": "try-runtime-snap-v0.10.1-mainnet", + "expired": False, + "digest": f"sha256:{'a' * 64}", + } + ] + } + ), + encoding="utf-8", + ) + (bin_dir / "gh").write_text( + """#!/usr/bin/env bash +set -euo pipefail +case "$*" in + *actions/runs/789/artifacts*) cat "$MOCK_METADATA" ;; + *actions/artifacts/123/zip*) printf 'immutable artifact zip' ;; + *) echo "unexpected gh invocation: $*" >&2; exit 2 ;; +esac +""", + encoding="utf-8", + ) + (bin_dir / "python3").write_text( + """#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == */r2-artifact-mirror.py ]] +[[ -s "$2" ]] +printf '%s\n' "${@:3}" > "$PUBLISH_RECORD" +""", + encoding="utf-8", + ) + (bin_dir / "gh").chmod(0o755) + (bin_dir / "python3").chmod(0o755) + environment = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "GH_TOKEN": "token", + "GITHUB_REPOSITORY": "example/repository", + "GITHUB_RUN_ID": "789", + "GITHUB_SHA": "b" * 40, + "MOCK_METADATA": str(metadata), + "PUBLISH_RECORD": str(record), + } + result = subprocess.run( + [str(CURRENT_RUN_HELPER), "try-runtime-snap-v0.10.1-mainnet"], + env=environment, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert record.read_text(encoding="utf-8").splitlines() == [ + "123", + "try-runtime-snap-v0.10.1-mainnet", + f"sha256:{'a' * 64}", + "b" * 40, + ".github/workflows/refresh-mainnet-snapshot.yml", + ] + + duplicate = json.loads(metadata.read_text(encoding="utf-8")) + duplicate["artifacts"].append(dict(duplicate["artifacts"][0])) + metadata.write_text(json.dumps(duplicate), encoding="utf-8") + result = subprocess.run( + [str(CURRENT_RUN_HELPER), "try-runtime-snap-v0.10.1-mainnet"], + env=environment, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode != 0 + +print("R2 artifact mirror tests passed") diff --git a/.github/scripts/test-r2-sccache-warmset.py b/.github/scripts/test-r2-sccache-warmset.py new file mode 100755 index 0000000000..f32cbb9ec5 --- /dev/null +++ b/.github/scripts/test-r2-sccache-warmset.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +SCRIPT = Path(__file__).with_name("r2-sccache-warmset.py") +SPEC = importlib.util.spec_from_file_location("r2_sccache_warmset", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +first = "a" * 64 +second = "b" * 64 +third = "c" * 64 +assert MODULE.MAX_WARM_BYTES == 4 * 1024 * 1024 * 1024 + +with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "sccache.log" + keys = root / "keys.txt" + log.write_text( + "ignored\n" + f"[crate]: Hash key: {first}\n" + f"malformed Hash key: {'d' * 63}\n" + f"[crate]: Hash key: {second}\n" + f"[duplicate]: Hash key: {first}\n", + encoding="utf-8", + ) + assert MODULE.extract_hashes(log, keys) == 2 + assert keys.read_text(encoding="ascii").splitlines() == [first, second] + assert MODULE.load_hashes([keys]) == [first, second] + + malformed = root / "malformed.txt" + malformed.write_text("../escape\n", encoding="ascii") + try: + MODULE.load_hashes([malformed]) + except MODULE.WarmsetError: + pass + else: + raise AssertionError("malformed compiler cache key was accepted") + +inventory = { + f"{MODULE.COMPILER_PREFIX}/{MODULE.normalized_path(first)}": 10, + f"{MODULE.COMPILER_PREFIX}/{MODULE.normalized_path(second)}": 20, +} +manifest = MODULE.build_manifest([first, second], inventory, "d" * 40, 100) +assert manifest == { + "schema_version": 1, + "bucket": MODULE.BUCKET, + "key_prefix": MODULE.COMPILER_PREFIX, + "generation": f"{'d' * 40}-100", + "producer_sha": "d" * 40, + "published_at": 100, + "expires_at": 100 + MODULE.MANIFEST_TTL_SECONDS, + "max_bytes": MODULE.MAX_WARM_BYTES, + "captured_object_count": 2, + "captured_size_bytes": 30, + "selected_object_count": 2, + "selected_size_bytes": 30, + "objects": [ + {"path": MODULE.normalized_path(first), "size": 10}, + {"path": MODULE.normalized_path(second), "size": 20}, + ], +} + +try: + MODULE.build_manifest([first, third], inventory, "d" * 40, 100) +except MODULE.WarmsetError as error: + assert "absent" in str(error) +else: + raise AssertionError("incomplete durable cache set was accepted") + +with mock.patch.object(MODULE, "MAX_WARM_BYTES", 15): + capped = MODULE.build_manifest([first, second], inventory, "d" * 40, 100) +assert capped["selected_object_count"] == 1 +assert capped["selected_size_bytes"] == 10 + +publisher_environment = { + "SCCACHE_BUCKET": MODULE.BUCKET, + "SCCACHE_REGION": MODULE.REGION, + "SCCACHE_ENDPOINT": f"https://{MODULE.ACCOUNT_HOST}", + "AWS_ACCESS_KEY_ID": "test-access-key", + "AWS_SECRET_ACCESS_KEY": "test-secret-key", +} +with ( + mock.patch.dict(os.environ, publisher_environment, clear=True), + mock.patch.object(MODULE.shutil, "which", return_value="/usr/bin/aws"), +): + client = MODULE.R2Client() +assert client.endpoint == f"https://{MODULE.ACCOUNT_HOST}" +assert "test-access-key" not in repr(list(client.environment())) + + +def inventory_run(_command, **kwargs): + kwargs["stdout"].write( + json.dumps( + { + "Contents": [ + { + "Key": f"{MODULE.COMPILER_PREFIX}/.sccache_check", + "Size": 0, + }, + { + "Key": ( + f"{MODULE.COMPILER_PREFIX}/" + f"{MODULE.normalized_path(first)}" + ), + "Size": 10, + }, + ] + } + ).encode() + ) + return SimpleNamespace(returncode=0) + + +with ( + tempfile.TemporaryDirectory() as directory, + mock.patch.dict(os.environ, publisher_environment, clear=True), + mock.patch.object(MODULE.shutil, "which", return_value="/usr/bin/aws"), + mock.patch.object(MODULE.subprocess, "run", side_effect=inventory_run) as run, +): + client = MODULE.R2Client() + listed = client.inventory() + assert listed[f"{MODULE.COMPILER_PREFIX}/.sccache_check"] == 0 + assert listed[f"{MODULE.COMPILER_PREFIX}/{MODULE.normalized_path(first)}"] == 10 + body = Path(directory) / "manifest.json" + body.write_text("{}\n", encoding="utf-8") + run.reset_mock() + run.side_effect = None + run.return_value = SimpleNamespace(returncode=0) + client.put(f"{MODULE.MANIFEST_PREFIX}/latest.json", body) + command = run.call_args.args[0] + assert "test-access-key" not in command + assert "test-secret-key" not in command + assert command[command.index("--key") + 1] == ( + f"{MODULE.MANIFEST_PREFIX}/latest.json" + ) + +with tempfile.TemporaryDirectory() as directory: + key_file = Path(directory) / "keys.txt" + key_file.write_text(first + "\n", encoding="ascii") + published = [] + stub_client = SimpleNamespace( + inventory=lambda: { + f"{MODULE.COMPILER_PREFIX}/{MODULE.normalized_path(first)}": 10 + }, + put=lambda key, source: published.append( + (key, json.loads(source.read_text(encoding="utf-8"))) + ), + ) + with ( + mock.patch.dict( + os.environ, + {**publisher_environment, "GITHUB_SHA": "d" * 40}, + clear=True, + ), + mock.patch.object(MODULE, "R2Client", return_value=stub_client), + mock.patch.object(MODULE.time, "time", return_value=100), + ): + MODULE.publish([key_file]) + assert [key for key, _manifest in published] == [ + f"{MODULE.MANIFEST_PREFIX}/latest.json" + ] + assert published[0][1]["generation"] == f"{'d' * 40}-100" + +for endpoint in ( + f"http://{MODULE.ACCOUNT_HOST}", + "https://example.com", + f"https://{MODULE.ACCOUNT_HOST}/escape", +): + try: + MODULE.validate_endpoint(endpoint) + except MODULE.WarmsetError: + pass + else: + raise AssertionError(f"unsafe endpoint was accepted: {endpoint}") diff --git a/.github/scripts/test-runtime-change-filter.sh b/.github/scripts/test-runtime-change-filter.sh index 8ee778d3cb..543013ac96 100755 --- a/.github/scripts/test-runtime-change-filter.sh +++ b/.github/scripts/test-runtime-change-filter.sh @@ -4,6 +4,7 @@ set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) classifier="$script_dir/classify-runtime-changes.sh" +runtime_workflow="$script_dir/../workflows/runtime-checks.yml" tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT @@ -17,25 +18,75 @@ assert_classification() { all_false=$'runtime=false\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' runtime_only=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' runtime_and_sdk=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=true\nsnapshot_ci=false' +runtime_sdk_python=$'runtime=true\ndocs=false\npython_sdk=true\nsdk_drift=true\nsnapshot_ci=false' runtime_and_docs=$'runtime=true\ndocs=true\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' runtime_and_snapshot_only=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=true' runtime_and_snapshot=$'runtime=true\ndocs=true\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=true' docs_and_python=$'runtime=false\ndocs=true\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=false' python_only=$'runtime=false\ndocs=false\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=false' +docs_only=$'runtime=false\ndocs=true\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' assert_classification README.md "$all_false" +assert_classification pallets/subtensor/src/tests/staking.rs "$all_false" +assert_classification pallets/shield/src/tests.rs "$all_false" +assert_classification pallets/drand/src/mock.rs "$all_false" +assert_classification chain-extensions/src/tests.rs "$all_false" +assert_classification chain-extensions/src/mock.rs "$all_false" +assert_classification precompiles/src/mock.rs "$all_false" +assert_classification node/tests/chain_spec.rs "$all_false" +assert_classification runtime/tests/metadata.rs "$all_false" +assert_classification support/macros/tests/tests.rs "$all_false" +assert_classification support/procedural-fork/src/pallet/parse/tests/tasks.rs "$all_false" assert_classification .github/actions/rust-setup/action.yml "$runtime_only" assert_classification .github/actions/sccache-setup/action.yml "$runtime_only" +assert_classification .github/scripts/rust-setup-preflight.sh "$runtime_only" +assert_classification .github/scripts/install-rust-toolchain.sh "$runtime_only" +assert_classification .github/scripts/sccache-report.sh "$runtime_only" assert_classification .github/scripts/classify-runtime-changes.sh "$runtime_and_snapshot_only" assert_classification .github/scripts/test-runtime-change-filter.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/select-shared-release-artifact.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/test-select-shared-release-artifact.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/prewarm-exact-runtime.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/test-prewarm-exact-runtime.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/benchmark-sccache-paired.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/benchmark-artifact-cache.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/publish-current-run-artifact-mirror.sh "$runtime_and_snapshot_only" assert_classification clones/scripts/start-local-clone-and-wait.sh "$runtime_only" +assert_classification clones/scripts/run-clone-regression-phase.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/test-clone-regression-phase.sh "$runtime_and_snapshot_only" assert_classification .github/workflows/refresh-mainnet-snapshot.yml "$runtime_and_snapshot_only" assert_classification .github/workflows/runtime-checks.yml "$runtime_and_snapshot" assert_classification website/apps/bittensor-website/scripts/generate-metadata.mjs "$runtime_and_docs" +assert_classification docs/concepts/client.mdx "$docs_only" assert_classification sdk/bittensor-core/src/lib.rs "$python_only" -assert_classification Cargo.lock "$python_only" +assert_classification Cargo.lock "$runtime_sdk_python" +assert_classification .cargo/config.toml "$runtime_and_sdk" +assert_classification future-runtime-crate/src/lib.rs "$runtime_and_sdk" assert_classification rust-toolchain.toml "$runtime_and_sdk" assert_classification $'README.md\nsdk/python/example.py' "$docs_and_python" assert_classification $'README.md\nnode/src/renamed-service.rs' "$runtime_and_sdk" +# The snapshot-backed clone split must remain fail-closed: both independent +# phases run when a trusted artifact exists, while planner/fresh-state fallback +# preserves the complete sequential suite. These are static workflow contract +# checks; snapshot selection itself is exercised by test-snapshot-artifact.sh. +grep -Fq 'matrix={"phase":["pristine","remaining"]}' "$runtime_workflow" +grep -Fq 'matrix={"phase":["combined"]}' "$runtime_workflow" +grep -Fq "needs.clone-plan.outputs.matrix || '{\"phase\":[\"combined\"]}'" "$runtime_workflow" +grep -Fq './clones/scripts/run-clone-regression-phase.sh "${{ matrix.phase }}"' "$runtime_workflow" +grep -Fq 'RUN_SDK_DRIFT: ${{ github.event_name != '\''pull_request'\'' || needs.changes.outputs.sdk_drift == '\''true'\'' }}' "$runtime_workflow" +grep -Fq 'artifact_id: ${{ steps.plan.outputs.artifact_id }}' "$runtime_workflow" +grep -Fq 'ARTIFACT_ID: ${{ needs.clone-plan.outputs.artifact_id }}' "$runtime_workflow" +grep -Fq 'gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID"' "$runtime_workflow" +grep -Fq '"$(jq -er '\''.digest'\'' "$metadata")"' "$runtime_workflow" +grep -Fq '"$(jq -er '\''.size_in_bytes'\'' "$metadata")"' "$runtime_workflow" + +# A transient Files API outage must fail closed by selecting the full matrix, +# not fail the classifier before required aggregate contexts can be reported. +grep -Fq 'if ! pages=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --slurp); then' "$runtime_workflow" +grep -Fq '::warning::PR file listing failed; enabling every check.' "$runtime_workflow" +grep -Fq 'Runtime classifier failed or emitted invalid outputs; enabling every check.' "$runtime_workflow" +grep -Fq 'RUNTIME_RELEVANT: ${{ needs.changes.outputs.runtime }}' "$runtime_workflow" +grep -Fq '[ "$RUNTIME_RELEVANT" = "false" ]' "$runtime_workflow" + echo "runtime change filter tests passed" diff --git a/.github/scripts/test-rust-ci-paths.sh b/.github/scripts/test-rust-ci-paths.sh new file mode 100755 index 0000000000..2a7f38f165 --- /dev/null +++ b/.github/scripts/test-rust-ci-paths.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +registry="$repo_root/.github/rust-ci-paths.txt" +validator="$script_dir/validate-rust-ci-paths.sh" +classifier="$script_dir/classify-rust-changes.sh" +rust_workflow="$repo_root/.github/workflows/check-rust.yml" +docker_workflow="$repo_root/.github/workflows/check-docker.yml" +eco_workflow="$repo_root/.github/workflows/eco-tests.yml" +validation_workflow="$repo_root/.github/workflows/validate-sccache.yml" +incomplete=$(mktemp) +output=$(mktemp) +trap 'rm -f "$incomplete" "$output"' EXIT + +classify() { + : > "$output" + printf '%s\n' "$@" | "$classifier" "$output" + sed -n 's/^rust=//p' "$output" +} + +"$validator" "$registry" + +grep -vx 'sdk/bittensor-core-wasm' "$registry" > "$incomplete" +if "$validator" "$incomplete" > "$output" 2>&1; then + echo "expected an unowned Rust workspace package to fail validation" >&2 + exit 1 +fi +grep -Fq 'Rust workspace packages have no CI path owner:' "$output" +grep -Fq ' - sdk/bittensor-core-wasm' "$output" +grep -Fq 'Fix: add the narrowest stable parent directory' "$output" +grep -Fq 'Verify: .github/scripts/test-rust-ci-paths.sh' "$output" + +[[ "$(classify README.md)" == false ]] +[[ "$(classify sdk/python/README.md)" == false ]] +[[ "$(classify Cargo.toml)" == true ]] +[[ "$(classify future-workspace/member/Cargo.toml)" == true ]] +[[ "$(classify .cargo/config.toml)" == true ]] +[[ "$(classify .github/rust-ci-paths.txt)" == true ]] +[[ "$(classify .github/scripts/rust-setup-preflight.sh)" == true ]] +[[ "$(classify .github/scripts/install-rust-toolchain.sh)" == true ]] +while IFS= read -r prefix; do + [[ "$(classify "$prefix/future.rs")" == true ]] +done < "$registry" + +grep -Fq "needs.changes.outputs.rust == 'false'" "$rust_workflow" +grep -Fq 'trusted Rust classifier failed or emitted an invalid selection; running all Rust checks' "$rust_workflow" + +for path in \ + '.cargo/**' \ + '.dockerignore' \ + '*.json' \ + 'chainspecs/**' \ + 'scripts/docker_entrypoint.sh' \ + '.github/scripts/rust-setup-preflight.sh' \ + '.github/scripts/install-rust-toolchain.sh'; do + grep -Fq -- "- \"$path\"" "$docker_workflow" || { + echo "Docker coverage path is missing: $path" >&2 + exit 1 + } +done + +grep -Fq 'PR file listing failed; running eco-tests.' "$eco_workflow" +grep -Fq '(.previous_filename // empty)' "$eco_workflow" +grep -Fq 'install-rust-toolchain' "$eco_workflow" +grep -Fq 'name: cargo test (eco-tests)' "$eco_workflow" +grep -Fq 'RUST_RELEVANT: ${{ needs.changes.outputs.rust }}' "$eco_workflow" +grep -Fq 'true) [ "$TRUSTED" = success ] && [ "$TEST_RESULT" = success ]' "$eco_workflow" +grep -Fq '".github/scripts/install-rust-toolchain.sh"' "$validation_workflow" + +echo "Rust CI path registry tests passed" diff --git a/.github/scripts/test-rust-setup-preflight.sh b/.github/scripts/test-rust-setup-preflight.sh new file mode 100755 index 0000000000..e8beab89ed --- /dev/null +++ b/.github/scripts/test-rust-setup-preflight.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +preflight="${repo_root}/.github/scripts/rust-setup-preflight.sh" +installer="${repo_root}/.github/scripts/install-rust-toolchain.sh" +action="${repo_root}/.github/actions/rust-setup/action.yml" +repo_toolchain="$( + sed -nE 's/^[[:space:]]*channel[[:space:]]*=[[:space:]]*"([^"[:space:]]+)".*$/\1/p' \ + "${repo_root}/rust-toolchain.toml" +)" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +contract_dir="${tmp}/contract" +mock_bin="${tmp}/bin" +mkdir -p "${contract_dir}" "${mock_bin}" + +cat > "${mock_bin}/dpkg-query" <<'EOF' +#!/usr/bin/env bash +package="${!#}" +if [[ "${package}" == "${MOCK_MISSING_PACKAGE:-}" ]]; then + exit 1 +fi +printf 'ii ' +EOF + +cat > "${mock_bin}/cargo" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${MOCK_PROXY_LOG:-}" ]]; then + printf 'cargo invoked\n' >> "${MOCK_PROXY_LOG}" + exit 99 +fi +exit 0 +EOF + +cat > "${mock_bin}/rustc" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${MOCK_PROXY_LOG:-}" ]]; then + printf 'rustc invoked\n' >> "${MOCK_PROXY_LOG}" + exit 99 +fi +exit 0 +EOF + +cat > "${mock_bin}/rustup" <<'EOF' +#!/usr/bin/env bash +if [[ -n "${MOCK_RUSTUP_LOG:-}" ]]; then + printf '%s\n' "$*" >> "${MOCK_RUSTUP_LOG}" +fi +if [[ "${1:-}" == component && "${2:-}" == list && "${3:-}" == --installed ]]; then + printf '%s\n' ${MOCK_RUST_COMPONENTS:-cargo-x86_64-unknown-linux-gnu rustc-x86_64-unknown-linux-gnu rust-std-x86_64-unknown-linux-gnu} + exit 0 +fi +if [[ "${1:-}" == toolchain && "${2:-}" == list ]]; then + printf '%s\n' ${MOCK_RUST_TOOLCHAINS:-1.89-x86_64-unknown-linux-gnu} + exit 0 +fi +if [[ "${1:-}" == toolchain && "${2:-}" == install ]]; then + exit 0 +fi +if [[ "${1:-}" == component && "${2:-}" == add ]]; then + exit 0 +fi +exit 1 +EOF +chmod +x "${mock_bin}"/* + +cat > "${contract_dir}/packages.txt" <<'EOF' +build-essential +clang +curl +git +libssl-dev +libudev-dev +llvm +make +pkg-config +protobuf-compiler +python3 +python3-dev +EOF +touch "${contract_dir}/image-contract.env" +proxy_log="${tmp}/proxy.log" +: > "${proxy_log}" + +run_preflight() { + local output="${tmp}/output" + : > "${output}" + PATH="${mock_bin}:${PATH}" \ + FIREACTIONS_RUNNER_IMAGE_CONTRACT_DIR="${contract_dir}" \ + RUST_SETUP_COMPONENTS="${1:-}" \ + MOCK_MISSING_PACKAGE="${MOCK_MISSING_PACKAGE:-}" \ + MOCK_RUST_COMPONENTS="${MOCK_RUST_COMPONENTS:-}" \ + MOCK_RUST_TOOLCHAINS="${MOCK_RUST_TOOLCHAINS:-${repo_toolchain}-x86_64-unknown-linux-gnu}" \ + MOCK_PROXY_LOG="${proxy_log}" \ + "${preflight}" "${output}" >/dev/null + cat "${output}" +} + +assert_output() { + local output="$1" + local expected="$2" + grep -Fxq "${expected}" <<< "${output}" || { + echo "missing '${expected}' in preflight output:" >&2 + echo "${output}" >&2 + exit 1 + } +} + +output="$(run_preflight)" +assert_output "${output}" "system_ready=true" +assert_output "${output}" "rustup_ready=true" +assert_output "${output}" "toolchain_ready=true" +test ! -s "${proxy_log}" + +MOCK_RUST_COMPONENTS="cargo-x86_64-unknown-linux-gnu rustc-x86_64-unknown-linux-gnu rust-std-x86_64-unknown-linux-gnu clippy-x86_64-unknown-linux-gnu rustfmt-x86_64-unknown-linux-gnu" +output="$(run_preflight 'clippy,rustfmt')" +assert_output "${output}" "system_ready=true" +assert_output "${output}" "toolchain_ready=true" + +output="$(run_preflight 'clippy,rust-src')" +assert_output "${output}" "toolchain_ready=false" + +MOCK_MISSING_PACKAGE=llvm +output="$(run_preflight)" +assert_output "${output}" "system_ready=false" +assert_output "${output}" "toolchain_ready=true" +unset MOCK_MISSING_PACKAGE + +MOCK_RUST_TOOLCHAINS=0.0-x86_64-unknown-linux-gnu +output="$(run_preflight)" +assert_output "${output}" "toolchain_ready=false" +unset MOCK_RUST_TOOLCHAINS + +mv "${contract_dir}/image-contract.env" "${contract_dir}/image-contract.env.missing" +output="$(run_preflight)" +assert_output "${output}" "system_ready=false" +assert_output "${output}" "toolchain_ready=false" + +grep -Fq 'run: .github/scripts/rust-setup-preflight.sh "$GITHUB_OUTPUT"' "${action}" +grep -Fq "if: steps.runner-image.outputs.system_ready != 'true'" "${action}" +grep -Fq "if: steps.runner-image.outputs.toolchain_ready != 'true'" "${action}" +grep -Fq "steps.runner-image.outputs.rustup_ready != 'true'" "${action}" +grep -Fq 'RUST_SETUP_COMPONENTS: ${{ inputs.components }}' "${action}" +grep -Fq 'run: .github/scripts/install-rust-toolchain.sh "$RUST_SETUP_COMPONENTS"' "${action}" + +rustup_log="${tmp}/rustup.log" +: > "${rustup_log}" +PATH="${mock_bin}:${PATH}" \ + MOCK_RUSTUP_LOG="${rustup_log}" \ + "${installer}" 'clippy, rustfmt' >/dev/null +grep -Fxq 'toolchain install --no-self-update' "${rustup_log}" +grep -Fxq 'component add clippy' "${rustup_log}" +grep -Fxq 'component add rustfmt' "${rustup_log}" + +if PATH="${mock_bin}:${PATH}" "${installer}" 'clippy,bad/component' >/dev/null 2>&1; then + echo "expected an invalid extra component to fail" >&2 + exit 1 +fi + +echo "rust setup preflight tests passed" diff --git a/.github/scripts/test-sccache-configure.sh b/.github/scripts/test-sccache-configure.sh index 352e9bb024..2eef682f0c 100755 --- a/.github/scripts/test-sccache-configure.sh +++ b/.github/scripts/test-sccache-configure.sh @@ -43,7 +43,13 @@ cat > "$tmp/bin/sccache" <<'EOF' #!/usr/bin/env bash case "${1:-}" in --stop-server) exit 0 ;; - --start-server) [[ "${MOCK_START_FAIL:-false}" != true ]] ;; + --start-server) + [[ "${MOCK_START_FAIL:-false}" != true ]] || exit 1 + if [[ "${MOCK_LOCAL_START_FAIL:-false}" == true && -n "${SCCACHE_MULTILEVEL_CHAIN:-}" ]]; then + exit 1 + fi + exit 0 + ;; --show-stats) printf 'Compile requests 1\nCache write errors 1\n'; exit 0 ;; *) exit 0 ;; esac @@ -62,7 +68,7 @@ export SCCACHE_PATH="$tmp/bin/sccache" write_metadata() { local endpoint="${1:-https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com}" cat > "$tmp/metadata.json" < "$tmp/output" : > "$tmp/env" rm -f "$tmp/config.json" - unset MOCK_MMDS_FAIL MOCK_START_FAIL SCCACHE_GHA_FALLBACK AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + unset MOCK_MMDS_FAIL MOCK_START_FAIL MOCK_LOCAL_START_FAIL SCCACHE_GHA_FALLBACK + unset SCCACHE_LOCAL_TIER_MODE AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY export GITHUB_OUTPUT="$tmp/output" export GITHUB_ENV="$tmp/env" export GITHUB_EVENT_PATH="$tmp/event.json" @@ -121,10 +128,32 @@ assert_contains "$tmp/env" 'RUSTC_WRAPPER=sccache' assert_contains "$tmp/env" 'CARGO_INCREMENTAL=0' assert_contains "$tmp/env" 'SCCACHE_S3_KEY_PREFIX=subtensor/v1' assert_contains "$tmp/env" 'SCCACHE_IGNORE_SERVER_IO_ERROR=1' +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=true' +assert_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=webdav,s3' +assert_contains "$tmp/env" 'SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=ignore' +assert_contains "$tmp/env" 'SCCACHE_WEBDAV_ENDPOINT=http://192.168.128.1:8092' grep -v '^::add-mask::' "$tmp/activate.log" > "$tmp/activate-public.log" assert_not_contains "$tmp/activate-public.log" "$ACCESS_KEY" assert_not_contains "$tmp/activate-public.log" "$SECRET_KEY" +write_metadata +reset_outputs +export SCCACHE_LOCAL_TIER_MODE=disabled +"$CONFIGURE" prepare reader "$tmp/config.json" "$tmp/output" >/dev/null +SCCACHE_INSTALL_OUTCOME=success "$CONFIGURE" activate "$tmp/config.json" "$tmp/env" "$tmp/output" >/dev/null +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=false' +assert_not_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=' + +write_metadata +reset_outputs +"$CONFIGURE" prepare reader "$tmp/config.json" "$tmp/output" >/dev/null +export MOCK_LOCAL_START_FAIL=true +SCCACHE_INSTALL_OUTCOME=success "$CONFIGURE" activate "$tmp/config.json" "$tmp/env" "$tmp/output" >"$tmp/local-start-fail.log" +assert_contains "$tmp/output" 'enabled=true' +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=false' +assert_not_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=' +assert_contains "$tmp/local-start-fail.log" 'retrying direct R2' + reset_outputs export MOCK_MMDS_FAIL=true "$CONFIGURE" prepare reader "$tmp/config.json" "$tmp/output" >"$tmp/unavailable.log" @@ -172,6 +201,7 @@ export AWS_SECRET_ACCESS_KEY=writer-secret-key-test "$CONFIGURE" prepare auto "$tmp/config.json" "$tmp/output" >"$tmp/auto-pr.log" assert_contains "$tmp/output" 'available=true' assert_contains "$tmp/config.json" '"mode":"writer"' +assert_contains "$tmp/config.json" '"local":' for reader_case in fork dependabot malformed target missing-credentials partial-credentials malformed-credentials; do reset_outputs @@ -222,10 +252,54 @@ assert_contains "$tmp/output" 'available=true' grep -v '^::add-mask::' "$tmp/writer-main.log" > "$tmp/writer-main-public.log" assert_not_contains "$tmp/writer-main-public.log" 'writer-access-key-test' assert_not_contains "$tmp/writer-main-public.log" 'writer-secret-key-test' +assert_contains "$tmp/config.json" '"local":' SCCACHE_INSTALL_OUTCOME=success "$CONFIGURE" activate "$tmp/config.json" "$tmp/env" "$tmp/output" >"$tmp/writer-activate.log" assert_contains "$tmp/output" 'enabled=true' assert_contains "$tmp/env" 'SCCACHE_BACKEND=r2' assert_contains "$tmp/env" 'RUSTC_WRAPPER=sccache' +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=true' +assert_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=webdav,s3' +assert_contains "$tmp/env" 'SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all' +assert_contains "$tmp/env" 'AWS_ACCESS_KEY_ID=writer-access-key-test' +assert_contains "$tmp/env" 'SCCACHE_WEBDAV_USERNAME=reader-access-key-test' + +write_metadata +reset_outputs +export AWS_ACCESS_KEY_ID=writer-access-key-test +export AWS_SECRET_ACCESS_KEY=writer-secret-key-test +export GITHUB_EVENT_NAME=push +export GITHUB_REF=refs/heads/main +"$CONFIGURE" prepare writer "$tmp/config.json" "$tmp/output" >/dev/null +export MOCK_LOCAL_START_FAIL=true +SCCACHE_INSTALL_OUTCOME=success "$CONFIGURE" activate "$tmp/config.json" "$tmp/env" "$tmp/output" >"$tmp/writer-local-start-fail.log" +assert_contains "$tmp/output" 'enabled=true' +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=false' +assert_not_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=' +assert_contains "$tmp/writer-local-start-fail.log" 'retrying direct R2' + +reset_outputs +export AWS_ACCESS_KEY_ID=writer-access-key-test +export AWS_SECRET_ACCESS_KEY=writer-secret-key-test +export GITHUB_EVENT_NAME=push +export GITHUB_REF=refs/heads/main +export MOCK_MMDS_FAIL=true +"$CONFIGURE" prepare writer "$tmp/config.json" "$tmp/output" >"$tmp/writer-local-unavailable.log" +assert_contains "$tmp/output" 'available=true' +assert_contains "$tmp/config.json" '"mode":"writer"' +assert_not_contains "$tmp/config.json" '"local":' +assert_contains "$tmp/writer-local-unavailable.log" 'using direct R2 writer' +SCCACHE_INSTALL_OUTCOME=success "$CONFIGURE" activate "$tmp/config.json" "$tmp/env" "$tmp/output" >/dev/null +assert_contains "$tmp/env" 'SCCACHE_LOCAL_TIER=false' +assert_not_contains "$tmp/env" 'SCCACHE_MULTILEVEL_CHAIN=' + +reset_outputs +export AWS_ACCESS_KEY_ID=writer-access-key-test +export AWS_SECRET_ACCESS_KEY=writer-secret-key-test +export GITHUB_EVENT_NAME=push +export GITHUB_REF=refs/heads/main +export SCCACHE_LOCAL_TIER_MODE=invalid +"$CONFIGURE" prepare writer "$tmp/config.json" "$tmp/output" >"$tmp/writer-invalid-local.log" +assert_contains "$tmp/output" 'available=false' for untrusted_branch in bittensor-core-exploration codex/subtensor-r2-sccache; do for untrusted_event in push workflow_dispatch; do diff --git a/.github/scripts/test-sccache-report.sh b/.github/scripts/test-sccache-report.sh new file mode 100755 index 0000000000..1f868432c3 --- /dev/null +++ b/.github/scripts/test-sccache-report.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail +trap 'printf "sccache report test failed at line %s: %s\n" "$LINENO" "$BASH_COMMAND" >&2' ERR + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPORT="$SCRIPT_DIR/sccache-report.sh" + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" "$tmp/runner" + +cat >"$tmp/bin/sccache" <<'EOF' +#!/usr/bin/env bash +set -u +[[ "${MOCK_ALL_FAIL:-false}" != true ]] || exit 1 +case "${1:-}" in + --show-adv-stats) + [[ "${MOCK_ADV_FAIL:-false}" != true ]] || exit 1 + printf 'Cache hits (rust) 7\nCache misses (rust) 1\n' + ;; + --show-stats) + if [[ "$*" == *--stats-format=json* ]]; then + printf '{"stats":{"cache_hits":{"counts":{"Rust":7}},"cache_misses":{"counts":{"Rust":1}}}}\n' + else + printf 'Cache hits (Rust) 7\nCache misses (Rust) 1\n' + fi + ;; + *) exit 2 ;; +esac +EOF +chmod +x "$tmp/bin/sccache" + +export PATH="$tmp/bin:$PATH" +export RUNNER_TEMP="$tmp/runner" +export GITHUB_STEP_SUMMARY="$tmp/summary" +export SCCACHE_ENABLED=true +export SCCACHE_BACKEND=r2 +export SCCACHE_LOCAL_TIER=true + +"$REPORT" "Runtime cache" "$tmp/advanced" +grep -Fq 'Cache hits (rust) 7' "$tmp/advanced.txt" +grep -Fq '"Rust":7' "$tmp/advanced.json" +grep -Fq 'Configured backend: r2' "$GITHUB_STEP_SUMMARY" +grep -Fq 'combines host-local and R2 hits' "$GITHUB_STEP_SUMMARY" + +export MOCK_ADV_FAIL=true +"$REPORT" "Fallback cache" "$tmp/fallback" +grep -Fq 'Cache hits (Rust) 7' "$tmp/fallback.txt" + +export MOCK_ALL_FAIL=true +"$REPORT" "Unavailable cache" "$tmp/unavailable" +grep -Fq 'statistics were unavailable' "$tmp/unavailable.txt" + +export SCCACHE_ENABLED=false +"$REPORT" "Disabled cache" "$tmp/disabled" +grep -Fq 'sccache is disabled' "$tmp/disabled.txt" + +echo "sccache report tests passed" diff --git a/.github/scripts/test-select-shared-release-artifact.sh b/.github/scripts/test-select-shared-release-artifact.sh new file mode 100755 index 0000000000..033363a118 --- /dev/null +++ b/.github/scripts/test-select-shared-release-artifact.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +selector="$script_dir/select-shared-release-artifact.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" + +cat > "$tmp/bin/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +endpoint="${!#}" +case "$endpoint" in + */actions/artifacts*) cat "$MOCK_ARTIFACTS" ;; + */actions/runs/777) cat "$MOCK_RUN" ;; + *) echo "unexpected endpoint: $endpoint" >&2; exit 2 ;; +esac +EOF +chmod +x "$tmp/bin/gh" + +export PATH="$tmp/bin:$PATH" +export GH_TOKEN=test-job-token +export GITHUB_REPOSITORY=RaoFoundation/subtensor +export GITHUB_REPOSITORY_ID=608683796 +export GITHUB_SHA=cccccccccccccccccccccccccccccccccccccccc +export GITHUB_PR_HEAD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export MOCK_ARTIFACTS="$tmp/artifacts.json" +export MOCK_RUN="$tmp/run.json" + +cat > "$MOCK_ARTIFACTS" <<'EOF' +{"artifacts":[{"id":123,"name":"node-subtensor-release-cccccccccccccccccccccccccccccccccccccccc","size_in_bytes":456,"digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","expired":false,"created_at":"2026-07-17T00:00:00Z","workflow_run":{"id":777,"head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository_id":608683796}}]} +EOF +cat > "$MOCK_RUN" <<'EOF' +{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml"} +EOF + +: > "$tmp/output" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=true' "$tmp/output" +grep -qx 'artifact_id=123' "$tmp/output" +grep -qx 'run_id=777' "$tmp/output" +grep -qx 'size=456' "$tmp/output" +grep -qx 'digest=sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' "$tmp/output" + +# A matching PR head is insufficient after the base branch changes: the +# synthetic merge SHA in the artifact name must match this run exactly. +jq '.artifacts[0].name = "node-subtensor-release-dddddddddddddddddddddddddddddddddddddddd"' \ + "$MOCK_ARTIFACTS" > "$tmp/stale-merge-artifacts.json" +export MOCK_ARTIFACTS="$tmp/stale-merge-artifacts.json" +: > "$tmp/output" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=false' "$tmp/output" + +# Exact source SHAs alone are still insufficient: refuse an artifact produced +# by any other workflow, then use the unchanged local-build fallback. +export MOCK_ARTIFACTS="$tmp/artifacts.json" +jq '.path = ".github/workflows/untrusted-producer.yml"' "$MOCK_RUN" > "$tmp/wrong-run.json" +export MOCK_RUN="$tmp/wrong-run.json" +: > "$tmp/output" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=false' "$tmp/output" + +# Reject malformed integrity metadata before it reaches the downloader. +export MOCK_RUN="$tmp/run.json" +jq '.artifacts[0].digest = "sha256:bad"' "$MOCK_ARTIFACTS" > "$tmp/bad-artifacts.json" +export MOCK_ARTIFACTS="$tmp/bad-artifacts.json" +: > "$tmp/output" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=false' "$tmp/output" + +echo "shared release artifact selector tests passed" diff --git a/.github/scripts/test-snapshot-artifact.sh b/.github/scripts/test-snapshot-artifact.sh index a76d59f5d8..633aac1149 100755 --- a/.github/scripts/test-snapshot-artifact.sh +++ b/.github/scripts/test-snapshot-artifact.sh @@ -113,6 +113,8 @@ selected=$(select_fixture) grep -qx 'artifact-id=12' <<<"$selected" grep -qx 'run-id=102' <<<"$selected" grep -qx 'producer-sha=0000000000000000000000000000000000000066' <<<"$selected" +grep -qx 'artifact-size-bytes=1234' <<<"$selected" +grep -qx 'artifact-digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' <<<"$selected" # Exactly 72 hours is accepted, one second older fails, and optional lookup # reports a miss. An artifact older than 36 hours remains usable with a warning. diff --git a/.github/scripts/validate-rust-ci-paths.sh b/.github/scripts/validate-rust-ci-paths.sh new file mode 100755 index 0000000000..fdd131bec3 --- /dev/null +++ b/.github/scripts/validate-rust-ci-paths.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +registry=${1:-$repo_root/.github/rust-ci-paths.txt} + +if [[ ! -s "$registry" ]]; then + echo "Rust CI path registry is missing or empty: $registry" >&2 + echo "Fix: restore .github/rust-ci-paths.txt with one sorted repository-relative directory prefix per line." >&2 + echo "Verify: .github/scripts/test-rust-ci-paths.sh" >&2 + exit 1 +fi + +prefixes=() +while IFS= read -r prefix; do + prefixes+=("$prefix") +done < <(sed '/^[[:space:]]*$/d' "$registry") +for prefix in "${prefixes[@]}"; do + if [[ ! "$prefix" =~ ^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*$ ]]; then + echo "Invalid Rust CI path prefix: $prefix" >&2 + echo "Fix: use a repository-relative directory such as pallets or sdk/bittensor-core in .github/rust-ci-paths.txt." >&2 + echo "Verify: .github/scripts/test-rust-ci-paths.sh" >&2 + exit 1 + fi + if [[ ! -d "$repo_root/$prefix" ]]; then + echo "Registered Rust CI path does not exist: $prefix" >&2 + echo "Fix: correct or remove the stale line in .github/rust-ci-paths.txt." >&2 + echo "Verify: .github/scripts/test-rust-ci-paths.sh" >&2 + exit 1 + fi +done + +sorted=$(printf '%s\n' "${prefixes[@]}" | LC_ALL=C sort -u) +if [[ "$sorted" != "$(printf '%s\n' "${prefixes[@]}")" ]]; then + echo "Rust CI path registry must be sorted and contain no duplicates" >&2 + echo "Fix: run LC_ALL=C sort -u .github/rust-ci-paths.txt -o .github/rust-ci-paths.txt" >&2 + echo "Verify: .github/scripts/test-rust-ci-paths.sh" >&2 + exit 1 +fi + +metadata=$(cd "$repo_root" && cargo metadata --format-version 1 --no-deps) +package_directories=() +while IFS= read -r package_directory; do + package_directories+=("$package_directory") +done < <( + jq -r --arg root "$repo_root/" ' + .workspace_members[] as $member + | .packages[] + | select(.id == $member) + | .manifest_path + | sub("/Cargo.toml$"; "") + | select(. != ($root | rtrimstr("/"))) + | ltrimstr($root) + ' <<< "$metadata" | LC_ALL=C sort -u +) + +missing=() +for package_directory in "${package_directories[@]}"; do + covered=false + for prefix in "${prefixes[@]}"; do + if [[ "$package_directory" == "$prefix" || "$package_directory" == "$prefix/"* ]]; then + covered=true + break + fi + done + if [[ "$covered" != true ]]; then + missing+=("$package_directory") + fi +done + +if (( ${#missing[@]} > 0 )); then + echo "Rust workspace packages have no CI path owner:" >&2 + printf ' - %s\n' "${missing[@]}" >&2 + echo "Fix: add the narrowest stable parent directory for each package to .github/rust-ci-paths.txt; do not add individual crates when an existing area such as pallets or support owns them." >&2 + echo "The trusted classifier reads this registry automatically; do not add a second path pattern to check-rust.yml." >&2 + echo "Verify: .github/scripts/test-rust-ci-paths.sh" >&2 + exit 1 +fi + +echo "Validated ${#package_directories[@]} non-root Rust workspace packages across ${#prefixes[@]} CI path prefixes." diff --git a/.github/workflows/cargo-audit.yml b/.github/workflows/cargo-audit.yml index 765a4a2c90..9da049c6ce 100644 --- a/.github/workflows/cargo-audit.yml +++ b/.github/workflows/cargo-audit.yml @@ -34,8 +34,23 @@ jobs: sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - - name: Install cargo-audit - run: cargo install --force --locked cargo-audit + - name: Ensure cargo-audit + run: | + # The pinned runner image provides this exact binary. Old images and + # hosted runners keep the normal Cargo install fallback. + contract=/etc/fireactions-runner-image/image-contract.env + system_bin=/usr/local/bin/cargo-audit + if [[ -r "$contract" ]] && \ + grep -Fxq 'CARGO_AUDIT_VERSION=0.22.2' "$contract" && \ + [[ -x "$system_bin" ]] && \ + "$system_bin" --version 2>/dev/null | grep -Fq 'cargo-audit 0.22.2'; then + # rust-cache restores $HOME/.cargo/bin ahead of /usr/local/bin. + # Replace any cached copy with the verified image binary. + install -m 0755 "$system_bin" "$HOME/.cargo/bin/cargo-audit" + echo "Using verified runner-image cargo-audit 0.22.2" + else + cargo install --force --locked cargo-audit --version 0.22.2 + fi - name: cargo audit # Each ignore is a known, accepted advisory; revisit when bumping diff --git a/.github/workflows/check-bittensor-e2e-tests.yml b/.github/workflows/check-bittensor-e2e-tests.yml index 0fcfa2de74..45e30b061e 100644 --- a/.github/workflows/check-bittensor-e2e-tests.yml +++ b/.github/workflows/check-bittensor-e2e-tests.yml @@ -12,34 +12,13 @@ concurrency: group: e2e-cli-${{ github.ref }} cancel-in-progress: true -# Path-filtered: none of this workflow's checks are required for merge, so a -# workflow-level `paths:` filter is safe. If any check here is ever made -# required, this filter MUST move to job-level `if:` gating (a required check -# whose workflow never triggers is stuck "Expected" and blocks the merge). +# Always run the cheap trusted-base classifier. A workflow-level `paths:` +# allowlist cannot fail closed for newly introduced build inputs because the +# classifier would never see them. Known-safe surfaces exit after planning. on: pull_request: branches: ["*"] types: [opened, synchronize, reopened, labeled, unlabeled] - paths: - # Anything that changes the node/runtime the e2e suites run against. - - "common/**" - - "node/**" - - "pallets/**" - - "precompiles/**" - - "primitives/**" - - "runtime/**" - - "support/**" - - "chain-extensions/**" - - "src/**" - - "vendor/**" - - "Cargo.toml" - - "Cargo.lock" - - "build.rs" - - "rust-toolchain.toml" - - "Dockerfile-localnet" - # The SDK/CLI code under test. - - "sdk/**" - - ".github/workflows/check-bittensor-e2e-tests.yml" workflow_dispatch: inputs: @@ -55,6 +34,8 @@ env: # jobs retag the CI image to this compatibility name locally; it is # never pushed. LOCALNET_IMAGE: ghcr.io/raofoundation/subtensor-localnet:ci + # SDK-only runs can reuse a public image published from the exact base SHA. + LOCALNET_IMAGE_REPOSITORY: ghcr.io/raofoundation/subtensor-localnet # Per-commit tag in the CI-only package: built once, pulled by every test # job. Keep these transport artifacts out of the public release package. LOCALNET_IMAGE_CI: ghcr.io/raofoundation/subtensor-localnet-ci:ci-${{ github.event.pull_request.head.sha || github.sha }} @@ -67,100 +48,206 @@ jobs: steps: - run: echo "Non-fork PR; self-hosted runners may execute checkout." - check-label: + plan: + name: Plan Rust SDK E2E coverage runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + packages: read outputs: - skip-bittensor-e2e-tests: ${{ steps.get-labels.outputs.skip-bittensor-e2e-tests || steps.set-default.outputs.skip-bittensor-e2e-tests }} + run_e2e: ${{ steps.filter.outputs.e2e }} + build_image: ${{ steps.image.outputs.build_image }} + base_image_ref: ${{ steps.image.outputs.base_image_ref }} + test_count: ${{ steps.matrix.outputs.test_count }} + shard_count: ${{ steps.matrix.outputs.shard_count }} + test_matrix: ${{ steps.matrix.outputs.test_matrix }} steps: - - name: Install dependencies - run: | - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" gh jq - - - name: Check out repository + # A PR must not be able to route itself around E2E by editing its own + # classifier. Bootstrap absence and all lookup failures select everything. + - name: Check out trusted Rust SDK E2E classifier uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} - - - name: Get labels from PR - id: get-labels - if: github.event_name == 'pull_request' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} + sparse-checkout: | + .github/scripts/classify-bittensor-e2e-changes.sh + .github/scripts/build-bittensor-e2e-matrix.py + .github/scripts/extract-pull-file-paths.sh + path: .trusted-rust-sdk-e2e + persist-credentials: false + + - name: Classify changed paths + id: filter + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EXPECTED_CHANGED_FILES: ${{ github.event.pull_request.changed_files }} + SKIP_BY_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'skip-bittensor-e2e-tests') }} run: | - LABELS=$(gh pr -R ${{ github.repository }} view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name') - echo "Current labels: $LABELS" - if echo "$LABELS" | grep -q "skip-bittensor-e2e-tests"; then - echo "skip-bittensor-e2e-tests=true" >> $GITHUB_OUTPUT - else - echo "skip-bittensor-e2e-tests=false" >> $GITHUB_OUTPUT + set -euo pipefail + classifier=.trusted-rust-sdk-e2e/.github/scripts/classify-bittensor-e2e-changes.sh + extractor=.trusted-rust-sdk-e2e/.github/scripts/extract-pull-file-paths.sh + + select_full_suite() { + { + echo 'e2e=true' + echo 'build_image=true' + } >> "$GITHUB_OUTPUT" + } + + publish_classification() { + local classification=$1 key value + if [[ "$(wc -l < "$classification" | tr -d '[:space:]')" != 2 ]]; then + return 1 + fi + for key in e2e build_image; do + value=$(sed -n "s/^${key}=//p" "$classification") + [[ "$value" == true || "$value" == false ]] || return 1 + [[ "$(grep -c "^${key}=" "$classification")" == 1 ]] || return 1 + done + cat "$classification" >> "$GITHUB_OUTPUT" + } + + if [[ "$GITHUB_EVENT_NAME" == pull_request && "$SKIP_BY_LABEL" == true ]]; then + echo 'e2e=false' >> "$GITHUB_OUTPUT" + echo 'build_image=false' >> "$GITHUB_OUTPUT" + exit 0 fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Set default skip value for workflow_dispatch - id: set-default - if: github.event_name == 'workflow_dispatch' - run: | - echo "skip-bittensor-e2e-tests=false" >> $GITHUB_OUTPUT + if [[ "$GITHUB_EVENT_NAME" != pull_request ]]; then + classification=$(mktemp) + trap 'rm -f "$classification"' EXIT + if ! "$classifier" --all "$classification" || ! publish_classification "$classification"; then + echo "trusted Rust SDK E2E classifier failed or emitted invalid outputs; running the full suite" >&2 + select_full_suite + fi + exit 0 + fi - find-e2e-tests: - needs: check-label - if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false' - runs-on: ubuntu-latest - outputs: - test-files: ${{ steps.get-tests.outputs.test-files }} - steps: - - name: Install dependencies - run: | - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" jq + if [[ ! -x "$classifier" || ! -x "$extractor" ]]; then + echo "trusted Rust SDK E2E classifier unavailable; running the full suite" >&2 + select_full_suite + exit 0 + fi - - name: Check out repository + if ! files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate); then + echo "changed-file lookup failed; running the full Rust SDK E2E suite" >&2 + select_full_suite + exit 0 + fi + if ! paths=$(printf '%s\n' "$files" | "$extractor" "$EXPECTED_CHANGED_FILES"); then + echo "changed-file response invalid; running the full Rust SDK E2E suite" >&2 + select_full_suite + exit 0 + fi + classification=$(mktemp) + trap 'rm -f "$classification"' EXIT + if ! printf '%s\n' "$paths" | "$classifier" "$classification" || + ! publish_classification "$classification"; then + echo "trusted Rust SDK E2E classifier failed or emitted invalid outputs; running the full suite" >&2 + select_full_suite + fi + + - name: Check out proposed E2E manifest + if: steps.filter.outputs.e2e == 'true' uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + sparse-checkout: | + sdk/bittensor-core/tests/e2e-manifest.json + .github/scripts/build-bittensor-e2e-matrix.py + sparse-checkout-cone-mode: false + path: .rust-sdk-e2e-manifest + persist-credentials: false + + - name: Build balanced test matrix + id: matrix + if: steps.filter.outputs.e2e == 'true' + run: | + set -euo pipefail + manifest=.rust-sdk-e2e-manifest/sdk/bittensor-core/tests/e2e-manifest.json + trusted_builder=.trusted-rust-sdk-e2e/.github/scripts/build-bittensor-e2e-matrix.py + if [[ -x "$trusted_builder" ]]; then + "$trusted_builder" "$manifest" "$GITHUB_OUTPUT" 32 + else + # One-time bootstrap until this PR lands. This is the same strict + # all-112 round-robin matrix enforced by the repository test. + .rust-sdk-e2e-manifest/.github/scripts/build-bittensor-e2e-matrix.py \ + "$manifest" "$GITHUB_OUTPUT" 32 + fi - - name: Verify the Rust e2e migration manifest - id: get-tests - shell: bash + - name: Pin baseline localnet image for SDK-only changes + id: image + env: + RUN_E2E: ${{ steps.filter.outputs.e2e }} + CHAIN_CHANGED: ${{ steps.filter.outputs.build_image }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -euo pipefail - manifest=sdk/bittensor-core/tests/e2e-manifest.json - jq -e 'length == 112' "$manifest" >/dev/null - jq -e 'map(.test) | length == (unique | length)' "$manifest" >/dev/null - jq -e 'all(.[]; (.test | startswith("intent_")) or (.test | startswith("test_")))' "$manifest" >/dev/null - test_matrix=$(jq -c . "$manifest") - echo "Found $(jq length "$manifest") migrated Rust e2e tests" - echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT" + if [[ "$RUN_E2E" != true ]]; then + echo 'build_image=false' >> "$GITHUB_OUTPUT" + exit 0 + fi + if [[ "$CHAIN_CHANGED" == true ]]; then + echo 'build_image=true' >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Reuse only a source-immutable image published from the exact PR + # base. Older branches and publication lag simply build the PR image. + base_image_tag="$LOCALNET_IMAGE_REPOSITORY:sha-$BASE_SHA" + + for attempt in 1 2 3; do + digest=$(docker buildx imagetools inspect "$base_image_tag" \ + --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"') || true + if [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo 'build_image=false' >> "$GITHUB_OUTPUT" + echo "base_image_ref=$base_image_tag@$digest" >> "$GITHUB_OUTPUT" + exit 0 + fi + sleep $((attempt * 2)) + done + + echo "::notice::No immutable localnet image exists for base $BASE_SHA; building the PR image." + echo 'build_image=true' >> "$GITHUB_OUTPUT" build-rust-e2e-test-binary: - needs: [check-label, find-e2e-tests] - if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false' - runs-on: ubuntu-latest + needs: [trusted-pr, plan] + if: needs.plan.outputs.run_e2e == 'true' + # This producer is short and finishes well before the localnet image even + # on eight cores. Reserve the constrained 16-core lane for the image build + # that controls this workflow's wall time. + runs-on: [self-hosted, fireactions-turbo-8] + environment: + name: ${{ github.event_name == 'workflow_dispatch' && 'sccache-reader' || 'sccache-writer' }} + deployment: false timeout-minutes: 60 + env: + SKIP_WASM_BUILD: 1 steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Install Rust + build dependencies - run: | - chmod +x ./scripts/install_build_env.sh - ./scripts/install_build_env.sh + - name: Set up cached Rust build environment + uses: ./.github/actions/rust-setup + with: + cache-key: rust-sdk-e2e-test-binary + sccache-credential-mode: auto + sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} + sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - name: Compile once and prove manifest completeness shell: bash run: | set -euo pipefail - source "$HOME/.cargo/env" mkdir -p build/rust-e2e - # The workspace/localnet build also compiles the Python binding. Keep - # CoreError/API compatibility in this faster job so failures surface - # before the expensive node builds. - cargo check -p bittensor-core-py + # Python-binding compatibility is already compiled by the required + # workspace/all-features Rust test job. This producer owns only the + # native chain-facing harness consumed below. cargo test -p bittensor-core --test e2e --no-run --message-format=json \ > build/rust-e2e/cargo-messages.json executable=$( @@ -182,6 +269,10 @@ jobs: | sort > build/rust-e2e/compiled-tests.txt diff -u build/rust-e2e/expected-tests.txt build/rust-e2e/compiled-tests.txt + - name: Report Rust SDK E2E compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Rust SDK E2E compiler cache" + - name: Upload compiled Rust e2e binary uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -189,68 +280,63 @@ jobs: path: build/rust-e2e/bittensor-core-e2e if-no-files-found: error - artifacts: - name: Node • fast-runtime • ${{ matrix.platform.arch }} - needs: [check-label, trusted-pr] - if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false' - strategy: - fail-fast: false - matrix: - platform: - - runner: [self-hosted, fireactions-turbo-8] - triple: x86_64-unknown-linux-gnu - arch: amd64 - - runs-on: ${{ matrix.platform.runner }} + build-localnet-image: + name: Build and publish PR localnet image + needs: [trusted-pr, plan] + if: needs.plan.outputs.build_image == 'true' + # Swap lanes with the short E2E harness build instead of consuming an + # additional turbo-16 slot. A full PR still needs four 16-core runners, so + # the eight-runner lane retains capacity for two simultaneous PRs. + runs-on: [self-hosted, fireactions-turbo-16] environment: name: ${{ github.event_name == 'workflow_dispatch' && 'sccache-reader' || 'sccache-writer' }} deployment: false + outputs: + image_ref: ${{ steps.pin.outputs.image_ref }} + env: + BUILD_TRIPLE: x86_64-unknown-linux-gnu + RUNTIME: fast-runtime steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Install Rust + dependencies - run: | - chmod +x ./scripts/install_build_env.sh - ./scripts/install_build_env.sh - - - name: Add Rust target triple - run: | - source "$HOME/.cargo/env" - rustup target add ${{ matrix.platform.triple }} - - - name: Shared R2 compiler cache - uses: ./.github/actions/sccache-setup + - name: Set up cached Rust build environment + uses: ./.github/actions/rust-setup with: - credential-mode: auto - writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} - writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + cache-key: rust-sdk-localnet-fast + fast-linker: "false" + sccache-credential-mode: auto + sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} + sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + + - name: Ensure native target is installed + run: rustup target add "$BUILD_TRIPLE" - name: Patch limits for local run - run: | - chmod +x ./scripts/localnet_patch.sh - ./scripts/localnet_patch.sh + run: ./scripts/localnet_patch.sh - - name: Build binaries + - name: Build only the fast-runtime node run: | - export PATH="$HOME/.cargo/bin:$PATH" - export CARGO_BUILD_TARGET="${{ matrix.platform.triple }}" - - # The SDK e2e suite validates the fast-runtime localnet image used - # by developer and CI localnet runs. - ./scripts/localnet.sh --build-only - - - name: Prepare artifacts for upload + CARGO_TARGET_DIR="target/$RUNTIME" cargo build \ + --locked \ + --profile release \ + --features "pow-faucet metadata-hash fast-runtime" \ + --package node-subtensor \ + --package node-subtensor-runtime \ + --target "$BUILD_TRIPLE" + + - name: Report localnet compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Rust SDK localnet compiler cache" + + - name: Prepare Docker build context run: | - RUNTIME="fast-runtime" - TRIPLE="${{ matrix.platform.triple }}" - - BINARY_PATH="target/${RUNTIME}/${TRIPLE}/release/node-subtensor" - WASM_PATH="target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" + BINARY_PATH="target/$RUNTIME/$BUILD_TRIPLE/release/node-subtensor" + WASM_PATH="target/$RUNTIME/$BUILD_TRIPLE/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" if [[ ! -f "$BINARY_PATH" ]]; then echo "Error: Binary not found at $BINARY_PATH" @@ -262,53 +348,16 @@ jobs: exit 1 fi - mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/ - cp -v "$BINARY_PATH" \ - build/ci_target/${RUNTIME}/${TRIPLE}/release/ - - mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/ - cp -v "$WASM_PATH" \ - build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/ - - - name: Upload artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: binaries-${{ matrix.platform.triple }}-fast-runtime - path: build/ - if-no-files-found: error - - build-image-with-current-branch: - needs: [check-label, artifacts] - if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false' - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} - - - name: Download all binary artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 - with: - pattern: binaries-* - path: build/ - merge-multiple: true - - - name: Move Docker data-root to /mnt/data - run: | - sudo systemctl stop docker - sudo mkdir -p /mnt/data/docker - sudo chown -R runner:runner /mnt/data - sudo chmod -R 777 /mnt/data - echo '{"data-root": "/mnt/data/docker"}' | sudo tee /etc/docker/daemon.json - sudo systemctl start docker - docker info | grep "Docker Root Dir" + destination="build/ci_target/$RUNTIME/$BUILD_TRIPLE/release" + mkdir -p "$destination/wbuild/node-subtensor-runtime" + cp -v "$BINARY_PATH" "$destination/" + cp -v "$WASM_PATH" "$destination/wbuild/node-subtensor-runtime/" - name: Build Docker Image env: SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | + docker info >/dev/null docker build \ -f sdk/bittensor-core/tests/Dockerfile.localnet-fast \ --label "org.opencontainers.image.revision=$SOURCE_SHA" \ @@ -354,22 +403,42 @@ jobs: exit 1 done + - name: Pin published image digest + id: pin + run: | + set -euo pipefail + for attempt in 1 2 3; do + digest=$(docker buildx imagetools inspect "$LOCALNET_IMAGE_CI" \ + --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"') || true + if [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "image_ref=$LOCALNET_IMAGE_CI@$digest" >> "$GITHUB_OUTPUT" + exit 0 + fi + sleep $((attempt * 2)) + done + echo "::error::unable to resolve the published localnet image digest" + exit 1 + run-rust-e2e-tests: needs: - - check-label - - find-e2e-tests + - plan - build-rust-e2e-test-binary - - build-image-with-current-branch - if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false' + - build-localnet-image + if: >- + always() && + needs.plan.outputs.run_e2e == 'true' && + needs.build-rust-e2e-test-binary.result == 'success' && + (needs.build-localnet-image.result == 'success' || needs.build-localnet-image.result == 'skipped') runs-on: ubuntu-latest strategy: fail-fast: false max-parallel: 32 - matrix: - include: ${{ fromJson(needs.find-e2e-tests.outputs.test-files) }} + matrix: ${{ fromJSON(needs.plan.outputs.test_matrix) }} timeout-minutes: 60 - name: "rust-e2e: ${{ matrix.test }}" + name: "rust-e2e shard ${{ matrix.shard }}/${{ needs.plan.outputs.shard_count }}" + env: + IMAGE_REF: ${{ needs.build-localnet-image.outputs.image_ref || needs.plan.outputs.base_image_ref }} steps: - name: Download compiled Rust e2e binary uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 @@ -386,44 +455,56 @@ jobs: - name: Pull Docker Image # GHCR intermittently returns permission_denied/timeouts under the - # ~120-job pull fan-out, so retry with backoff instead of failing the shard. + # concurrent pull fan-out, so retry with backoff instead of failing the shard. run: | + test -n "$IMAGE_REF" for i in 1 2 3; do - docker pull "$LOCALNET_IMAGE_CI" && exit 0 + docker pull "$IMAGE_REF" && exit 0 echo "docker pull failed (attempt $i), retrying in $((i * 15))s..." sleep $((i * 15)) done - docker pull "$LOCALNET_IMAGE_CI" + docker pull "$IMAGE_REF" - name: Retag Docker Image - run: docker tag "$LOCALNET_IMAGE_CI" "$LOCALNET_IMAGE" + run: docker tag "$IMAGE_REF" "$LOCALNET_IMAGE" - - name: Run with retry + - name: Run shard with per-test retry env: SKIP_PULL: 1 RUST_BACKTRACE: 1 - TEST_NAME: ${{ matrix.test }} + TESTS_JSON: ${{ toJSON(matrix.tests) }} run: | - set +e + set -uo pipefail chmod +x rust-e2e/bittensor-core-e2e - for i in 1 2; do - echo "Attempt $i: Running $TEST_NAME" - rust-e2e/bittensor-core-e2e \ - "$TEST_NAME" \ - --exact \ - --nocapture \ - --test-threads=1 - status=$? - if [ $status -eq 0 ]; then - echo "Test passed on attempt $i" - break - else - echo "Test failed on attempt $i" - if [ $i -eq 2 ]; then - echo "Test failed after 2 attempts" - exit 1 + mapfile -t tests < <(jq -r '.[]' <<< "$TESTS_JSON") + failures=() + + for test_name in "${tests[@]}"; do + passed=false + for attempt in 1 2; do + echo "Attempt $attempt: Running $test_name" + if rust-e2e/bittensor-core-e2e \ + "$test_name" \ + --exact \ + --nocapture \ + --test-threads=1; then + echo "$test_name passed on attempt $attempt" + passed=true + break + fi + echo "$test_name failed on attempt $attempt" + if [[ "$attempt" -eq 1 ]]; then + echo "Retrying..." + sleep 5 fi - echo "Retrying..." - sleep 5 + done + if [[ "$passed" != true ]]; then + echo "::error::$test_name failed after 2 attempts" + failures+=("$test_name") fi done + + if (( ${#failures[@]} > 0 )); then + printf 'Failed tests: %s\n' "${failures[*]}" + exit 1 + fi diff --git a/.github/workflows/check-docker.yml b/.github/workflows/check-docker.yml index 7cde42f02a..17eec6e650 100644 --- a/.github/workflows/check-docker.yml +++ b/.github/workflows/check-docker.yml @@ -14,23 +14,45 @@ on: - "precompiles/**" - "primitives/**" - "runtime/**" - - "sdk/bittensor-core/**" - - "sdk/bittensor-core-py/**" - - "sdk/bittensor-core-wasm/**" - "support/**" - "chain-extensions/**" - "src/**" - "vendor/**" + # The bittensor-core SDK crates are not in node-subtensor's dependency + # graph. Their shared dependency movement still enters through Cargo.lock. - "Cargo.toml" - "Cargo.lock" - "build.rs" - "rust-toolchain.toml" + - ".cargo/**" + - ".dockerignore" + - "*.json" + - "chainspecs/**" + - "scripts/docker_entrypoint.sh" - "Dockerfile" - "Dockerfile.prebuilt" - "Cross.toml" - ".github/actions/build-production-binary/**" - ".github/actions/rust-setup/**" + - ".github/actions/sccache-setup/**" + - ".github/scripts/rust-setup-preflight.sh" + - ".github/scripts/install-rust-toolchain.sh" + - ".github/scripts/sccache-configure.sh" + - ".github/scripts/sccache-config.py" + - ".github/scripts/sccache-report.sh" - ".github/workflows/check-docker.yml" + # These modules are compiled only by Rust tests. Check Rust owns them; + # they cannot change the production binary copied into the image. + - "!pallets/*/src/tests/**" + - "!pallets/*/src/tests.rs" + - "!pallets/*/src/mock.rs" + - "!chain-extensions/src/tests.rs" + - "!chain-extensions/src/mock.rs" + - "!precompiles/src/mock.rs" + - "!node/tests/**" + - "!runtime/tests/**" + - "!support/*/tests/**" + - "!support/procedural-fork/src/pallet/parse/tests/**" concurrency: group: check-docker-${{ github.ref }} @@ -40,16 +62,12 @@ permissions: contents: read jobs: - trusted-pr: - name: Trusted PR source (non-fork) - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false - runs-on: ubuntu-latest - steps: - - run: echo "Non-fork PR; self-hosted runners may execute checkout." - - binary: - name: production binary (amd64) - needs: trusted-pr + image: + name: production binary + docker image (amd64, no push) + # Job-level gating is evaluated before a self-hosted runner is assigned. + # Keep untrusted fork code off persistent organization infrastructure + # without serializing the production build behind a hosted guard job. + if: github.event.pull_request.head.repo.fork == false runs-on: [self-hosted, fireactions-turbo-16] environment: name: sccache-writer @@ -67,37 +85,18 @@ jobs: sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - - uses: actions/upload-artifact@v4 - with: - name: production-node-amd64 - path: build/ci_target/amd64/node-subtensor - if-no-files-found: error - - image: - name: docker image (amd64, no push) - needs: [trusted-pr, binary] - runs-on: [self-hosted, fireactions-turbo-8] - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v5 - with: - name: production-node-amd64 - path: build/ci_target/amd64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build Docker image without publishing - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.prebuilt - platforms: linux/amd64 - tags: subtensor-ci:amd64 - load: true - push: false + # Buildx is part of the pinned Fireactions image contract. Using its + # default local Docker builder avoids a second runner plus artifact + # handoff and does not need a per-job builder installation. + run: | + docker buildx version + docker buildx build \ + --file Dockerfile.prebuilt \ + --platform linux/amd64 \ + --tag subtensor-ci:amd64 \ + --load \ + . - name: Verify image architecture and executable shell: bash diff --git a/.github/workflows/check-rust.yml b/.github/workflows/check-rust.yml index 4fb0863d70..397469a3d3 100644 --- a/.github/workflows/check-rust.yml +++ b/.github/workflows/check-rust.yml @@ -33,16 +33,32 @@ jobs: # itself must always trigger (a required check whose workflow never runs is # stuck "Expected" and blocks the merge). Instead, jobs skip themselves when # the PR touches no Rust-relevant paths; a `skipped` conclusion satisfies - # branch protection. No checkout: for pull_request events the filter reads - # the changed-file list from the API, so no untrusted code is executed. + # branch protection. The filter checks out only routing code from the trusted + # base and reads proposed paths from the API, so no PR-controlled code runs. changes: name: detect rust changes runs-on: ubuntu-latest permissions: + contents: read pull-requests: read outputs: rust: ${{ steps.filter.outputs.rust }} steps: + # Path ownership comes from the base revision so a PR cannot classify its + # own new workspace area as irrelevant. + - name: Check out trusted Rust change classifier + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + sparse-checkout: | + .github/rust-ci-paths.txt + .github/scripts/classify-rust-changes.sh + .github/scripts/extract-pull-file-paths.sh + sparse-checkout-cone-mode: false + path: .trusted-rust-ci-paths + persist-credentials: false + # Plain gh-api file listing instead of a marketplace action: the org's # Actions allowlist rejects unlisted third-party actions (startup_failure). - name: Filter changed paths @@ -51,15 +67,51 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} + EXPECTED_CHANGED_FILES: ${{ github.event.pull_request.changed_files }} run: | set -euo pipefail - files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename') - pattern='^(common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^sdk/bittensor-core(-py|-wasm)?/|^(Cargo\.(toml|lock)|build\.rs|rust-toolchain\.toml|zepter\.yaml)$|^\.github/(workflows/check-rust\.yml|actions/(rust-setup|sccache-setup)/|scripts/sccache-configure\.sh$)' - if grep -qE "$pattern" <<< "$files"; then + classifier=.trusted-rust-ci-paths/.github/scripts/classify-rust-changes.sh + extractor=.trusted-rust-ci-paths/.github/scripts/extract-pull-file-paths.sh + + select_rust() { echo "rust=true" >> "$GITHUB_OUTPUT" - else - echo "no Rust-relevant paths changed" - echo "rust=false" >> "$GITHUB_OUTPUT" + } + + publish_classification() { + local classification=$1 + if [[ "$(wc -l < "$classification" | tr -d '[:space:]')" == 1 ]] && + grep -qxE 'rust=(true|false)' "$classification"; then + cat "$classification" >> "$GITHUB_OUTPUT" + return 0 + fi + return 1 + } + + # This branch introduces the trusted classifier. Until it lands, + # absence on the base revision runs every Rust check rather than + # executing proposed routing code or maintaining a second policy. + if [[ ! -x "$classifier" || ! -x "$extractor" ]]; then + echo "trusted Rust classifier unavailable; running all Rust checks" >&2 + select_rust + exit 0 + fi + + if ! files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate); then + echo "changed-file lookup failed; running all Rust checks" >&2 + select_rust + exit 0 + fi + if ! paths=$(printf '%s\n' "$files" | "$extractor" "$EXPECTED_CHANGED_FILES"); then + echo "changed-file response invalid; running all Rust checks" >&2 + select_rust + exit 0 + fi + classification=$(mktemp) + trap 'rm -f "$classification"' EXIT + if ! printf '%s\n' "$paths" | "$classifier" "$classification" || + ! publish_classification "$classification"; then + echo "trusted Rust classifier failed or emitted an invalid selection; running all Rust checks" >&2 + select_rust fi fmt: @@ -73,6 +125,8 @@ jobs: with: cache-key: cargo-fmt components: rustfmt + - name: Validate Rust CI path ownership + run: .github/scripts/test-rust-ci-paths.sh - name: cargo fmt --check run: | set -euo pipefail @@ -114,11 +168,40 @@ jobs: sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - name: cargo clippy run: cargo clippy --workspace --all-targets ${{ matrix.features.flags }} -- -D warnings + - name: Report clippy compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Clippy ${{ matrix.features.name }} compiler cache" + + # GitHub evaluates a matrix job's job-level `if` before expanding the + # matrix. On a non-Rust PR, the skipped `clippy` job above therefore reports + # one literal context named `cargo clippy (${{ matrix.features.name }})` + # instead of the two concrete contexts required by branch protection. Emit + # those exact contexts from cheap hosted jobs without checking out or + # executing PR-controlled code. These conditions are mutually exclusive + # with the real Clippy matrix above, so Rust-relevant PRs retain full lint + # coverage and never receive a no-op success in its place. + clippy-not-required: + name: cargo clippy (${{ matrix.features.name }}) + needs: [changes] + if: github.event_name == 'pull_request' && needs.changes.outputs.rust == 'false' + runs-on: ubuntu-latest + permissions: {} + strategy: + fail-fast: false + matrix: + features: + - name: default + - name: all + steps: + - name: Clippy not required + run: echo "No Rust-relevant paths changed; Clippy is not required." warnings: name: no cargo check warnings needs: [trusted-pr, changes] - if: github.event_name != 'pull_request' || needs.changes.outputs.rust == 'true' + # Clippy already denies warnings across the workspace and all targets on + # PRs. Retain this narrower cargo-check policy gate on main and manual runs. + if: github.event_name != 'pull_request' runs-on: [self-hosted, fireactions-turbo-8] environment: name: ${{ github.event_name == 'workflow_dispatch' && 'sccache-reader' || 'sccache-writer' }} @@ -140,6 +223,9 @@ jobs: # RUSTFLAGS: -D warnings promotes warnings to errors, so cargo's # exit code alone fails the job on any warning. cargo check + - name: Report warnings-check compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Warnings check compiler cache" test: name: cargo test @@ -162,7 +248,16 @@ jobs: sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - name: Install cargo-nextest run: | - curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "$HOME/.cargo/bin" + contract=/etc/fireactions-runner-image/image-contract.env + system_bin=/usr/local/bin/cargo-nextest + if [[ -r "$contract" ]] && \ + grep -Fxq 'CARGO_NEXTEST_VERSION=0.9.140' "$contract" && \ + [[ -x "$system_bin" ]] && \ + "$system_bin" --version 2>/dev/null | grep -Fq 'cargo-nextest 0.9.140 '; then + install -m 0755 "$system_bin" "$HOME/.cargo/bin/cargo-nextest" + else + curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "$HOME/.cargo/bin" + fi cargo nextest --version # nextest runs each test in its own process with better scheduling than # libtest's per-binary threading — typically 2-3x faster on suites this @@ -172,6 +267,9 @@ jobs: run: cargo nextest run --workspace --all-features - name: cargo test --doc --workspace --all-features run: cargo test --doc --workspace --all-features + - name: Report test compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Cargo test compiler cache" # The browser seam: bittensor-core without its `host` feature must keep # compiling for wasm32 (the future wasm-bindgen/TS binding target). This @@ -220,18 +318,32 @@ jobs: node-version: 22 - name: Install wasm-pack run: | - curl -LsSf https://github.com/rustwasm/wasm-pack/releases/download/v0.13.1/wasm-pack-v0.13.1-x86_64-unknown-linux-musl.tar.gz \ - | tar zxf - --strip-components=1 -C "$HOME/.cargo/bin" --wildcards '*/wasm-pack' + contract=/etc/fireactions-runner-image/image-contract.env + system_bin=/usr/local/bin/wasm-pack + if [[ -r "$contract" ]] && \ + grep -Fxq 'WASM_PACK_VERSION=0.13.1' "$contract" && \ + [[ -x "$system_bin" ]] && \ + "$system_bin" --version 2>/dev/null | grep -Fq 'wasm-pack 0.13.1'; then + install -m 0755 "$system_bin" "$HOME/.cargo/bin/wasm-pack" + else + curl -LsSf https://github.com/rustwasm/wasm-pack/releases/download/v0.13.1/wasm-pack-v0.13.1-x86_64-unknown-linux-musl.tar.gz \ + | tar zxf - --strip-components=1 -C "$HOME/.cargo/bin" --wildcards '*/wasm-pack' + fi wasm-pack --version - name: wasm smoke test (node) run: | wasm-pack build sdk/bittensor-core-wasm --target nodejs --out-dir pkg-node node sdk/bittensor-core-wasm/tests/smoke.mjs + - name: Report wasm compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Bittensor core wasm compiler cache" fix: name: cargo fix leaves no diff needs: [trusted-pr, changes] - if: github.event_name != 'pull_request' || needs.changes.outputs.rust == 'true' + # This is a mechanical cleanup policy, not test coverage. Keep it as a + # post-merge/manual guard without spending PR critical-path capacity. + if: github.event_name != 'pull_request' # Compile-bound, same reasoning as clippy. runs-on: [self-hosted, fireactions-turbo-8] environment: @@ -251,6 +363,9 @@ jobs: run: | cargo fix --workspace git diff --exit-code || { echo "'cargo fix --workspace' produced changes; apply them locally."; exit 1; } + - name: Report cargo-fix compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Cargo fix compiler cache" zepter: name: zepter feature propagation @@ -270,5 +385,14 @@ jobs: sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - name: Install and run zepter run: | - cargo install --locked -q zepter + contract=/etc/fireactions-runner-image/image-contract.env + system_bin=/usr/local/bin/zepter + if [[ -r "$contract" ]] && \ + grep -Fxq 'ZEPTER_VERSION=1.88.1' "$contract" && \ + [[ -x "$system_bin" ]] && \ + "$system_bin" --version 2>/dev/null | grep -Fq '1.88.1'; then + install -m 0755 "$system_bin" "$HOME/.cargo/bin/zepter" + else + cargo install --locked -q zepter + fi zepter run check diff --git a/.github/workflows/eco-tests.yml b/.github/workflows/eco-tests.yml index 9a531c4922..b4fcd42702 100644 --- a/.github/workflows/eco-tests.yml +++ b/.github/workflows/eco-tests.yml @@ -43,19 +43,56 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} + EXPECTED_CHANGED_FILES: ${{ github.event.pull_request.changed_files }} run: | set -euo pipefail - files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename') - pattern='^(eco-tests|common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^(Cargo\.(toml|lock)|build\.rs|rust-toolchain\.toml)$|^\.github/(workflows/eco-tests\.yml|actions/(rust-setup|sccache-setup)/|scripts/sccache-configure\.sh$)' - if grep -qE "$pattern" <<< "$files"; then + select_rust() { echo "rust=true" >> "$GITHUB_OUTPUT" - else - echo "no eco-tests-relevant paths changed" - echo "rust=false" >> "$GITHUB_OUTPUT" + } + + if ! pages=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --slurp); then + echo "::warning::PR file listing failed; running eco-tests." + select_rust + exit 0 + fi + if ! fetched_files=$(jq -er '[.[][]] | length' <<< "$pages"); then + echo "::warning::PR file response was invalid; running eco-tests." + select_rust + exit 0 fi + if [[ "$fetched_files" -ne "$EXPECTED_CHANGED_FILES" ]]; then + echo "::warning::PR file listing was incomplete ($fetched_files/$EXPECTED_CHANGED_FILES); running eco-tests." + select_rust + exit 0 + fi + + # Check both sides of renames. A production file moved to a nominally + # safe directory must still exercise the crate that previously used it. + if ! files=$(jq -er ' + if all(.[][]; + type == "object" and + (.filename | type == "string" and length > 0) and + ((has("previous_filename") | not) or (.previous_filename | type == "string")) + ) + then .[][] | .filename, (.previous_filename // empty) + else error("invalid pull-file entry") + end + ' <<< "$pages"); then + echo "::warning::PR file paths were invalid; running eco-tests." + select_rust + exit 0 + fi + pattern='^(eco-tests|common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^\.cargo/|^(Cargo\.(toml|lock)|build\.rs|rust-toolchain\.toml)$|^\.github/(workflows/eco-tests\.yml|actions/(rust-setup|sccache-setup)/|scripts/(rust-setup-preflight|install-rust-toolchain|sccache-configure)\.sh$|scripts/(sccache-config|sccache-report)\.(py|sh)$)' + if grep -qE "$pattern" <<< "$files"; then + select_rust + exit 0 + fi + + echo "no eco-tests-relevant paths changed" + echo "rust=false" >> "$GITHUB_OUTPUT" eco-tests: - name: cargo test (eco-tests) + name: run cargo test (eco-tests) needs: [trusted-pr, changes] if: github.event_name != 'pull_request' || needs.changes.outputs.rust == 'true' runs-on: [self-hosted, fireactions-turbo-8] @@ -78,3 +115,36 @@ jobs: - name: cargo test working-directory: eco-tests run: cargo test + + # Keep the required context fail-closed. If planning or the trusted-runner + # guard fails, the real job is skipped; GitHub otherwise treats that skipped + # required job as satisfied. This hosted fan-in distinguishes an intentional + # non-Rust skip from a missing test run. + eco-tests-gate: + name: cargo test (eco-tests) + needs: [trusted-pr, changes, eco-tests] + if: always() + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Evaluate eco-test result + env: + EVENT_NAME: ${{ github.event_name }} + TRUSTED: ${{ needs.trusted-pr.result }} + CHANGES: ${{ needs.changes.result }} + RUST_RELEVANT: ${{ needs.changes.outputs.rust }} + TEST_RESULT: ${{ needs.eco-tests.result }} + run: | + echo "event=$EVENT_NAME trusted=$TRUSTED changes=$CHANGES rust=$RUST_RELEVANT test=$TEST_RESULT" + if [ "$CHANGES" != success ]; then + exit 1 + fi + if [ "$EVENT_NAME" != pull_request ]; then + [ "$TEST_RESULT" = success ] + exit + fi + case "$RUST_RELEVANT" in + false) [ "$TEST_RESULT" = skipped ] ;; + true) [ "$TRUSTED" = success ] && [ "$TEST_RESULT" = success ] ;; + *) echo "invalid eco-test selection: '$RUST_RELEVANT'" >&2; exit 1 ;; + esac diff --git a/.github/workflows/refresh-mainnet-snapshot.yml b/.github/workflows/refresh-mainnet-snapshot.yml index cc98c77aa6..1cc63e6278 100644 --- a/.github/workflows/refresh-mainnet-snapshot.yml +++ b/.github/workflows/refresh-mainnet-snapshot.yml @@ -23,6 +23,13 @@ on: - cron: "0 2 * * *" workflow_dispatch: inputs: + source_ref: + description: "Trusted branch to refresh" + required: true + default: main + type: choice + options: + - main try_runtime_only: description: "Generate only try-runtime snapshots (targeted recovery)" type: boolean @@ -30,6 +37,7 @@ on: permissions: contents: read + actions: read concurrency: # Serialize every ref globally: diagnostic dispatches must not cancel a @@ -46,10 +54,12 @@ env: jobs: snapshot: name: scrape mainnet and publish snapshot - if: github.event_name != 'workflow_dispatch' || !inputs.try_runtime_only + if: >- + (github.event_name != 'workflow_dispatch' || !inputs.try_runtime_only) && + (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') runs-on: [self-hosted, fireactions-turbo-8] environment: - name: ${{ github.event_name == 'workflow_dispatch' && 'sccache-reader' || 'sccache-writer' }} + name: sccache-writer deployment: false timeout-minutes: 120 steps: @@ -89,6 +99,7 @@ jobs: ls -lh mainnet-snapshot.tar.gz - name: Upload snapshot artifact + id: upload-snapshot uses: actions/upload-artifact@v4 with: name: mainnet-snapshot @@ -98,6 +109,20 @@ jobs: # The tarball is already gzipped. compression-level: 0 + - name: Mirror snapshot for fleet prefetch + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.upload-snapshot.outputs.artifact-id }} + ARTIFACT_DIGEST: ${{ steps.upload-snapshot.outputs.artifact-digest }} + run: | + set -euo pipefail + .github/scripts/publish-artifact-mirror.sh \ + "$ARTIFACT_ID" \ + mainnet-snapshot \ + "$ARTIFACT_DIGEST" \ + "$GITHUB_SHA" \ + .github/workflows/refresh-mainnet-snapshot.yml + # try-runtime state snapshots, one job per network so one flaky endpoint # doesn't prevent the others from publishing. The .snap format # is tied to the try-runtime-cli release, so this version must match the @@ -243,3 +268,40 @@ jobs: ${{ matrix.network.name }}.manifest.json if-no-files-found: error retention-days: 7 + + mirror-try-runtime-snapshots: + name: mirror try-runtime snapshot ${{ matrix.network }} + needs: try-runtime-snapshot + # Publish every successful matrix artifact even if another network scrape + # failed; the missing network remains visible as a failed mirror leg. + if: >- + always() && + github.ref == 'refs/heads/main' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.source_ref == 'main')) + runs-on: ubuntu-latest + environment: + name: sccache-writer + deployment: false + timeout-minutes: 45 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: + network: [mainnet, testnet, devnet] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Publish immutable artifact and latest manifest + env: + GH_TOKEN: ${{ github.token }} + SCCACHE_BUCKET: subtensor-ci-sccache + SCCACHE_ENDPOINT: https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com + SCCACHE_REGION: auto + AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + run: >- + .github/scripts/publish-current-run-artifact-mirror.sh + "try-runtime-snap-v${TRY_RUNTIME_VERSION}-${{ matrix.network }}" diff --git a/.github/workflows/runtime-checks.yml b/.github/workflows/runtime-checks.yml index a5e9f100e6..ef1a82e26a 100644 --- a/.github/workflows/runtime-checks.yml +++ b/.github/workflows/runtime-checks.yml @@ -88,13 +88,29 @@ jobs: echo "snapshot_ci=true" } >> "$GITHUB_OUTPUT" } + publish_classification() { + local classification=$1 key value + if [[ "$(wc -l < "$classification" | tr -d '[:space:]')" != 5 ]]; then + return 1 + fi + for key in runtime docs python_sdk sdk_drift snapshot_ci; do + value=$(sed -n "s/^${key}=//p" "$classification") + [[ "$value" == true || "$value" == false ]] || return 1 + [[ "$(grep -c "^${key}=" "$classification")" == 1 ]] || return 1 + done + cat "$classification" >> "$GITHUB_OUTPUT" + } classifier=.trusted-runtime-filter/.github/scripts/classify-runtime-changes.sh if [[ ! -f "$classifier" ]]; then echo "::warning::Trusted base predates the path classifier; enabling every check." enable_all exit 0 fi - pages=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --slurp) + if ! pages=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --slurp); then + echo "::warning::PR file listing failed; enabling every check." + enable_all + exit 0 + fi fetched_files=$(jq '[.[][]] | length' <<< "$pages") if [[ "$fetched_files" -ne "$CHANGED_FILES" ]]; then echo "::warning::PR file listing was incomplete ($fetched_files/$CHANGED_FILES); enabling every check." @@ -103,8 +119,14 @@ jobs: fi # A rename can move a runtime file outside a matched directory, so # classify both the current and previous paths. - jq -r '.[][] | .filename, (.previous_filename // empty)' <<< "$pages" \ - | bash "$classifier" "$GITHUB_OUTPUT" + classification=$(mktemp) + trap 'rm -f "$classification"' EXIT + if ! jq -r '.[][] | .filename, (.previous_filename // empty)' <<< "$pages" \ + | bash "$classifier" "$classification" || + ! publish_classification "$classification"; then + echo "::warning::Runtime classifier failed or emitted invalid outputs; enabling every check." + enable_all + fi snapshot-artifact-tests: name: snapshot artifact contract tests @@ -114,7 +136,10 @@ jobs: steps: - uses: actions/checkout@v4 - run: .github/scripts/test-snapshot-artifact.sh + - run: .github/scripts/test-download-artifact.sh + - run: .github/scripts/test-select-shared-release-artifact.sh - run: .github/scripts/test-runtime-change-filter.sh + - run: .github/scripts/test-clone-regression-phase.sh # Build the try-runtime wasm independently so its consumers do not wait for # the unrelated release node build. The source check also makes the @@ -167,6 +192,10 @@ jobs: - name: Build runtime wasm (production, try-runtime feature) run: cargo build --profile production -p node-subtensor-runtime --features try-runtime -q --locked + - name: Report try-runtime compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Try-runtime wasm compiler cache" + - name: Upload try-runtime wasm uses: actions/upload-artifact@v4 with: @@ -184,6 +213,7 @@ jobs: if: >- (github.event_name != 'pull_request' || needs.changes.outputs.runtime == 'true') && !(github.event_name == 'workflow_dispatch' && inputs.try_runtime_only) + # Turbo runners probe the host-local tier and fail open to direct R2. runs-on: [self-hosted, fireactions-turbo-8] environment: name: ${{ github.event_name == 'workflow_dispatch' && 'sccache-reader' || 'sccache-writer' }} @@ -207,12 +237,33 @@ jobs: rustup component add rust-src - name: Build node-subtensor (release) - run: cargo build --release -p node-subtensor + run: | + set -euo pipefail + sccache --zero-stats + started=$(date -u +%s) + cargo build --release -p node-subtensor + seconds=$(($(date -u +%s) - started)) + { + echo "### Release node compile" + echo "- Wall time: ${seconds}s" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Report release-node compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Release node compiler cache" + + - name: Validate release node artifact + run: | + test -s target/release/node-subtensor + test -s target/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm + chmod +x target/release/node-subtensor - name: Upload release node + runtime wasm uses: actions/upload-artifact@v4 with: - name: node-release + # The exact merge SHA lets TypeScript E2E prove this artifact was + # built from the same source while clone consumers reuse it in-run. + name: node-subtensor-release-${{ github.sha }} path: | target/release/node-subtensor target/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm @@ -225,7 +276,7 @@ jobs: try-runtime: name: try-runtime ${{ matrix.network.name }} needs: build-try-runtime - runs-on: ubuntu-latest + runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 90 permissions: contents: read @@ -316,15 +367,18 @@ jobs: run: echo "SNAPSHOT_RESTORE_STARTED=$(date -u +%s)" >> "$GITHUB_ENV" - name: Download selected try-runtime state snapshot + id: download-snapshot if: steps.state-mode.outputs.fresh-state != 'true' - uses: actions/download-artifact@v4 - with: - artifact-ids: ${{ steps.snapshot.outputs.artifact-id }} - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ steps.snapshot.outputs.run-id }} - path: snap - merge-multiple: true + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + run: | + .github/scripts/download-artifact.sh \ + "${{ steps.snapshot.outputs.artifact-id }}" \ + "try-runtime-snap-v${TRY_RUNTIME_VERSION}-${{ matrix.network.name }}" \ + "${{ steps.snapshot.outputs.artifact-digest }}" \ + "${{ steps.snapshot.outputs.artifact-size-bytes }}" \ + snap "$GITHUB_OUTPUT" - name: Validate try-runtime state snapshot if: steps.state-mode.outputs.fresh-state != 'true' @@ -343,6 +397,7 @@ jobs: { echo "### ${{ matrix.network.name }} cached state" echo "- Artifact: ${{ steps.snapshot.outputs.artifact-id }} (run ${{ steps.snapshot.outputs.run-id }})" + echo "- Download source: ${{ steps.download-snapshot.outputs.source }}" echo "- Age: ${{ steps.snapshot.outputs.age-hours }}h" echo "- Finalized block: $block" echo "- Runtime: $spec" @@ -447,25 +502,101 @@ jobs: uv run python -m codegen.check --coverage uv run python -m codegen.check --names + # Decide whether the clone regressions can fan out over the immutable + # snapshot. Fresh-state diagnostics and a missing trusted snapshot retain + # the single-job live-scrape path, avoiding duplicate archive RPC load. + # Execute the selector from the trusted base revision so a PR cannot choose + # the smaller matrix by modifying its own routing helper. + clone-plan: + name: plan clone regression execution + needs: changes + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + use_snapshot: ${{ steps.plan.outputs.use_snapshot }} + artifact_id: ${{ steps.plan.outputs.artifact_id }} + run_id: ${{ steps.plan.outputs.run_id }} + steps: + - name: Check out trusted snapshot selector + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} + sparse-checkout: .github/scripts/snapshot-artifact.sh + sparse-checkout-cone-mode: false + path: .trusted-clone-plan + persist-credentials: false + + - name: Select parallel or combined execution + id: plan + env: + GH_TOKEN: ${{ github.token }} + RUNTIME_RELEVANT: ${{ needs.changes.outputs.runtime }} + FRESH_CLONE: ${{ contains(github.event.pull_request.labels.*.name, 'fresh-mainnet-clone') }} + run: | + set -euo pipefail + combined() { + echo 'matrix={"phase":["combined"]}' >> "$GITHUB_OUTPUT" + echo 'use_snapshot=false' >> "$GITHUB_OUTPUT" + } + + if [[ "$GITHUB_EVENT_NAME" == pull_request && "$RUNTIME_RELEVANT" != true ]]; then + combined + exit 0 + fi + if [[ "$FRESH_CLONE" == true ]]; then + echo "fresh-mainnet-clone requested; retaining one live scrape" + combined + exit 0 + fi + + selector=.trusted-clone-plan/.github/scripts/snapshot-artifact.sh + if [[ ! -x "$selector" ]]; then + echo "trusted snapshot selector unavailable; retaining one live scrape" + combined + exit 0 + fi + + selection=$(mktemp) + trap 'rm -f "$selection"' EXIT + if "$selector" select \ + mainnet-snapshot \ + "${{ github.event.repository.default_branch }}" \ + "${{ github.repository_id }}" \ + .github/workflows/refresh-mainnet-snapshot.yml \ + 168 "$selection" optional && + grep -qx 'found=true' "$selection"; then + echo 'matrix={"phase":["pristine","remaining"]}' >> "$GITHUB_OUTPUT" + echo 'use_snapshot=true' >> "$GITHUB_OUTPUT" + sed -n \ + -e 's/^artifact-id=/artifact_id=/p' \ + -e 's/^run-id=/run_id=/p' \ + "$selection" >> "$GITHUB_OUTPUT" + echo "trusted snapshot available; pinning one artifact for parallel clone phases" + else + echo "trusted snapshot unavailable; retaining one live scrape" + combined + fi + # For every PR: sudo-upgrade a local clone of mainnet with the proposed # runtime, then run the clone regression suite and the SDK metadata drift # gate against the upgraded chain. (Chain-facing SDK e2e coverage lives in # the Rust suite under check-bittensor-e2e-tests.yml.) Consumes the - # node-release artifact instead of building, and restores the nightly + # exact-merge release artifact instead of building, and restores the nightly # mainnet-snapshot artifact instead of re-scraping mainnet state (falls back # to a live scrape when no snapshot exists or the `fresh-mainnet-clone` # label is set). Auto-skips (via the `build-node-release` need) when the PR touches # nothing runtime-relevant. # - # Interval sealing advances runtime time by one 12-second slot every 250ms, - # so the block-driven regressions can run sequentially on one runner. The - # issuance-invariant test gets a pristine clone; the already-downloaded - # checkpoint is then restored locally for the remaining tests, before - # forceSetBalance-based fixtures make the issuance mirrors diverge. This - # avoids four runners and four downloads without sharing destructive state. + # Interval sealing advances runtime time by one 12-second slot every 250ms. + # With a trusted snapshot available, the pristine issuance invariant and + # the state-mutating remaining regressions restore independent copies and + # run in parallel. A live scrape retains the combined sequential path. clone-upgrade: - name: clone-upgrade - needs: [trusted-pr, changes, build-node-release] + name: clone-upgrade (${{ matrix.phase }}) + needs: [trusted-pr, changes, clone-plan, build-node-release] runs-on: [self-hosted, fireactions-turbo-8] if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-clone-upgrade') }} timeout-minutes: 90 @@ -473,13 +604,18 @@ jobs: contents: read # Cross-run artifact download (the nightly mainnet snapshot). actions: read + strategy: + fail-fast: false + # Missing planner output falls back to the complete sequential suite; + # it must never create an empty matrix that silently skips coverage. + matrix: ${{ fromJSON(needs.clone-plan.outputs.matrix || '{"phase":["combined"]}') }} steps: - uses: actions/checkout@v4 - name: Download release node + runtime wasm uses: actions/download-artifact@v4 with: - name: node-release + name: node-subtensor-release-${{ github.sha }} path: target/release - name: Make node binary executable @@ -490,63 +626,65 @@ jobs: with: node-version: 20 - - name: Select mainnet clone snapshot - id: clone-snapshot - if: ${{ !contains(github.event.pull_request.labels.*.name, 'fresh-mainnet-clone') }} - continue-on-error: true + # The trusted base selector pins provenance and the immutable artifact + # ID. Read integrity metadata for that exact ID here so rollout remains + # compatible with base revisions that predate digest/size outputs. + - name: Validate pinned mainnet snapshot metadata + if: needs.clone-plan.outputs.use_snapshot == 'true' env: GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ needs.clone-plan.outputs.artifact_id }} run: | set -euo pipefail - .github/scripts/snapshot-artifact.sh select \ - mainnet-snapshot \ - "${{ github.event.repository.default_branch }}" \ - "${{ github.repository_id }}" \ - .github/workflows/refresh-mainnet-snapshot.yml \ - 168 "$GITHUB_OUTPUT" optional + metadata="$RUNNER_TEMP/pinned-mainnet-snapshot.json" + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID" > "$metadata" + jq -e --arg id "$ARTIFACT_ID" ' + (.id | tostring) == $id and + .name == "mainnet-snapshot" and + .expired == false and + (.size_in_bytes | type) == "number" and .size_in_bytes > 0 and + (.digest | type) == "string" and + (.digest | test("^sha256:[0-9a-f]{64}$")) + ' "$metadata" > /dev/null - name: Download selected mainnet clone snapshot id: download-clone-snapshot - if: steps.clone-snapshot.outputs.found == 'true' - continue-on-error: true - uses: actions/download-artifact@v4 - with: - artifact-ids: ${{ steps.clone-snapshot.outputs.artifact-id }} - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ steps.clone-snapshot.outputs.run-id }} - path: /tmp/mainnet-snapshot - merge-multiple: true + if: needs.clone-plan.outputs.use_snapshot == 'true' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + ARTIFACT_ID: ${{ needs.clone-plan.outputs.artifact_id }} + run: | + set -euo pipefail + metadata="$RUNNER_TEMP/pinned-mainnet-snapshot.json" + .github/scripts/download-artifact.sh \ + "$ARTIFACT_ID" \ + mainnet-snapshot \ + "$(jq -er '.digest' "$metadata")" \ + "$(jq -er '.size_in_bytes' "$metadata")" \ + /tmp/mainnet-snapshot "$GITHUB_OUTPUT" - name: Restore mainnet clone snapshot id: restore-clone-snapshot - if: steps.download-clone-snapshot.outcome == 'success' - continue-on-error: true + if: needs.clone-plan.outputs.use_snapshot == 'true' run: | set -euo pipefail checkpoint=/tmp/mainnet-snapshot/mainnet-snapshot.tar.gz ./clones/scripts/local-clone-checkpoint.sh restore "$checkpoint" - echo "Restored artifact ${{ steps.clone-snapshot.outputs.artifact-id }} from run ${{ steps.clone-snapshot.outputs.run-id }}." + echo "Restored artifact ${{ needs.clone-plan.outputs.artifact_id }} from run ${{ needs.clone-plan.outputs.run_id }}." + { + echo "### Mainnet clone snapshot" + echo "- Download source: ${{ steps.download-clone-snapshot.outputs.source }}" + echo "- Download and extraction: ${{ steps.download-clone-snapshot.outputs.seconds }}s" + echo "- Artifact: ${{ needs.clone-plan.outputs.artifact_id }}" + } >> "$GITHUB_STEP_SUMMARY" # Tells start-local-clone.sh to keep the pre-initialized database # instead of wiping it and re-running genesis init. echo "KEEP_CLONE_DATA=1" >> "$GITHUB_ENV" echo "CLONE_CHECKPOINT=$checkpoint" >> "$GITHUB_ENV" - - name: Fall back after an unusable clone snapshot - if: >- - steps.clone-snapshot.outputs.found == 'true' && - (steps.download-clone-snapshot.outcome != 'success' || - steps.restore-clone-snapshot.outcome != 'success') - run: | - echo "::warning::selected clone snapshot was unusable; falling back to a live mainnet scrape" - ./clones/scripts/local-clone-checkpoint.sh clear - rm -rf /tmp/mainnet-snapshot - - - name: Fall back after clone snapshot lookup failure - if: steps.clone-snapshot.outcome == 'failure' - run: echo "::warning::clone snapshot lookup failed; falling back to a live mainnet scrape" - - name: Create mainnet clone + if: needs.clone-plan.outputs.use_snapshot != 'true' # No-op when the snapshot restored the chainspec; otherwise scrapes # mainnet state live (first run, expired snapshot, or the # fresh-mainnet-clone label). The live scrape's warp sync sometimes @@ -568,7 +706,7 @@ jobs: exit 1 - name: Prepare a reusable checkpoint after live fallback - if: steps.restore-clone-snapshot.outcome != 'success' + if: needs.clone-plan.outputs.use_snapshot != 'true' run: | set -euo pipefail ./clones/scripts/start-local-clone-and-wait.sh manual @@ -581,64 +719,12 @@ jobs: working-directory: clones/js-tests run: npm ci - - name: Start isolated issuance clone - run: ./clones/scripts/start-local-clone-and-wait.sh accelerated - - - name: Sudo-upgrade issuance clone with proposed runtime - working-directory: clones/js-tests - run: | - # Right after genesis the node occasionally rejects the first - # extrinsic with "bad signature" (observed in the former sharded - # workflow while identical transactions succeeded seconds later). - # Setting the same runtime code twice is idempotent, so retry once. - npm run runtime:update:alice || { sleep 15; npm run runtime:update:alice; } - - - name: Run isolated issuance regression - working-directory: clones/js-tests + - name: Run clone regression phase env: - CLONE_REGRESSION_PHASE: pristine - CLONE_REGRESSION_TIMEOUT_MS: 1800000 - run: npm run test:clone-regressions - - - name: Stop isolated issuance clone - run: ./clones/scripts/stop-local-clone.sh - - - name: Restore pristine clone for remaining regressions - run: ./clones/scripts/local-clone-checkpoint.sh restore "$CLONE_CHECKPOINT" - - - name: Start regression clone - run: ./clones/scripts/start-local-clone-and-wait.sh accelerated - - - name: Sudo-upgrade regression clone with proposed runtime - working-directory: clones/js-tests - run: npm run runtime:update:alice || { sleep 15; npm run runtime:update:alice; } - - - name: Run clone smoke tests - working-directory: clones/js-tests - run: npm test - - - name: Run remaining clone regression tests - working-directory: clones/js-tests - env: - CLONE_REGRESSION_PHASE: remaining - CLONE_REGRESSION_TIMEOUT_MS: 1800000 - run: npm run test:clone-regressions - - - name: Install uv - if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} - run: | - curl -LsSf https://astral.sh/uv/0.11.28/install.sh | sh - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Sync SDK environment - if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} - working-directory: sdk/python - run: uv sync --locked --all-extras --dev - - - name: Metadata drift gate (committed _generated vs upgraded clone) - if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} - working-directory: sdk/python - run: uv run python -m codegen.check --drift ${{ env.WS_ENDPOINT }} + # The harness owns the phase lifecycle and cleanup. YAML only decides + # which independent phase runs and whether SDK drift is relevant. + RUN_SDK_DRIFT: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} + run: ./clones/scripts/run-clone-regression-phase.sh "${{ matrix.phase }}" - name: Dump clone node and harness logs if: failure() @@ -651,10 +737,6 @@ jobs: cat "$f" done - - name: Stop local clone - if: always() - run: ./clones/scripts/stop-local-clone.sh - # Branch protection requires a check named exactly "Sudo-upgrade mainnet # clone and test"; this fan-in preserves the required-check name. It passes # when the clone job succeeded, or when the whole suite legitimately skipped @@ -671,9 +753,10 @@ jobs: CHANGES: ${{ needs.changes.result }} CLONE: ${{ needs.clone-upgrade.result }} BUILD: ${{ needs.build-node-release.result }} + RUNTIME_RELEVANT: ${{ needs.changes.outputs.runtime }} LABEL_SKIP: ${{ contains(github.event.pull_request.labels.*.name, 'skip-clone-upgrade') }} run: | - echo "changes=$CHANGES clone=$CLONE build=$BUILD label-skip=$LABEL_SKIP" + echo "changes=$CHANGES clone=$CLONE build=$BUILD runtime=$RUNTIME_RELEVANT label-skip=$LABEL_SKIP" # A failed path-filter job must not read as "nothing runtime-related # changed" — that would let build+clone skip and the gate pass. if [ "$CHANGES" != "success" ]; then @@ -682,7 +765,14 @@ jobs: if [ "$CLONE" = "success" ]; then exit 0 fi - if [ "$CLONE" = "skipped" ] && { [ "$LABEL_SKIP" = "true" ] || [ "$BUILD" = "skipped" ]; }; then + if [ "$CLONE" = "skipped" ] && [ "$LABEL_SKIP" = "true" ]; then + exit 0 + fi + case "$RUNTIME_RELEVANT" in + true|false) ;; + *) echo "invalid runtime selection: '$RUNTIME_RELEVANT'" >&2; exit 1 ;; + esac + if [ "$CLONE" = "skipped" ] && [ "$BUILD" = "skipped" ] && [ "$RUNTIME_RELEVANT" = "false" ]; then exit 0 fi exit 1 diff --git a/.github/workflows/sccache-warm.yml b/.github/workflows/sccache-warm.yml index 9ee10b4d21..e98c68228f 100644 --- a/.github/workflows/sccache-warm.yml +++ b/.github/workflows/sccache-warm.yml @@ -2,7 +2,7 @@ name: Warm R2 sccache # Normal same-repository PR and main CI writes through while it compiles. This # workflow is maintenance only: refresh deployed-state branches when they move, -# repair expired/missing main entries at 12:00 UTC, or recover manually. +# repair expired/missing main entries once daily, or recover manually. on: push: @@ -10,7 +10,7 @@ on: - devnet - testnet schedule: - - cron: "0 12 * * *" + - cron: "0 11 * * *" workflow_dispatch: inputs: source_ref: @@ -45,12 +45,24 @@ jobs: timeout-minutes: 120 env: SKIP_WASM_BUILD: 1 + CAPTURE_HOST_WARM_SET: >- + ${{ github.ref == 'refs/heads/main' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.source_ref == 'main')) && + 'true' || 'false' }} steps: - name: Check out trusted cache source uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.source_ref || github.sha }} + - name: Enable exact compiler-key capture + if: env.CAPTURE_HOST_WARM_SET == 'true' + shell: bash + run: | + echo "SCCACHE_ERROR_LOG=$RUNNER_TEMP/sccache-warm-native.log" >> "$GITHUB_ENV" + echo "SCCACHE_LOG=sccache::compiler::compiler=debug" >> "$GITHUB_ENV" + - name: Install Rust and enable trusted R2 writer uses: ./.github/actions/rust-setup with: @@ -59,6 +71,9 @@ jobs: sccache-credential-mode: writer sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + # This maintenance run must observe durable R2 directly. A local hit + # must not hide an origin object that has expired or gone missing. + sccache-local-tier: disabled - name: Require protected writer activation shell: bash @@ -70,47 +85,24 @@ jobs: exit 1 fi - - name: Warm default workspace check artifacts - run: cargo check --workspace --locked - - - name: Warm all-feature workspace check artifacts - run: cargo check --workspace --all-features --locked - - - name: Warm default clippy artifacts - run: cargo clippy --workspace --all-targets --locked -- -D warnings - - - name: Warm all-feature clippy artifacts - run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - - - name: Warm deny-warnings check artifacts - run: RUSTFLAGS="-D warnings" cargo check --locked - - - name: Warm workspace test artifacts - run: cargo test --workspace --all-features --locked --no-run + - name: Warm native Rust compiler cache + uses: ./.github/actions/warm-native-cache - - name: Warm eco-tests artifacts - working-directory: eco-tests - run: cargo test --no-run + - name: Extract exact native compiler object keys + if: env.CAPTURE_HOST_WARM_SET == 'true' + run: >- + .github/scripts/r2-sccache-warmset.py extract + "$SCCACHE_ERROR_LOG" + "$RUNNER_TEMP/sccache-warm-keys/native.keys" - - name: Install runtime wasm toolchain components - run: | - rustup target add wasm32-unknown-unknown - rustup component add rust-src - - - name: Warm try-runtime wasm artifacts - run: | - unset SKIP_WASM_BUILD - cargo build --profile production -p node-subtensor-runtime --features try-runtime -q --locked - - - name: Warm release node artifacts - run: | - unset SKIP_WASM_BUILD - cargo build --release --locked -p node-subtensor - - - name: Warm fast-runtime release node artifacts - run: | - unset SKIP_WASM_BUILD - cargo build --release --locked -p node-subtensor --features fast-runtime + - name: Upload native compiler object keys + if: env.CAPTURE_HOST_WARM_SET == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sccache-warm-keys-native-${{ github.run_id }} + path: ${{ runner.temp }}/sccache-warm-keys/native.keys + if-no-files-found: error + retention-days: 1 production-amd64: name: Populate amd64 production cache @@ -124,12 +116,25 @@ jobs: deployment: false runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 120 + env: + CAPTURE_HOST_WARM_SET: >- + ${{ github.ref == 'refs/heads/main' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.source_ref == 'main')) && + 'true' || 'false' }} steps: - name: Check out trusted cache source uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.source_ref || github.sha }} + - name: Enable exact compiler-key capture + if: env.CAPTURE_HOST_WARM_SET == 'true' + shell: bash + run: | + echo "SCCACHE_ERROR_LOG=$RUNNER_TEMP/sccache-warm-production.log" >> "$GITHUB_ENV" + echo "SCCACHE_LOG=sccache::compiler::compiler=debug" >> "$GITHUB_ENV" + - name: Warm amd64 production binary artifacts uses: ./.github/actions/build-production-binary with: @@ -139,3 +144,68 @@ jobs: sccache-credential-mode: writer sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + sccache-local-tier: disabled + + - name: Extract exact production compiler object keys + if: env.CAPTURE_HOST_WARM_SET == 'true' + run: >- + .github/scripts/r2-sccache-warmset.py extract + "$SCCACHE_ERROR_LOG" + "$RUNNER_TEMP/sccache-warm-keys/production.keys" + + - name: Upload production compiler object keys + if: env.CAPTURE_HOST_WARM_SET == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sccache-warm-keys-production-${{ github.run_id }} + path: ${{ runner.temp }}/sccache-warm-keys/production.keys + if-no-files-found: error + retention-days: 1 + + publish-host-warm-set: + name: Publish bounded host warm set + needs: [native, production-amd64] + if: >- + always() && + needs.native.result == 'success' && + needs.production-amd64.result == 'success' && + github.ref == 'refs/heads/main' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.source_ref == 'main')) + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: sccache-writer + deployment: false + permissions: + contents: read + actions: read + steps: + - name: Check out trusted publisher + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download production compiler keys + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: sccache-warm-keys-production-${{ github.run_id }} + path: ${{ runner.temp }}/sccache-warm-keys/production + + - name: Download native compiler keys + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: sccache-warm-keys-native-${{ github.run_id }} + path: ${{ runner.temp }}/sccache-warm-keys/native + + # Production keys are passed first so the node binary remains in the + # bounded set even if every native variant together exceeds 6 GiB. + - name: Publish the latest generation-stamped warm-set manifest + env: + SCCACHE_BUCKET: subtensor-ci-sccache + SCCACHE_ENDPOINT: https://3dc4cebb791314d78848969042fb3382.r2.cloudflarestorage.com + SCCACHE_REGION: auto + AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} + run: >- + .github/scripts/r2-sccache-warmset.py publish + "$RUNNER_TEMP/sccache-warm-keys/production/production.keys" + "$RUNNER_TEMP/sccache-warm-keys/native/native.keys" diff --git a/.github/workflows/typescript-e2e.yml b/.github/workflows/typescript-e2e.yml index 45d76274c2..86a0bf8f89 100644 --- a/.github/workflows/typescript-e2e.yml +++ b/.github/workflows/typescript-e2e.yml @@ -2,6 +2,8 @@ name: Typescript E2E Tests on: pull_request: + push: + branches: [staging, main] concurrency: group: typescript-e2e-${{ github.ref }} @@ -29,10 +31,50 @@ jobs: name: detect e2e-relevant changes runs-on: ubuntu-latest permissions: + contents: read pull-requests: read outputs: - e2e: ${{ steps.filter.outputs.e2e }} + evm: ${{ steps.filter.outputs.evm }} + shield: ${{ steps.filter.outputs.shield }} + topology_audit: ${{ steps.filter.outputs.topology_audit }} + state_count: ${{ steps.plan.outputs.state_count }} + state_matrix: ${{ steps.plan.outputs.state_matrix }} + shield_count: ${{ steps.plan.outputs.shield_count }} + shield_matrix: ${{ steps.plan.outputs.shield_matrix }} + build_count: ${{ steps.plan.outputs.build_count }} + build_matrix: ${{ steps.plan.outputs.build_matrix }} + runtime_release: ${{ steps.filter.outputs.runtime_release }} steps: + # Routing decisions for pull requests must come from the trusted base + # revision. A PR may change these scripts, but those changes take effect + # only after merge; during initial rollout, their absence fails closed. + - name: Check out trusted E2E classifier + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} + sparse-checkout: | + .github/scripts/classify-typescript-e2e-changes.sh + .github/scripts/classify-runtime-changes.sh + .github/scripts/extract-pull-file-paths.sh + ts-tests/scripts/e2e-shard-plan.mjs + path: .trusted-e2e-filter + persist-credentials: false + + # Shard topology is data from the proposed revision. A generic builder + # from the trusted base validates and expands it, so a new shard is + # exercised by the PR that introduces it rather than only after merge. + - name: Check out proposed E2E shard manifest + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + sparse-checkout: | + ts-tests/e2e-shards.json + ts-tests/scripts/e2e-shard-plan.mjs + sparse-checkout-cone-mode: false + path: .proposed-e2e-plan + persist-credentials: false + # Plain gh-api file listing instead of a marketplace action: the org's # Actions allowlist rejects unlisted third-party actions (startup_failure). - name: Filter changed paths @@ -40,18 +82,92 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} + EXPECTED_CHANGED_FILES: ${{ github.event.pull_request.changed_files }} run: | set -euo pipefail - files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename') - # SDK-only lockfile movement is covered by SDK checks; do not run - # zombienet unless chain/runtime or ts-tests inputs changed. - pattern='^(ts-tests|common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^(Cargo\.toml|build\.rs|rust-toolchain\.toml)$|^\.github/(workflows/typescript-e2e\.yml|actions/(run-typescript-e2e|rust-setup|sccache-setup)/|scripts/sccache-configure\.sh$)' - if grep -qE "$pattern" <<< "$files"; then - echo "e2e=true" >> "$GITHUB_OUTPUT" + classifier=.trusted-e2e-filter/.github/scripts/classify-typescript-e2e-changes.sh + extractor=.trusted-e2e-filter/.github/scripts/extract-pull-file-paths.sh + + select_all() { + { + echo 'evm=true' + echo 'staking=true' + echo 'coldkey_swap=true' + echo 'dev=true' + echo 'subnets=true' + echo 'shield=true' + echo 'topology_audit=true' + echo 'runtime_release=true' + } >> "$GITHUB_OUTPUT" + } + + # During rollout the trusted scripts may be absent from the base + # revision. Never fall back to PR-controlled code in that bootstrap + # window: run every E2E suite instead. + if [[ ! -x "$classifier" || ! -x "$extractor" ]]; then + echo "trusted E2E classifier unavailable; running the full matrix" >&2 + select_all + exit 0 + fi + + if [[ "$GITHUB_EVENT_NAME" != pull_request ]]; then + "$classifier" --all "$GITHUB_OUTPUT" + exit 0 + fi + + if ! files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate); then + echo "changed-file lookup failed; running the full E2E matrix" >&2 + "$classifier" --all "$GITHUB_OUTPUT" + exit 0 + fi + + if ! paths=$(printf '%s\n' "$files" | "$extractor" "$EXPECTED_CHANGED_FILES"); then + echo "changed-file response invalid; running the full E2E matrix" >&2 + "$classifier" --all "$GITHUB_OUTPUT" + exit 0 + fi + + printf '%s\n' "$paths" | "$classifier" "$GITHUB_OUTPUT" + + # Runtime Checks is the faster canonical producer for the identical + # release binary. Only wait for it when its trusted classifier says + # that workflow will build; TypeScript-suite-only changes keep their + # immediate local build. + runtime_release=false + runtime_classifier=.trusted-e2e-filter/.github/scripts/classify-runtime-changes.sh + runtime_output=$(mktemp) + trap 'rm -f "$runtime_output"' EXIT + if [[ -x "$runtime_classifier" ]] && + printf '%s\n' "$paths" | "$runtime_classifier" "$runtime_output" >/dev/null; then + grep -qx 'runtime=true' "$runtime_output" && runtime_release=true else - echo "no e2e-relevant paths changed" - echo "e2e=false" >> "$GITHUB_OUTPUT" + # Runtime's own missing-classifier fallback enables all checks, so + # its release producer will exist in this bootstrap case as well. + runtime_release=true + fi + echo "runtime_release=$runtime_release" >> "$GITHUB_OUTPUT" + + - name: Build matrix from proposed shard manifest + id: plan + env: + EVM: ${{ steps.filter.outputs.evm }} + STAKING: ${{ steps.filter.outputs.staking }} + COLDKEY_SWAP: ${{ steps.filter.outputs.coldkey_swap }} + DEV: ${{ steps.filter.outputs.dev }} + SUBNETS: ${{ steps.filter.outputs.subnets }} + SHIELD: ${{ steps.filter.outputs.shield }} + run: | + set -euo pipefail + builder=.trusted-e2e-filter/ts-tests/scripts/e2e-shard-plan.mjs + if [[ ! -x "$builder" ]]; then + # One-time bootstrap only. Future PRs always execute the trusted + # base builder over proposed data, matching the Rust SDK pattern. + builder=.proposed-e2e-plan/ts-tests/scripts/e2e-shard-plan.mjs fi + node "$builder" plan \ + .proposed-e2e-plan/ts-tests/e2e-shards.json \ + "$GITHUB_OUTPUT" \ + "$EVM" "$STAKING" "$COLDKEY_SWAP" "$DEV" "$SUBNETS" "$SHIELD" typescript-formatting: runs-on: ubuntu-latest @@ -70,7 +186,12 @@ jobs: node-version-file: ts-tests/.nvmrc - name: Validate E2E configuration - run: node ts-tests/scripts/validate-e2e-config.mjs + run: | + node ts-tests/scripts/test-e2e-shard-plan.mjs + node ts-tests/scripts/validate-e2e-config.mjs + + - name: Test E2E suite routing + run: .github/scripts/test-classify-typescript-e2e-changes.sh - name: Install e2e dependencies working-directory: ts-tests @@ -81,71 +202,113 @@ jobs: cd ts-tests pnpm run fmt - # Build the node binary in both variants and share as artifacts. + # Build each variant required by the selected suites exactly once and share + # it as an artifact. Ambiguous/shared changes select every suite and therefore + # still build both production release and fast-runtime binaries. build: runs-on: [self-hosted, fireactions-turbo-8] environment: name: sccache-writer deployment: false needs: [trusted-pr, changes] - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.build_count != '0' timeout-minutes: 60 strategy: - matrix: - include: - - variant: release - flags: "" - - variant: fast - flags: "--features fast-runtime" + matrix: ${{ fromJSON(needs.changes.outputs.build_matrix) }} env: RUST_BACKTRACE: full + permissions: + contents: read + actions: read steps: - name: Check-out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Select exact-merge Runtime release artifact + id: shared + if: >- + github.event_name == 'pull_request' && + matrix.variant == 'release' && + needs.changes.outputs.runtime_release == 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: .github/scripts/select-shared-release-artifact.sh "$GITHUB_OUTPUT" + + - name: Reuse verified Runtime release build + id: reuse + if: steps.shared.outputs.found == 'true' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + run: | + set -euo pipefail + output=$(mktemp) + trap 'rm -f "$output"' EXIT + if .github/scripts/download-artifact.sh \ + "${{ steps.shared.outputs.artifact_id }}" \ + "node-subtensor-release-${GITHUB_SHA}" \ + "${{ steps.shared.outputs.digest }}" \ + "${{ steps.shared.outputs.size }}" \ + target/release "$output" && + [[ -s target/release/node-subtensor ]] && + [[ -s target/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm ]]; then + cat "$output" >> "$GITHUB_OUTPUT" + echo "reused=true" >> "$GITHUB_OUTPUT" + chmod +x target/release/node-subtensor + { + echo "### E2E release node" + echo "- Reused exact-merge Runtime Checks artifact" + echo "- Producer wait: ${{ steps.shared.outputs.waited_seconds }}s" + } >> "$GITHUB_STEP_SUMMARY" + else + rm -rf target/release + echo "::warning::Shared release artifact could not be verified; rebuilding locally." + echo "reused=false" >> "$GITHUB_OUTPUT" + fi + - name: Set up Rust build environment + if: steps.reuse.outputs.reused != 'true' uses: ./.github/actions/rust-setup with: cache-key: e2e-${{ matrix.variant }} - fast-linker: "false" sccache-credential-mode: auto sccache-writer-access-key-id: ${{ secrets.SCCACHE_R2_WRITE_ACCESS_KEY_ID }} sccache-writer-secret-access-key: ${{ secrets.SCCACHE_R2_WRITE_SECRET_ACCESS_KEY }} - name: Build node-subtensor (${{ matrix.variant }}) + if: steps.reuse.outputs.reused != 'true' run: cargo build --profile release ${{ matrix.flags }} -p node-subtensor + - name: Report ${{ matrix.variant }} compiler cache + if: always() && steps.reuse.outputs.reused != 'true' + run: .github/scripts/sccache-report.sh "E2E ${{ matrix.variant }} compiler cache" + - name: Upload binary uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: node-subtensor-${{ matrix.variant }} path: target/release/node-subtensor if-no-files-found: error + retention-days: 3 # State-focused suites do not exercise multi-validator consensus. They use a - # single immediately-finalized node; Shield retains the production-like - # multi-node topology below. Two workers here plus three Shield workers peak - # at five self-hosted runners while the E2E phase is active. + # single immediately-finalized node; EVM and staking are split by measured + # file runtime without changing coverage. Shield retains the production-like + # multi-node topology below. Seven state workers plus five Shield workers use + # 12 of the 32 turbo-8 runners, so two full PRs still leave eight slots free. run-e2e-tests: - needs: [trusted-pr, build] + needs: [trusted-pr, changes, build] + if: needs.changes.outputs.state_count != '0' runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 30 strategy: fail-fast: false - max-parallel: 2 - matrix: - include: - - test: zombienet_evm - binary: fast - - test: zombienet_staking - binary: fast - - test: zombienet_coldkey_swap - binary: fast - - test: dev - binary: release - - test: zombienet_subnets - binary: fast + max-parallel: 7 + matrix: ${{ fromJSON(needs.changes.outputs.state_matrix) }} name: "typescript-e2e-${{ matrix.test }}" @@ -158,18 +321,63 @@ jobs: with: binary: ${{ matrix.binary }} test: ${{ matrix.test }} + additional-test: ${{ matrix.additional_test }} + + # Keep the historical EVM check context stable while the implementation is + # split into balanced _a/_b/_c jobs. This also reports success when path routing + # intentionally skips state E2E, avoiding a permanently Expected context. + evm-result: + name: typescript-e2e-zombienet_evm + if: always() + needs: [trusted-pr, changes, build, run-e2e-tests, sharding-audit] + runs-on: ubuntu-latest + steps: + - name: Check EVM shard prerequisites + env: + TRUSTED_RESULT: ${{ needs.trusted-pr.result }} + CHANGES_RESULT: ${{ needs.changes.result }} + EVM_SELECTED: ${{ needs.changes.outputs.evm }} + BUILD_RESULT: ${{ needs.build.result }} + STATE_RESULT: ${{ needs.run-e2e-tests.result }} + TOPOLOGY_AUDIT: ${{ needs.changes.outputs.topology_audit }} + AUDIT_RESULT: ${{ needs.sharding-audit.result }} + run: | + if [[ "$TRUSTED_RESULT" != "success" || "$CHANGES_RESULT" != "success" ]]; then + echo "EVM E2E routing failed: trusted=$TRUSTED_RESULT changes=$CHANGES_RESULT" >&2 + exit 1 + fi + case "$EVM_SELECTED" in + false) + echo "EVM E2E was intentionally skipped by trusted path routing." + exit 0 + ;; + true) ;; + *) + echo "EVM E2E routing produced an invalid selection: '$EVM_SELECTED'" >&2 + exit 1 + ;; + esac + if [[ "$BUILD_RESULT" != "success" || "$STATE_RESULT" != "success" ]]; then + echo "EVM E2E prerequisites failed: build=$BUILD_RESULT state-shards=$STATE_RESULT" >&2 + exit 1 + fi + if [[ "$TOPOLOGY_AUDIT" == true && "$AUDIT_RESULT" != success ]]; then + echo "Canonical unsharded audit failed: $AUDIT_RESULT" >&2 + exit 1 + fi # Shield validates consensus, key rotation, timing, and transaction - # mortality, so preserve its six-node release topology and parallelize only - # by running measured-time-balanced file groups on independent networks. + # mortality, so every shard retains the six-node topology and production + # slot duration. A 250 ms runtime outruns the asynchronous key-rotation + # pipeline and can finalize a wrapper without unshielding its inner call. run-shield-tests: - needs: [trusted-pr, build] + needs: [trusted-pr, changes, build] + if: needs.changes.outputs.shield_count != '0' runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 30 strategy: fail-fast: false - matrix: - test: [zombienet_shield_a, zombienet_shield_b, zombienet_shield_c] + matrix: ${{ fromJSON(needs.changes.outputs.shield_matrix) }} name: "typescript-e2e-${{ matrix.test }}" steps: - name: Check-out repository @@ -178,22 +386,74 @@ jobs: - name: Run Shield shard uses: ./.github/actions/run-typescript-e2e with: - binary: release + binary: ${{ matrix.binary }} + test: ${{ matrix.test }} + + # Only topology/matrix changes pay for this audit. It proves that the fast + # file shards remain equivalent to the original canonical environments and + # catches hidden cross-file state or concurrency assumptions before merge. + sharding-audit: + name: Audit canonical unsharded ${{ matrix.test }} + needs: [trusted-pr, changes, build] + if: needs.changes.outputs.topology_audit == 'true' + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - test: zombienet_evm + binary: fast + - test: zombienet_staking + binary: fast + - test: zombienet_shield + binary: release + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Run canonical environment + uses: ./.github/actions/run-typescript-e2e + with: + binary: ${{ matrix.binary }} test: ${{ matrix.test }} shield-result: name: typescript-e2e-zombienet_shield if: always() - needs: [build, run-shield-tests] + needs: [trusted-pr, changes, build, run-shield-tests, sharding-audit] runs-on: ubuntu-latest steps: - name: Check Shield shard results env: + TRUSTED_RESULT: ${{ needs.trusted-pr.result }} + CHANGES_RESULT: ${{ needs.changes.result }} + SHIELD_SELECTED: ${{ needs.changes.outputs.shield }} BUILD_RESULT: ${{ needs.build.result }} SHIELD_RESULT: ${{ needs.run-shield-tests.result }} + TOPOLOGY_AUDIT: ${{ needs.changes.outputs.topology_audit }} + AUDIT_RESULT: ${{ needs.sharding-audit.result }} run: | - if [[ "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" || \ - "$SHIELD_RESULT" == "failure" || "$SHIELD_RESULT" == "cancelled" ]]; then + if [[ "$TRUSTED_RESULT" != "success" || "$CHANGES_RESULT" != "success" ]]; then + echo "Shield E2E routing failed: trusted=$TRUSTED_RESULT changes=$CHANGES_RESULT" >&2 + exit 1 + fi + case "$SHIELD_SELECTED" in + false) + echo "Shield E2E was intentionally skipped by trusted path routing." + exit 0 + ;; + true) ;; + *) + echo "Shield E2E routing produced an invalid selection: '$SHIELD_SELECTED'" >&2 + exit 1 + ;; + esac + if [[ "$BUILD_RESULT" != "success" || "$SHIELD_RESULT" != "success" ]]; then echo "Shield E2E prerequisites failed: build=$BUILD_RESULT shards=$SHIELD_RESULT" >&2 exit 1 fi + if [[ "$TOPOLOGY_AUDIT" == true && "$AUDIT_RESULT" != success ]]; then + echo "Canonical unsharded audit failed: $AUDIT_RESULT" >&2 + exit 1 + fi diff --git a/.github/workflows/validate-sccache.yml b/.github/workflows/validate-sccache.yml index 06e9a535b2..72c10e1250 100644 --- a/.github/workflows/validate-sccache.yml +++ b/.github/workflows/validate-sccache.yml @@ -6,8 +6,38 @@ on: - ".github/actions/rust-setup/**" - ".github/actions/sccache-setup/**" - ".github/scripts/sccache-configure.sh" + - ".github/scripts/sccache-config.py" - ".github/scripts/test-sccache-configure.sh" + - ".github/scripts/rust-setup-preflight.sh" + - ".github/scripts/install-rust-toolchain.sh" + - ".github/scripts/test-rust-setup-preflight.sh" + - ".github/scripts/sccache-report.sh" + - ".github/scripts/test-sccache-report.sh" + - ".github/scripts/classify-bittensor-e2e-changes.sh" + - ".github/rust-ci-paths.txt" + - ".github/scripts/classify-rust-changes.sh" + - ".github/scripts/validate-rust-ci-paths.sh" + - ".github/scripts/test-rust-ci-paths.sh" + - ".github/scripts/build-bittensor-e2e-matrix.py" + - ".github/scripts/test-classify-bittensor-e2e-changes.sh" + - ".github/scripts/download-artifact.sh" + - ".github/scripts/test-download-artifact.sh" + - ".github/scripts/select-shared-release-artifact.sh" + - ".github/scripts/test-select-shared-release-artifact.sh" + - ".github/scripts/prewarm-exact-runtime.sh" + - ".github/scripts/test-prewarm-exact-runtime.sh" + - ".github/scripts/benchmark-sccache-paired.sh" + - ".github/scripts/benchmark-artifact-cache.sh" + - ".github/scripts/r2-artifact-mirror.py" + - ".github/scripts/publish-artifact-mirror.sh" + - ".github/scripts/publish-current-run-artifact-mirror.sh" + - ".github/scripts/test-r2-artifact-mirror.py" + - ".github/scripts/r2-sccache-warmset.py" + - ".github/scripts/test-r2-sccache-warmset.py" + - ".github/scripts/snapshot-artifact.sh" + - ".github/scripts/test-snapshot-artifact.sh" - ".github/workflows/check-bittensor-e2e-tests.yml" + - ".github/workflows/docker-localnet.yml" - ".github/workflows/check-node-compat.yml" - ".github/workflows/mainnet-clone-preview.yml" - ".github/workflows/runtime-checks.yml" @@ -15,6 +45,19 @@ on: - ".github/workflows/typescript-e2e.yml" - ".github/workflows/validate-sccache.yml" workflow_dispatch: + inputs: + benchmark_host_cache: + description: "Run paired cache benchmarks on the turbo-8 production pool" + type: boolean + default: false + benchmark_artifact_cache: + description: "Include the large GitHub artifact comparison" + type: boolean + default: true + benchmark_fleet_cache: + description: "Sample 20 fresh production-pool placements and report p50/p95" + type: boolean + default: false permissions: contents: read @@ -27,4 +70,217 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Test fail-open and writer event gates - run: .github/scripts/test-sccache-configure.sh + run: | + .github/scripts/test-rust-setup-preflight.sh + .github/scripts/test-sccache-configure.sh + .github/scripts/test-sccache-report.sh + + - name: Test immutable artifact selection and download boundary + run: | + .github/scripts/test-snapshot-artifact.sh + .github/scripts/test-download-artifact.sh + .github/scripts/test-select-shared-release-artifact.sh + .github/scripts/test-r2-artifact-mirror.py + .github/scripts/test-r2-sccache-warmset.py + .github/scripts/test-classify-bittensor-e2e-changes.sh + .github/scripts/test-rust-ci-paths.sh + + - name: Check benchmark and prewarm script syntax + run: | + .github/scripts/test-prewarm-exact-runtime.sh + bash -n .github/scripts/benchmark-sccache-paired.sh + bash -n .github/scripts/benchmark-artifact-cache.sh + + benchmark-sccache-paired: + if: github.event_name == 'workflow_dispatch' && inputs.benchmark_host_cache + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 120 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Configure direct R2 sccache baseline + uses: ./.github/actions/rust-setup + with: + cache-key: host-cache-benchmark-paired + sccache-local-tier: disabled + + - name: Measure paired origin and warm-local reads in one VM + env: + SKIP_WASM_BUILD: 1 + run: .github/scripts/benchmark-sccache-paired.sh + + # One clean runtime check per fresh runner placement. The paired benchmark + # above controls noise on one VM; this matrix answers the operationally more + # important question: what latency does a randomly scheduled CI job see? + benchmark-sccache-fleet: + name: fleet cache sample ${{ matrix.sample }}/20 + if: github.event_name == 'workflow_dispatch' && inputs.benchmark_fleet_cache + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 30 + strategy: + fail-fast: false + # Avoid turning the benchmark itself into an artificial load test. + max-parallel: 5 + matrix: + sample: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] + env: + SKIP_WASM_BUILD: 1 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Configure production reader cache + uses: ./.github/actions/rust-setup + with: + cache-key: host-cache-benchmark-fleet + sccache-credential-mode: reader + + - name: Measure a clean runtime check + id: measure + run: | + set -euo pipefail + cargo clean + if [[ "${SCCACHE_ENABLED:-false}" == true ]]; then + sccache --zero-stats >/dev/null + fi + started=$(date -u +%s) + cargo check --locked -p node-subtensor-runtime + echo "seconds=$(($(date -u +%s) - started))" >> "$GITHUB_OUTPUT" + + - name: Report fleet-sample compiler cache + if: always() + run: .github/scripts/sccache-report.sh "Fleet sample ${{ matrix.sample }} compiler cache" "$RUNNER_TEMP/sccache-fleet-${{ matrix.sample }}" + + - name: Record fleet sample + env: + SAMPLE: ${{ matrix.sample }} + SECONDS: ${{ steps.measure.outputs.seconds }} + PLACEMENT: ${{ runner.name }} + run: | + set -euo pipefail + stats="$RUNNER_TEMP/sccache-fleet-${SAMPLE}.json" + hits=$(jq '([.stats.cache_hits.counts[]?] | add) // 0' "$stats") + misses=$(jq '([.stats.cache_misses.counts[]?] | add) // 0' "$stats") + enabled=false + local_tier=false + [[ "${SCCACHE_ENABLED:-false}" != true ]] || enabled=true + [[ "${SCCACHE_LOCAL_TIER:-false}" != true ]] || local_tier=true + jq -n \ + --argjson sample "$SAMPLE" \ + --argjson seconds "$SECONDS" \ + --arg placement "$PLACEMENT" \ + --arg backend "${SCCACHE_BACKEND:-disabled}" \ + --argjson enabled "$enabled" \ + --argjson local_tier "$local_tier" \ + --argjson hits "$hits" \ + --argjson misses "$misses" \ + '{sample: $sample, seconds: $seconds, placement: $placement, + backend: $backend, enabled: $enabled, local_tier: $local_tier, + hits: $hits, misses: $misses}' \ + > "$RUNNER_TEMP/fleet-sample-${SAMPLE}.json" + + - name: Upload fleet sample + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: fleet-cache-sample-${{ matrix.sample }} + path: | + ${{ runner.temp }}/fleet-sample-${{ matrix.sample }}.json + ${{ runner.temp }}/sccache-fleet-${{ matrix.sample }}.* + if-no-files-found: error + retention-days: 7 + + benchmark-sccache-fleet-summary: + name: fleet cache p50/p95 summary + if: always() && github.event_name == 'workflow_dispatch' && inputs.benchmark_fleet_cache + needs: benchmark-sccache-fleet + runs-on: ubuntu-latest + steps: + - name: Download fleet samples + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: fleet-cache-sample-* + path: ${{ runner.temp }}/fleet-cache-samples + merge-multiple: true + + - name: Summarize fleet distribution + env: + SAMPLE_DIR: ${{ runner.temp }}/fleet-cache-samples + run: | + python3 - <<'PY' + import collections + import json + import math + import os + from pathlib import Path + + expected = 20 + root = Path(os.environ["SAMPLE_DIR"]) + samples = [json.loads(path.read_text()) for path in root.glob("fleet-sample-*.json")] + samples.sort(key=lambda sample: sample["sample"]) + seconds = sorted(sample["seconds"] for sample in samples) + + lines = ["### Production-pool sccache distribution", ""] + lines.append(f"- Complete samples: {len(samples)}/{expected}") + lines.append("- Sampling unit: fresh scheduler placements (physical-host uniqueness is not exposed)") + if samples: + percentile = lambda p: seconds[math.ceil(p * len(seconds)) - 1] + lines.extend([ + f"- Compile wall time: p50 {percentile(0.50)}s, p95 {percentile(0.95)}s", + f"- Range: {seconds[0]}s–{seconds[-1]}s; mean {sum(seconds) / len(seconds):.1f}s", + f"- sccache enabled: {sum(sample['enabled'] for sample in samples)}/{len(samples)}", + f"- Host-local route active: {sum(sample['local_tier'] for sample in samples)}/{len(samples)}", + "- Backend routes: " + ", ".join( + f"{name}={count}" for name, count in + sorted(collections.Counter(sample["backend"] for sample in samples).items()) + ), + "", + "| Sample | Seconds | Route | Local tier | Hits | Misses | Placement |", + "|---:|---:|---|:---:|---:|---:|---|", + ]) + for sample in samples: + lines.append( + f"| {sample['sample']} | {sample['seconds']} | {sample['backend']} | " + f"{str(sample['local_tier']).lower()} | {sample['hits']} | " + f"{sample['misses']} | {sample['placement']} |" + ) + + report = "\n".join(lines) + "\n" + print(report) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(report) + if len(samples) != expected: + raise SystemExit(f"expected {expected} complete samples, found {len(samples)}") + PY + + benchmark-artifact-cache: + if: >- + github.event_name == 'workflow_dispatch' && + inputs.benchmark_host_cache && + inputs.benchmark_artifact_cache + needs: benchmark-sccache-paired + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 45 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Select latest trusted mainnet clone snapshot + id: snapshot + env: + GH_TOKEN: ${{ github.token }} + run: | + .github/scripts/snapshot-artifact.sh select \ + mainnet-snapshot main "${{ github.repository_id }}" \ + .github/workflows/refresh-mainnet-snapshot.yml 168 \ + "$GITHUB_OUTPUT" required + + - name: Measure direct and cache-only artifact downloads + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + ARTIFACT_ID: ${{ steps.snapshot.outputs.artifact-id }} + ARTIFACT_DIGEST: ${{ steps.snapshot.outputs.artifact-digest }} + ARTIFACT_SIZE: ${{ steps.snapshot.outputs.artifact-size-bytes }} + run: .github/scripts/benchmark-artifact-cache.sh diff --git a/clones/js-tests/scripts/run-clone-regressions.ts b/clones/js-tests/scripts/run-clone-regressions.ts index f0205022e3..f30e7edad9 100644 --- a/clones/js-tests/scripts/run-clone-regressions.ts +++ b/clones/js-tests/scripts/run-clone-regressions.ts @@ -9,6 +9,14 @@ import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const testsDir = path.join(__dirname, "..", "tests"); +// IMPORTANT CI POLICY — MANUAL DIAGNOSTIC: +// test-alpha-deprecated-stake-histogram.ts intentionally remains available as +// `npm run test:alpha-deprecated-stake-histogram`, but is not a required clone +// regression. It scans the complete legacy Alpha map and reports observational +// distribution data without comparing it to a pass/fail baseline. Run it +// manually against an upgraded clone when that data is needed; do not silently +// add its full-state scan back to per-PR CI without defining a useful invariant. + // Per-test ceiling; override with CLONE_REGRESSION_TIMEOUT_MS. Healthy tests // finish in 2-10 min, so 15 min is a hang, not a slow pass — failing fast // beats burning half an hour of runner time per stuck test. @@ -29,7 +37,6 @@ const CLONE_REGRESSIONS = [ { name: "test-proxy-filter-security-regressions.ts", phase: "remaining" }, { name: "test-hotkey-swap-and-proxy-stake.ts", phase: "remaining" }, { name: "test-net-tao-flow-emission-allocation.ts", phase: "remaining" }, - { name: "test-alpha-deprecated-stake-histogram.ts", phase: "remaining" }, ] as const satisfies ReadonlyArray<{ name: string; phase: CloneRegressionPhase }>; const regressionNames = new Set(CLONE_REGRESSIONS.map(({ name }) => name)); diff --git a/clones/scripts/run-clone-regression-phase.sh b/clones/scripts/run-clone-regression-phase.sh new file mode 100755 index 0000000000..82e963a054 --- /dev/null +++ b/clones/scripts/run-clone-regression-phase.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd)" +JS_TESTS="$REPO_ROOT/clones/js-tests" + +usage() { + echo "usage: run-clone-regression-phase.sh pristine|remaining|combined" >&2 + exit 2 +} + +[[ $# -eq 1 ]] || usage +phase=$1 +[[ "$phase" == pristine || "$phase" == remaining || "$phase" == combined ]] || usage + +run_sdk_drift=${RUN_SDK_DRIFT:-false} +[[ "$run_sdk_drift" == true || "$run_sdk_drift" == false ]] || { + echo "RUN_SDK_DRIFT must be true or false" >&2 + exit 2 +} + +cleanup() { + local status=$? + "$SCRIPT_DIR/stop-local-clone.sh" || true + exit "$status" +} +trap cleanup EXIT + +start_clone() { + "$SCRIPT_DIR/start-local-clone-and-wait.sh" accelerated +} + +upgrade_runtime() { + ( + cd "$JS_TESTS" + npm run runtime:update:alice || { sleep 15; npm run runtime:update:alice; } + ) +} + +run_regressions() { + local selected_phase=$1 + ( + cd "$JS_TESTS" + CLONE_REGRESSION_PHASE="$selected_phase" \ + CLONE_REGRESSION_TIMEOUT_MS=1800000 \ + npm run test:clone-regressions + ) +} + +run_pristine() { + start_clone + upgrade_runtime + run_regressions pristine +} + +run_sdk_metadata_drift() { + [[ "$run_sdk_drift" == true ]] || return 0 + if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/0.11.28/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" + fi + ( + cd "$REPO_ROOT/sdk/python" + uv sync --locked --all-extras --dev + uv run python -m codegen.check --drift "${WS_ENDPOINT:-ws://127.0.0.1:9944}" + ) +} + +run_remaining() { + start_clone + upgrade_runtime + ( + cd "$JS_TESTS" + npm test + ) + run_regressions remaining + run_sdk_metadata_drift +} + +case "$phase" in + pristine) + run_pristine + ;; + remaining) + run_remaining + ;; + combined) + : "${CLONE_CHECKPOINT:?CLONE_CHECKPOINT is required for combined execution}" + run_pristine + "$SCRIPT_DIR/stop-local-clone.sh" + "$SCRIPT_DIR/local-clone-checkpoint.sh" restore "$CLONE_CHECKPOINT" + run_remaining + ;; +esac diff --git a/ts-tests/e2e-shards.json b/ts-tests/e2e-shards.json new file mode 100644 index 0000000000..6d33e13c8d --- /dev/null +++ b/ts-tests/e2e-shards.json @@ -0,0 +1,90 @@ +{ + "version": 1, + "suites": { + "evm": { + "shards": [ + { + "name": "zombienet_evm_a", + "files": ["suites/zombienet_evm/01-contract-deploy-call.test.ts"] + }, + { + "name": "zombienet_evm_b", + "files": [ + "suites/zombienet_evm/03-wasm-contract.test.ts", + "suites/zombienet_evm/00-evm-substrate-transfer.test.ts", + "suites/zombienet_evm/04-edge-cases.test.ts" + ] + }, + { + "name": "zombienet_evm_c", + "files": [ + "suites/zombienet_evm/05-direct-call-precompile.test.ts", + "suites/zombienet_evm/02-precompile-gas.test.ts" + ] + } + ] + }, + "staking": { + "shards": [ + { + "name": "zombienet_staking_a", + "files": [ + "suites/zombienet_staking/01-add-stake-limit.test.ts", + "suites/zombienet_staking/02.00-claim-root.test.ts", + "suites/zombienet_staking/02.01-claim-root.test.ts", + "suites/zombienet_staking/02.04-claim-root-hotkey-swap.test.ts", + "suites/zombienet_staking/04-remove-stake.test.ts", + "suites/zombienet_staking/06-remove-stake-limit.test.ts", + "suites/zombienet_staking/08-swap-stake-limit.test.ts" + ] + }, + { + "name": "zombienet_staking_b", + "files": [ + "suites/zombienet_staking/00-add-stake.test.ts", + "suites/zombienet_staking/02.02-claim-root.test.ts", + "suites/zombienet_staking/02.03-claim-root.test.ts", + "suites/zombienet_staking/03-move-stake.test.ts", + "suites/zombienet_staking/05-remove-stake-full-limit.test.ts", + "suites/zombienet_staking/07-swap-stake.test.ts", + "suites/zombienet_staking/09-transfer-stake.test.ts", + "suites/zombienet_staking/10-unstake-all.test.ts", + "suites/zombienet_staking/11-unstake-all-alpha.test.ts" + ] + } + ] + }, + "shield": { + "shards": [ + { + "name": "zombienet_shield_a", + "files": ["suites/zombienet_shield/00.01-basic.test.ts"], + "maxConcurrency": 6 + }, + { + "name": "zombienet_shield_b", + "files": ["suites/zombienet_shield/03-timing.test.ts"], + "maxConcurrency": 4 + }, + { + "name": "zombienet_shield_c", + "files": [ + "suites/zombienet_shield/00.00-basic.test.ts", + "suites/zombienet_shield/01-scaling.test.ts" + ], + "maxConcurrency": 6 + }, + { + "name": "zombienet_shield_d", + "files": ["suites/zombienet_shield/02-edge-cases.test.ts"], + "maxConcurrency": 2 + }, + { + "name": "zombienet_shield_e", + "files": ["suites/zombienet_shield/04-mortality.test.ts"], + "maxConcurrency": 1 + } + ] + } + } +} diff --git a/ts-tests/e2e-suite-ownership.json b/ts-tests/e2e-suite-ownership.json new file mode 100644 index 0000000000..1d30f2f033 --- /dev/null +++ b/ts-tests/e2e-suite-ownership.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "suites": { + "dev": { + "owner": "pull_request", + "selector": "dev", + "environments": ["dev"] + }, + "smoke": { + "owner": "scheduled", + "selector": null, + "environments": ["smoke_devnet", "smoke_mainnet", "smoke_testnet"] + }, + "zombienet_coldkey_swap": { + "owner": "pull_request", + "selector": "coldkey_swap", + "environments": ["zombienet_coldkey_swap"] + }, + "zombienet_evm": { + "owner": "pull_request", + "selector": "evm", + "environments": ["zombienet_evm"] + }, + "zombienet_shield": { + "owner": "pull_request", + "selector": "shield", + "environments": ["zombienet_shield"] + }, + "zombienet_staking": { + "owner": "pull_request", + "selector": "staking", + "environments": ["zombienet_staking"] + }, + "zombienet_subnets": { + "owner": "pull_request", + "selector": "subnets", + "environments": ["zombienet_subnets"] + } + } +} diff --git a/ts-tests/moonwall.config.json b/ts-tests/moonwall.config.json index c8fcaaca64..e3c23933e6 100644 --- a/ts-tests/moonwall.config.json +++ b/ts-tests/moonwall.config.json @@ -7,7 +7,9 @@ { "name": "dev", "timeout": 120000, - "envVars": ["DEBUG_COLORS=1"], + "envVars": [ + "DEBUG_COLORS=1" + ], "testFileDir": [ "suites/dev" ], @@ -15,7 +17,9 @@ "generate-types.sh" ], "multiThreads": true, - "reporters": ["basic"], + "reporters": [ + "basic" + ], "foundation": { "type": "dev", "launchSpec": [ @@ -44,10 +48,12 @@ { "name": "zombienet_staking", "timeout": 600000, - "testFileDir": ["suites/zombienet_staking"], + "testFileDir": [ + "suites/zombienet_staking" + ], "runScripts": [ - "generate-types.sh", - "build-spec.sh" + "build-spec.sh", + "generate-types-from-chain-spec.sh" ], "foundation": { "type": "zombie", @@ -63,89 +69,24 @@ { "name": "Node", "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"] - } - ] - }, { - "name": "zombienet_shield", - "timeout": 600000, - "testFileDir": ["suites/zombienet_shield"], - "runScripts": [ - "generate-types.sh", - "build-spec.sh" - ], - "foundation": { - "type": "zombie", - "zombieSpec": { - "configPath": "./configs/zombie_extended.json", - "skipBlockCheck": [] - } - }, - "vitestArgs": { - "bail": 1 - }, - "connections": [ - { - "name": "Node", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"], - "descriptor": "subtensor" - }, - { - "name": "NodeFull", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9950"], - "descriptor": "subtensor" + "endpoints": [ + "ws://127.0.0.1:9947" + ] } ] }, { - "name": "zombienet_shield_a", + "name": "zombienet_shield", "timeout": 600000, - "testFileDir": ["suites/zombienet_shield"], - "include": [ - "suites/zombienet_shield/00.01-basic.test.ts" + "envVars": [ + "SHIELD_RUNTIME=release" ], - "runScripts": [ - "generate-types.sh", - "build-spec.sh" - ], - "foundation": { - "type": "zombie", - "zombieSpec": { - "configPath": "./configs/zombie_extended.json", - "skipBlockCheck": [] - } - }, - "vitestArgs": { - "bail": 1 - }, - "connections": [ - { - "name": "Node", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"], - "descriptor": "subtensor" - }, - { - "name": "NodeFull", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9950"], - "descriptor": "subtensor" - } - ] - }, - { - "name": "zombienet_shield_b", - "timeout": 600000, - "testFileDir": ["suites/zombienet_shield"], - "include": [ - "suites/zombienet_shield/03-timing.test.ts", - "suites/zombienet_shield/04-mortality.test.ts" + "testFileDir": [ + "suites/zombienet_shield" ], "runScripts": [ - "generate-types.sh", - "build-spec.sh" + "build-spec.sh", + "generate-types-from-chain-spec.sh" ], "foundation": { "type": "zombie", @@ -161,34 +102,35 @@ { "name": "Node", "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"], + "endpoints": [ + "ws://127.0.0.1:9947" + ], "descriptor": "subtensor" }, { "name": "NodeFull", "type": "papi", - "endpoints": ["ws://127.0.0.1:9950"], + "endpoints": [ + "ws://127.0.0.1:9950" + ], "descriptor": "subtensor" } ] }, { - "name": "zombienet_shield_c", + "name": "zombienet_coldkey_swap", "timeout": 600000, - "testFileDir": ["suites/zombienet_shield"], - "include": [ - "suites/zombienet_shield/00.00-basic.test.ts", - "suites/zombienet_shield/01-scaling.test.ts", - "suites/zombienet_shield/02-edge-cases.test.ts" + "testFileDir": [ + "suites/zombienet_coldkey_swap" ], "runScripts": [ - "generate-types.sh", - "build-spec.sh" + "build-spec.sh", + "generate-types-from-chain-spec.sh" ], "foundation": { "type": "zombie", "zombieSpec": { - "configPath": "./configs/zombie_extended.json", + "configPath": "./configs/zombie_single_node.json", "skipBlockCheck": [] } }, @@ -199,50 +141,22 @@ { "name": "Node", "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"], - "descriptor": "subtensor" - }, - { - "name": "NodeFull", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9950"], - "descriptor": "subtensor" + "endpoints": [ + "ws://127.0.0.1:9947" + ] } ] }, { - "name": "zombienet_coldkey_swap", - "timeout": 600000, - "testFileDir": ["suites/zombienet_coldkey_swap"], - "runScripts": [ - "generate-types.sh", - "build-spec.sh" - ], - "foundation": { - "type": "zombie", - "zombieSpec": { - "configPath": "./configs/zombie_single_node.json", - "skipBlockCheck": [] - } - }, - "vitestArgs": { - "bail": 1 - }, - "connections": [ - { - "name": "Node", - "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"] - } - ] - }, { "name": "zombienet_evm", "timeout": 600000, - "testFileDir": ["suites/zombienet_evm"], + "testFileDir": [ + "suites/zombienet_evm" + ], "runScripts": [ - "generate-types.sh", - "generate-ink-types.sh", - "build-spec.sh" + "build-spec.sh", + "generate-types-from-chain-spec.sh", + "generate-ink-types.sh" ], "foundation": { "type": "zombie", @@ -258,23 +172,30 @@ { "name": "Node", "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"], + "endpoints": [ + "ws://127.0.0.1:9947" + ], "descriptor": "subtensor" }, { "name": "EVM", "type": "ethers", - "endpoints": ["http://127.0.0.1:9947"], + "endpoints": [ + "http://127.0.0.1:9947" + ], "descriptor": "evm" } ] - }, { + }, + { "name": "zombienet_subnets", "timeout": 600000, - "testFileDir": ["suites/zombienet_subnets"], + "testFileDir": [ + "suites/zombienet_subnets" + ], "runScripts": [ - "generate-types.sh", - "build-spec.sh" + "build-spec.sh", + "generate-types-from-chain-spec.sh" ], "foundation": { "type": "zombie", @@ -290,49 +211,75 @@ { "name": "Node", "type": "papi", - "endpoints": ["ws://127.0.0.1:9947"] + "endpoints": [ + "ws://127.0.0.1:9947" + ] } ] - }, { + }, + { "name": "smoke_devnet", - "testFileDir": ["suites/smoke"], + "testFileDir": [ + "suites/smoke" + ], "foundation": { "type": "read_only" }, - "reporters": ["basic","html"], + "reporters": [ + "basic", + "html" + ], "connections": [ { "name": "node", "type": "polkadotJs", - "endpoints": ["wss://dev.chain.opentensor.ai:443"] + "endpoints": [ + "wss://dev.chain.opentensor.ai:443" + ] } ] - }, { + }, + { "name": "smoke_mainnet", - "testFileDir": ["suites/smoke"], + "testFileDir": [ + "suites/smoke" + ], "foundation": { "type": "read_only" }, - "reporters": ["basic","html"], + "reporters": [ + "basic", + "html" + ], "connections": [ { "name": "node", "type": "polkadotJs", - "endpoints": ["wss://openrpc.taostats.io"] + "endpoints": [ + "wss://openrpc.taostats.io" + ] } ] - }, { + }, + { "name": "smoke_testnet", - "testFileDir": ["suites/smoke"], + "testFileDir": [ + "suites/smoke" + ], "foundation": { "type": "read_only" }, - "reporters": ["basic","html"], + "reporters": [ + "basic", + "html" + ], "connections": [ { "name": "node", "type": "polkadotJs", - "endpoints": ["wss://test.finney.opentensor.ai:443"] + "endpoints": [ + "wss://test.finney.opentensor.ai:443" + ] } ] } diff --git a/ts-tests/scripts/e2e-shard-plan.mjs b/ts-tests/scripts/e2e-shard-plan.mjs new file mode 100755 index 0000000000..3da89d7aa0 --- /dev/null +++ b/ts-tests/scripts/e2e-shard-plan.mjs @@ -0,0 +1,268 @@ +#!/usr/bin/env node + +import { readFileSync, readdirSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SUITES = { + evm: { + baseEnvironment: "zombienet_evm", + directory: "suites/zombienet_evm", + binary: "fast", + lane: "state", + }, + staking: { + baseEnvironment: "zombienet_staking", + directory: "suites/zombienet_staking", + binary: "fast", + lane: "state", + }, + shield: { + baseEnvironment: "zombienet_shield", + directory: "suites/zombienet_shield", + binary: "release", + lane: "shield", + }, +}; + +const BOOLEAN_NAMES = ["evm", "staking", "coldkey_swap", "dev", "subnets", "shield"]; + +function exactKeys(value, expected, label) { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + throw new Error(`${label} keys must be exactly [${wanted.join(", ")}]`); + } +} + +function objectValue(value, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value; +} + +export function validateShardManifest(payload) { + const manifest = objectValue(payload, "shard manifest"); + exactKeys(manifest, ["version", "suites"], "shard manifest"); + if (manifest.version !== 1) throw new Error("shard manifest version must be 1"); + + const suites = objectValue(manifest.suites, "shard manifest suites"); + exactKeys(suites, Object.keys(SUITES), "shard manifest suites"); + + for (const [suiteName, contract] of Object.entries(SUITES)) { + const suite = objectValue(suites[suiteName], `${suiteName} suite`); + exactKeys(suite, ["shards"], `${suiteName} suite`); + if (!Array.isArray(suite.shards) || suite.shards.length === 0 || suite.shards.length > 26) { + throw new Error(`${suiteName} must contain between 1 and 26 shards`); + } + + const seenNames = new Set(); + const seenFiles = new Set(); + for (const [index, shardValue] of suite.shards.entries()) { + const shard = objectValue(shardValue, `${suiteName} shard ${index}`); + const keys = suiteName === "shield" ? ["name", "files", "maxConcurrency"] : ["name", "files"]; + exactKeys(shard, keys, `${suiteName} shard ${index}`); + + const expectedName = `${contract.baseEnvironment}_${String.fromCharCode(97 + index)}`; + if (shard.name !== expectedName || seenNames.has(shard.name)) { + throw new Error(`${suiteName} shard ${index} must be named ${expectedName}`); + } + seenNames.add(shard.name); + + if (!Array.isArray(shard.files) || shard.files.length === 0) { + throw new Error(`${shard.name} must include at least one test file`); + } + for (const file of shard.files) { + if ( + typeof file !== "string" || + !file.startsWith(`${contract.directory}/`) || + !/^[A-Za-z0-9._/-]+\.test\.ts$/.test(file) || + file.split("/").some((segment) => segment === "." || segment === ".." || segment === "") || + seenFiles.has(file) + ) { + throw new Error(`${shard.name} contains an unsafe or duplicate test file: ${String(file)}`); + } + seenFiles.add(file); + } + + if ( + suiteName === "shield" && + (!Number.isInteger(shard.maxConcurrency) || shard.maxConcurrency < 1 || shard.maxConcurrency > 8) + ) { + throw new Error(`${shard.name} maxConcurrency must be an integer from 1 through 8`); + } + } + } + return manifest; +} + +export function loadShardManifest(path) { + return validateShardManifest(JSON.parse(readFileSync(path, "utf8"))); +} + +function testFiles(directory, relativeDirectory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const relativePath = `${relativeDirectory}/${entry.name}`; + if (entry.isDirectory()) return testFiles(join(directory, entry.name), relativePath); + return entry.isFile() && entry.name.endsWith(".test.ts") ? [relativePath] : []; + }); +} + +export function validateManifestCoverage(manifest, tsTestsDir) { + for (const [suiteName, contract] of Object.entries(SUITES)) { + const expected = testFiles(join(tsTestsDir, contract.directory), contract.directory).sort(); + const assigned = manifest.suites[suiteName].shards.flatMap(({ files }) => files).sort(); + if (JSON.stringify(assigned) !== JSON.stringify(expected)) { + const missing = expected.filter((file) => !assigned.includes(file)); + const unknown = assigned.filter((file) => !expected.includes(file)); + throw new Error( + [ + `${suiteName} shard coverage does not match the test directory.`, + `Unassigned test files: ${missing.join(", ") || "none"}`, + `Stale or invalid manifest entries: ${unknown.join(", ") || "none"}`, + `Fix: edit ts-tests/e2e-shards.json so every suites/zombienet_${suiteName}/**/*.test.ts file appears exactly once under suites.${suiteName}.shards[].files.`, + "Verify: node ts-tests/scripts/test-e2e-shard-plan.mjs && node ts-tests/scripts/validate-e2e-config.mjs", + ].join("\n") + ); + } + } +} + +function generatedShardName(name) { + return Object.values(SUITES).some(({ baseEnvironment }) => new RegExp(`^${baseEnvironment}_[a-z]+$`).test(name)); +} + +export function stripGeneratedShardEnvironments(config) { + const copy = structuredClone(config); + if (!Array.isArray(copy.environments)) throw new Error("Moonwall config has no environments array"); + copy.environments = copy.environments.filter(({ name }) => !generatedShardName(name)); + return copy; +} + +export function materializeShardEnvironments(config, manifest) { + const baseConfig = stripGeneratedShardEnvironments(config); + const shardsByBase = new Map( + Object.entries(SUITES).map(([suiteName, contract]) => [contract.baseEnvironment, { suiteName, contract }]) + ); + const environments = []; + + for (const environment of baseConfig.environments) { + environments.push(environment); + const selected = shardsByBase.get(environment.name); + if (!selected) continue; + for (const shard of manifest.suites[selected.suiteName].shards) { + const generated = structuredClone(environment); + generated.name = shard.name; + generated.include = [...shard.files]; + if (selected.suiteName === "shield") { + generated.vitestArgs = { + ...generated.vitestArgs, + sequence: { concurrent: true }, + maxConcurrency: shard.maxConcurrency, + }; + } + environments.push(generated); + } + } + baseConfig.environments = environments; + return baseConfig; +} + +function booleanValue(value, name) { + if (value === "true") return true; + if (value === "false") return false; + throw new Error(`${name} selection must be true or false`); +} + +export function buildE2EPlan(manifest, selected) { + const stateEntries = []; + for (const suiteName of ["evm", "staking"]) { + if (!selected[suiteName]) continue; + const binary = SUITES[suiteName].binary; + stateEntries.push(...manifest.suites[suiteName].shards.map(({ name }) => ({ test: name, binary }))); + } + if (selected.coldkey_swap && selected.subnets) { + stateEntries.push({ + test: "zombienet_coldkey_swap", + additional_test: "zombienet_subnets", + binary: "fast", + }); + } else if (selected.coldkey_swap) { + stateEntries.push({ test: "zombienet_coldkey_swap", binary: "fast" }); + } else if (selected.subnets) { + stateEntries.push({ test: "zombienet_subnets", binary: "fast" }); + } + if (selected.dev) stateEntries.push({ test: "dev", binary: "release" }); + + const shieldEntries = selected.shield + ? manifest.suites.shield.shards.map(({ name }) => ({ test: name, binary: SUITES.shield.binary })) + : []; + const needsRelease = selected.dev || selected.shield; + const needsFast = stateEntries.some(({ binary }) => binary === "fast"); + const buildEntries = []; + if (needsRelease) buildEntries.push({ variant: "release", flags: "" }); + if (needsFast) buildEntries.push({ variant: "fast", flags: "--features fast-runtime" }); + + return { + state_count: stateEntries.length, + state_matrix: { include: stateEntries }, + shield_count: shieldEntries.length, + shield_matrix: { include: shieldEntries }, + build_count: buildEntries.length, + build_matrix: { include: buildEntries }, + }; +} + +function appendPlan(path, plan) { + const lines = []; + for (const [name, value] of Object.entries(plan)) { + lines.push(`${name}=${typeof value === "number" ? value : JSON.stringify(value)}`); + } + writeFileSync(path, `${lines.join("\n")}\n`, { flag: "a" }); +} + +function writeConfig(path, config) { + const temporary = `${path}.generated`; + writeFileSync(temporary, `${JSON.stringify(config, null, 4)}\n`, { mode: 0o600 }); + renameSync(temporary, path); +} + +function usage() { + throw new Error( + "usage: e2e-shard-plan.mjs plan MANIFEST OUTPUT EVM STAKING COLDKEY_SWAP DEV SUBNETS SHIELD | materialize CONFIG MANIFEST | strip CONFIG" + ); +} + +function main(argv) { + const [command, ...args] = argv; + if (command === "plan" && args.length === 8) { + const [manifestPath, outputPath, ...selections] = args; + const selected = Object.fromEntries( + BOOLEAN_NAMES.map((name, index) => [name, booleanValue(selections[index], name)]) + ); + appendPlan(outputPath, buildE2EPlan(loadShardManifest(manifestPath), selected)); + return; + } + if (command === "materialize" && args.length === 2) { + const [configPath, manifestPath] = args; + const config = JSON.parse(readFileSync(configPath, "utf8")); + writeConfig(configPath, materializeShardEnvironments(config, loadShardManifest(manifestPath))); + return; + } + if (command === "strip" && args.length === 1) { + const [configPath] = args; + writeConfig(configPath, stripGeneratedShardEnvironments(JSON.parse(readFileSync(configPath, "utf8")))); + return; + } + usage(); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(`E2E shard plan error: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/ts-tests/scripts/extract-runtime-wasm.mjs b/ts-tests/scripts/extract-runtime-wasm.mjs new file mode 100644 index 0000000000..c8a4894218 --- /dev/null +++ b/ts-tests/scripts/extract-runtime-wasm.mjs @@ -0,0 +1,34 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const RUNTIME_CODE_KEY = "0x3a636f6465"; + +export function decodeRuntimeWasm(chainSpec) { + const encodedRuntime = chainSpec?.genesis?.raw?.top?.[RUNTIME_CODE_KEY]; + if (typeof encodedRuntime !== "string" || !/^0x(?:[0-9a-fA-F]{2})+$/.test(encodedRuntime)) { + throw new Error(`Chain spec is missing a valid ${RUNTIME_CODE_KEY} runtime entry`); + } + + const runtime = Buffer.from(encodedRuntime.slice(2), "hex"); + if (runtime.length === 0) { + throw new Error("Chain spec runtime entry decoded to an empty file"); + } + return runtime; +} + +export function extractRuntimeWasm(chainSpecPath, outputPath) { + const chainSpec = JSON.parse(readFileSync(chainSpecPath, "utf8")); + const runtime = decodeRuntimeWasm(chainSpec); + writeFileSync(outputPath, runtime); + return runtime.length; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [, , chainSpecPath, outputPath] = process.argv; + if (!chainSpecPath || !outputPath) { + throw new Error("usage: extract-runtime-wasm.mjs CHAIN_SPEC OUTPUT_WASM"); + } + const byteLength = extractRuntimeWasm(chainSpecPath, outputPath); + console.log(`Extracted ${byteLength} runtime bytes from ${chainSpecPath}`); +} diff --git a/ts-tests/scripts/generate-types-from-chain-spec.sh b/ts-tests/scripts/generate-types-from-chain-spec.sh new file mode 100755 index 0000000000..93e5c74679 --- /dev/null +++ b/ts-tests/scripts/generate-types-from-chain-spec.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}/.." + +export PAPI_CHAIN_SPEC_PATH="${PAPI_CHAIN_SPEC_PATH:-./specs/chain-spec.json}" +exec ./scripts/generate-types.sh diff --git a/ts-tests/scripts/generate-types.sh b/ts-tests/scripts/generate-types.sh index 8c65e561e5..a0314b6b52 100755 --- a/ts-tests/scripts/generate-types.sh +++ b/ts-tests/scripts/generate-types.sh @@ -1,19 +1,33 @@ -#!/bin/bash +#!/usr/bin/env bash # -# (Re)generate polkadot-api type descriptors using a running node. -# Checks that the node binary exists before running. +# (Re)generate polkadot-api type descriptors from a chain spec runtime or, +# when no chain spec is supplied, from a temporary development node. +# Checks that the node binary exists before running either path. # Generates types only if they are missing or empty. # # Usage: # ./generate-types.sh # -set -e +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}/.." BASE_DIR="./tmp" mkdir -p "$BASE_DIR" BINARY="${BINARY_PATH:-../target/release/node-subtensor}" NODE_LOG="${BASE_DIR}/node.log" +RUNTIME_WASM="${BASE_DIR}/runtime.compact.compressed.wasm" + +verify_generated_types() { + if [ ! -s "./.papi/metadata/subtensor.scale" ] || \ + [ ! -s "./.papi/descriptors/dist/index.mjs" ] || \ + [ ! -e "./node_modules/@polkadot-api/descriptors" ]; then + echo "ERROR: polkadot-api did not finish installing the generated descriptors" + exit 1 + fi +} if [ ! -f "$BINARY" ]; then echo "ERROR: Node binary not found at $BINARY" @@ -31,10 +45,32 @@ else fi if [ "$GENERATE_TYPES" = true ]; then + if [ -n "${PAPI_CHAIN_SPEC_PATH:-}" ]; then + if [ ! -f "$PAPI_CHAIN_SPEC_PATH" ]; then + echo "ERROR: Chain spec not found at $PAPI_CHAIN_SPEC_PATH" + exit 1 + fi + + echo "==> Extracting metadata from the chain spec runtime..." + node ./scripts/extract-runtime-wasm.mjs "$PAPI_CHAIN_SPEC_PATH" "$RUNTIME_WASM" + pnpm exec polkadot-api add subtensor --wasm "$RUNTIME_WASM" --skip-codegen + pnpm exec polkadot-api + verify_generated_types + echo "==> Done generating types from the chain spec runtime." + exit 0 + fi + echo "==> Starting dev node (logs at $NODE_LOG)..." "$BINARY" --one --dev &>"$NODE_LOG" & NODE_PID=$! - trap "kill $NODE_PID 2>/dev/null; wait $NODE_PID 2>/dev/null || true; exit 0" EXIT + cleanup_node() { + status=$? + trap - EXIT + kill "$NODE_PID" 2>/dev/null || true + wait "$NODE_PID" 2>/dev/null || true + exit "$status" + } + trap cleanup_node EXIT TIMEOUT=60 ELAPSED=0 @@ -53,6 +89,7 @@ if [ "$GENERATE_TYPES" = true ]; then echo "==> Generating papi types..." pnpm generate-types + verify_generated_types echo "==> Done generating types." exit 0 diff --git a/ts-tests/scripts/test-e2e-shard-plan.mjs b/ts-tests/scripts/test-e2e-shard-plan.mjs new file mode 100644 index 0000000000..db5a12e7e6 --- /dev/null +++ b/ts-tests/scripts/test-e2e-shard-plan.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + buildE2EPlan, + loadShardManifest, + materializeShardEnvironments, + validateManifestCoverage, + validateShardManifest, +} from "./e2e-shard-plan.mjs"; + +const tsTestsDir = join(dirname(fileURLToPath(import.meta.url)), ".."); +const manifest = loadShardManifest(join(tsTestsDir, "e2e-shards.json")); +const suiteOwnership = JSON.parse(readFileSync(join(tsTestsDir, "e2e-suite-ownership.json"), "utf8")); +const all = Object.fromEntries( + ["evm", "staking", "coldkey_swap", "dev", "subnets", "shield"].map((name) => [name, true]) +); +const registeredSelectors = Object.values(suiteOwnership.suites) + .filter(({ owner }) => owner === "pull_request") + .map(({ selector }) => selector) + .sort(); +const plannerSelectors = Object.keys(all).sort(); +if (JSON.stringify(registeredSelectors) !== JSON.stringify(plannerSelectors)) { + throw new Error( + [ + "PR suite ownership and planner selectors do not match.", + `Registered selectors: ${registeredSelectors.join(", ") || "none"}`, + `Planner selectors: ${plannerSelectors.join(", ") || "none"}`, + "Fix: update BOOLEAN_NAMES and buildE2EPlan in ts-tests/scripts/e2e-shard-plan.mjs, then update .github/scripts/classify-typescript-e2e-changes.sh for the same selector.", + "Verify: node ts-tests/scripts/test-e2e-shard-plan.mjs && .github/scripts/test-classify-typescript-e2e-changes.sh", + ].join("\n") + ); +} +const plan = buildE2EPlan(manifest, all); +assert.equal(plan.state_count, 7); +assert.equal(plan.shield_count, 5); +assert.deepEqual( + plan.build_matrix.include.map(({ variant }) => variant), + ["release", "fast"] +); + +for (const [suite, registration] of Object.entries(suiteOwnership.suites)) { + if (registration.owner !== "pull_request") continue; + const selected = Object.fromEntries(Object.keys(all).map((name) => [name, name === registration.selector])); + const suitePlan = buildE2EPlan(manifest, selected); + if (suitePlan.build_count !== 1 || suitePlan.state_count + suitePlan.shield_count === 0) { + throw new Error( + [ + `PR-owned suite "${suite}" with selector "${registration.selector}" is not fully routed.`, + `Planner result: builds=${suitePlan.build_count}, state jobs=${suitePlan.state_count}, Shield jobs=${suitePlan.shield_count}.`, + "Fix: add the selector's binary and execution lane to buildE2EPlan in ts-tests/scripts/e2e-shard-plan.mjs.", + "Verify: node ts-tests/scripts/test-e2e-shard-plan.mjs", + ].join("\n") + ); + } +} + +const evmOnly = buildE2EPlan(manifest, { + ...all, + staking: false, + coldkey_swap: false, + dev: false, + subnets: false, + shield: false, +}); +assert.equal(evmOnly.state_count, 3); +assert.equal(evmOnly.build_count, 1); +assert.equal(evmOnly.build_matrix.include[0].variant, "fast"); + +const future = structuredClone(manifest); +future.suites.evm.shards.push({ + name: "zombienet_evm_d", + files: ["suites/zombienet_evm/future.test.ts"], +}); +const futurePlan = buildE2EPlan(validateShardManifest(future), all); +assert.equal(futurePlan.state_count, 8, "a proposed shard must enter the matrix without changing trusted code"); + +const unsafe = structuredClone(manifest); +unsafe.suites.evm.shards[0].files = ["suites/zombienet_evm/../zombienet_shield/00.00-basic.test.ts"]; +assert.throws(() => validateShardManifest(unsafe), /unsafe or duplicate test file/); + +const coverageRoot = mkdtempSync(join(tmpdir(), "e2e-shard-coverage-")); +try { + for (const file of Object.values(manifest.suites).flatMap(({ shards }) => shards.flatMap(({ files }) => files))) { + const path = join(coverageRoot, file); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, ""); + } + validateManifestCoverage(manifest, coverageRoot); + const nested = join(coverageRoot, "suites/zombienet_evm/nested/future.test.ts"); + mkdirSync(dirname(nested), { recursive: true }); + writeFileSync(nested, ""); + assert.throws( + () => validateManifestCoverage(manifest, coverageRoot), + /Unassigned test files: suites\/zombienet_evm\/nested\/future\.test\.ts[\s\S]*Fix: edit ts-tests\/e2e-shards\.json/, + "nested test files must not bypass shard coverage" + ); +} finally { + rmSync(coverageRoot, { recursive: true, force: true }); +} + +const baseConfig = { + environments: [ + { name: "zombienet_evm", vitestArgs: { bail: 1 } }, + { name: "zombienet_staking", vitestArgs: { bail: 1 } }, + { name: "zombienet_shield", vitestArgs: { bail: 1 }, envVars: ["SHIELD_RUNTIME=release"] }, + ], +}; +const materialized = materializeShardEnvironments(baseConfig, manifest); +assert.equal(materialized.environments.length, 13); +assert.deepEqual(materialized.environments.find(({ name }) => name === "zombienet_evm_a").include, [ + "suites/zombienet_evm/01-contract-deploy-call.test.ts", +]); +assert.equal(materialized.environments.find(({ name }) => name === "zombienet_shield_a").vitestArgs.maxConcurrency, 6); + +console.log("TypeScript E2E shard plan tests passed"); diff --git a/ts-tests/scripts/validate-e2e-config.mjs b/ts-tests/scripts/validate-e2e-config.mjs index d0048c0c93..1ca5c231aa 100644 --- a/ts-tests/scripts/validate-e2e-config.mjs +++ b/ts-tests/scripts/validate-e2e-config.mjs @@ -1,19 +1,149 @@ +import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; +import { loadShardManifest, materializeShardEnvironments, validateManifestCoverage } from "./e2e-shard-plan.mjs"; +import { decodeRuntimeWasm } from "./extract-runtime-wasm.mjs"; + const tsTestsDir = join(dirname(fileURLToPath(import.meta.url)), ".."); -const config = JSON.parse(readFileSync(join(tsTestsDir, "moonwall.config.json"), "utf8")); +const baseConfigPath = process.env.E2E_CONFIG_PATH ?? join(tsTestsDir, "moonwall.config.json"); +const baseConfig = JSON.parse(readFileSync(baseConfigPath, "utf8")); +const suiteOwnership = JSON.parse(readFileSync(join(tsTestsDir, "e2e-suite-ownership.json"), "utf8")); +const shardManifest = loadShardManifest(join(tsTestsDir, "e2e-shards.json")); +validateManifestCoverage(shardManifest, tsTestsDir); + +const ownershipFix = [ + "Fix: edit ts-tests/e2e-suite-ownership.json.", + 'Use owner="pull_request" plus a selector for PR E2E coverage, or owner="scheduled" plus selector=null.', + "Then run: node ts-tests/scripts/validate-e2e-config.mjs && node ts-tests/scripts/test-e2e-shard-plan.mjs && .github/scripts/test-classify-typescript-e2e-changes.sh", +]; +const ownershipError = (summary, details = []) => { + throw new Error([`TypeScript E2E suite ownership error: ${summary}`, ...details, ...ownershipFix].join("\n")); +}; + +if (suiteOwnership.version !== 1) { + ownershipError(`expected registry version 1, found ${JSON.stringify(suiteOwnership.version)}`); +} +if ( + suiteOwnership.suites === null || + typeof suiteOwnership.suites !== "object" || + Array.isArray(suiteOwnership.suites) +) { + ownershipError('the registry must contain a "suites" object'); +} +const suiteDirectories = readdirSync(join(tsTestsDir, "suites"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map(({ name }) => name) + .sort(); +const registeredSuites = Object.keys(suiteOwnership.suites).sort(); +const unregisteredSuites = suiteDirectories.filter((suite) => !registeredSuites.includes(suite)); +const staleSuites = registeredSuites.filter((suite) => !suiteDirectories.includes(suite)); +if (unregisteredSuites.length > 0 || staleSuites.length > 0) { + ownershipError("suite directories and registry entries do not match", [ + `Unregistered ts-tests/suites directories: ${unregisteredSuites.join(", ") || "none"}`, + `Registry entries with no directory: ${staleSuites.join(", ") || "none"}`, + ]); +} + +const registeredEnvironments = []; +const pullRequestSelectors = new Set(); +const canonicalEnvironments = new Map(baseConfig.environments.map((environment) => [environment.name, environment])); +for (const [suite, registration] of Object.entries(suiteOwnership.suites)) { + const fields = Object.keys(registration).sort(); + if (!isDeepStrictEqual(fields, ["environments", "owner", "selector"])) { + ownershipError(`suite "${suite}" must contain exactly owner, selector, and environments`, [ + `Found fields: ${fields.join(", ") || "none"}`, + ]); + } + if (registration.owner !== "pull_request" && registration.owner !== "scheduled") { + ownershipError(`suite "${suite}" has invalid owner ${JSON.stringify(registration.owner)}`); + } + if (!Array.isArray(registration.environments) || registration.environments.length === 0) { + ownershipError(`suite "${suite}" must list at least one canonical Moonwall environment`); + } + for (const environment of registration.environments) { + if (typeof environment !== "string" || environment.length === 0) { + ownershipError(`suite "${suite}" contains an invalid Moonwall environment name`); + } + if (registeredEnvironments.includes(environment)) { + ownershipError(`Moonwall environment "${environment}" is owned by more than one suite`); + } + const environmentConfig = canonicalEnvironments.get(environment); + if (!environmentConfig) { + ownershipError(`suite "${suite}" references missing Moonwall environment "${environment}"`); + } + const expectedTestDirectory = `suites/${suite}`; + if (!isDeepStrictEqual(environmentConfig.testFileDir, [expectedTestDirectory])) { + ownershipError(`environment "${environment}" does not exclusively execute suite "${suite}"`, [ + `Expected testFileDir: [${expectedTestDirectory}]`, + `Found testFileDir: ${JSON.stringify(environmentConfig.testFileDir)}`, + "Fix: edit ts-tests/moonwall.config.json so the environment points exactly at its owned suite directory.", + ]); + } + for (const filter of ["include", "exclude", "skipTests"]) { + if (Object.hasOwn(environmentConfig, filter)) { + ownershipError(`canonical environment "${environment}" may not define ${filter}`, [ + `A canonical environment must discover every test beneath ${expectedTestDirectory}; ${filter} can silently omit owned coverage.`, + `Fix: remove ${filter} from ${environment} in ts-tests/moonwall.config.json. Shard include lists belong only in ts-tests/e2e-shards.json.`, + ]); + } + } + registeredEnvironments.push(environment); + } + if (registration.owner === "pull_request") { + if (typeof registration.selector !== "string" || !/^[a-z][a-z0-9_]*$/.test(registration.selector)) { + ownershipError(`PR-owned suite "${suite}" has invalid selector ${JSON.stringify(registration.selector)}`); + } + if (pullRequestSelectors.has(registration.selector)) { + ownershipError(`PR selector "${registration.selector}" is assigned to more than one suite`); + } + pullRequestSelectors.add(registration.selector); + } else if (registration.selector !== null) { + ownershipError(`scheduled suite "${suite}" must use selector=null`); + } +} +const configuredEnvironments = baseConfig.environments.map(({ name }) => name).sort(); +registeredEnvironments.sort(); +const unownedEnvironments = configuredEnvironments.filter((name) => !registeredEnvironments.includes(name)); +const staleEnvironments = registeredEnvironments.filter((name) => !configuredEnvironments.includes(name)); +if (unownedEnvironments.length > 0 || staleEnvironments.length > 0) { + ownershipError("canonical Moonwall environments and registry ownership do not match", [ + `Moonwall environments with no owner: ${unownedEnvironments.join(", ") || "none"}`, + `Registered environments missing from moonwall.config.json: ${staleEnvironments.join(", ") || "none"}`, + ]); +} + +const shardNames = new Set(Object.values(shardManifest.suites).flatMap(({ shards }) => shards.map(({ name }) => name))); +for (const { name } of baseConfig.environments) { + if (shardNames.has(name)) throw new Error(`${name} is generated from e2e-shards.json and must not be checked in`); +} +const config = materializeShardEnvironments(baseConfig, shardManifest); const singleNodeSpec = JSON.parse(readFileSync(join(tsTestsDir, "configs/zombie_single_node.json"), "utf8")); const shieldSpec = JSON.parse(readFileSync(join(tsTestsDir, "configs/zombie_extended.json"), "utf8")); const environments = new Map(config.environments.map((environment) => [environment.name, environment])); -const shieldFiles = readdirSync(join(tsTestsDir, "suites/zombienet_shield")) - .filter((file) => file.endsWith(".test.ts")) - .map((file) => `suites/zombienet_shield/${file}`) - .sort(); -const shieldShardNames = ["zombienet_shield_a", "zombienet_shield_b", "zombienet_shield_c"]; +assert.deepEqual( + [...decodeRuntimeWasm({ genesis: { raw: { top: { "0x3a636f6465": "0x0061736d" } } } })], + [0, 97, 115, 109] +); +assert.throws(() => decodeRuntimeWasm({}), /missing a valid/); +assert.throws( + () => decodeRuntimeWasm({ genesis: { raw: { top: { "0x3a636f6465": "0xnot-hex" } } } }), + /missing a valid/ +); + +const evmShards = shardManifest.suites.evm.shards; +const evmFiles = evmShards.flatMap(({ files }) => files).sort(); +const evmShardNames = evmShards.map(({ name }) => name); +const stakingShards = shardManifest.suites.staking.shards; +const stakingFiles = stakingShards.flatMap(({ files }) => files).sort(); +const stakingShardNames = stakingShards.map(({ name }) => name); +const shieldShards = shardManifest.suites.shield.shards; +const shieldFiles = shieldShards.flatMap(({ files }) => files).sort(); +const shieldShardNames = shieldShards.map(({ name }) => name); +const expectedShieldBinaries = new Map(shieldShardNames.map((name) => [name, "release"])); const shieldShardIncludes = shieldShardNames.map((name) => { const environment = environments.get(name); if (!environment) { @@ -27,6 +157,28 @@ const shieldShardIncludes = shieldShardNames.map((name) => { }); // File counts intentionally differ: the shards are balanced by measured runtime. const shieldIncludes = shieldShardIncludes.flat(); +const productionTimingFile = "suites/zombienet_shield/03-timing.test.ts"; +const shieldDefaultVitestArgs = { bail: 1 }; +const shieldConcurrency = new Map(shieldShards.map(({ name, maxConcurrency }) => [name, maxConcurrency])); + +for (const [index, includes] of shieldShardIncludes.entries()) { + const name = shieldShardNames[index]; + const binary = expectedShieldBinaries.get(name); + const containsProductionTiming = includes.includes(productionTimingFile); + if (containsProductionTiming && (binary !== "release" || includes.length !== 1)) { + throw new Error(`${productionTimingFile} must be the only file in one release-runtime shard`); + } + const environment = environments.get(name); + const maxConcurrency = shieldConcurrency.get(name); + const expectedVitestArgs = { + ...shieldDefaultVitestArgs, + sequence: { concurrent: true }, + maxConcurrency, + }; + if (!isDeepStrictEqual(environment?.vitestArgs, expectedVitestArgs)) { + throw new Error(`${name} must run its state-isolated cases with maxConcurrency=${maxConcurrency}`); + } +} const duplicateShieldFiles = shieldIncludes.filter((file, index) => shieldIncludes.indexOf(file) !== index); if (duplicateShieldFiles.length > 0) { @@ -40,6 +192,68 @@ if (JSON.stringify(sortedIncludes) !== JSON.stringify(shieldFiles)) { throw new Error(`Shield shard coverage mismatch; missing=[${missing.join(", ")}] unknown=[${unknown.join(", ")}]`); } +const evmShardIncludes = evmShardNames.map((name) => { + const includes = environments.get(name)?.include ?? []; + if (includes.length === 0) { + throw new Error(`${name} must include at least one EVM test`); + } + return includes; +}); +const evmIncludes = evmShardIncludes.flat(); +const duplicateEvmFiles = evmIncludes.filter((file, index) => evmIncludes.indexOf(file) !== index); +if (duplicateEvmFiles.length > 0) { + throw new Error(`EVM tests assigned to multiple shards: ${[...new Set(duplicateEvmFiles)].join(", ")}`); +} +const sortedEvmIncludes = [...evmIncludes].sort(); +if (!isDeepStrictEqual(sortedEvmIncludes, evmFiles)) { + const missing = evmFiles.filter((file) => !sortedEvmIncludes.includes(file)); + const unknown = sortedEvmIncludes.filter((file) => !evmFiles.includes(file)); + throw new Error(`EVM shard coverage mismatch; missing=[${missing.join(", ")}] unknown=[${unknown.join(", ")}]`); +} + +const canonicalEvmEnvironment = environments.get("zombienet_evm"); +if (!canonicalEvmEnvironment) { + throw new Error("Missing Moonwall environment: zombienet_evm"); +} +const sharedEvmSettings = ({ name: _name, include: _include, ...settings }) => settings; +const canonicalEvmSettings = sharedEvmSettings(canonicalEvmEnvironment); +for (const name of evmShardNames) { + if (!isDeepStrictEqual(sharedEvmSettings(environments.get(name)), canonicalEvmSettings)) { + throw new Error(`${name} settings must match zombienet_evm except for name and include`); + } +} + +const stakingShardIncludes = stakingShardNames.map((name) => { + const includes = environments.get(name)?.include ?? []; + if (includes.length === 0) { + throw new Error(`${name} must include at least one staking test`); + } + return includes; +}); +const stakingIncludes = stakingShardIncludes.flat(); +const duplicateStakingFiles = stakingIncludes.filter((file, index) => stakingIncludes.indexOf(file) !== index); +if (duplicateStakingFiles.length > 0) { + throw new Error(`Staking tests assigned to multiple shards: ${[...new Set(duplicateStakingFiles)].join(", ")}`); +} +const sortedStakingIncludes = [...stakingIncludes].sort(); +if (!isDeepStrictEqual(sortedStakingIncludes, stakingFiles)) { + const missing = stakingFiles.filter((file) => !sortedStakingIncludes.includes(file)); + const unknown = sortedStakingIncludes.filter((file) => !stakingFiles.includes(file)); + throw new Error(`Staking shard coverage mismatch; missing=[${missing.join(", ")}] unknown=[${unknown.join(", ")}]`); +} + +const canonicalStakingEnvironment = environments.get("zombienet_staking"); +if (!canonicalStakingEnvironment) { + throw new Error("Missing Moonwall environment: zombienet_staking"); +} +const sharedStakingSettings = ({ name: _name, include: _include, ...settings }) => settings; +const canonicalStakingSettings = sharedStakingSettings(canonicalStakingEnvironment); +for (const name of stakingShardNames) { + if (!isDeepStrictEqual(sharedStakingSettings(environments.get(name)), canonicalStakingSettings)) { + throw new Error(`${name} settings must match zombienet_staking except for name and include`); + } +} + const singleNodeConfig = "./configs/zombie_single_node.json"; const singleNodes = singleNodeSpec.relaychain?.nodes ?? []; if ( @@ -50,7 +264,24 @@ if ( throw new Error("Single-node state spec must contain one validator using --sealing=100"); } -for (const name of ["zombienet_staking", "zombienet_coldkey_swap", "zombienet_evm", "zombienet_subnets"]) { +const descriptorScripts = ["build-spec.sh", "generate-types-from-chain-spec.sh"]; +for (const environment of config.environments.filter(({ foundation }) => foundation?.type === "zombie")) { + if (!isDeepStrictEqual(environment.runScripts?.slice(0, 2), descriptorScripts)) { + throw new Error(`${environment.name} must build its chain spec before generating exact-runtime descriptors`); + } +} +if (!isDeepStrictEqual(environments.get("dev")?.runScripts, ["generate-types.sh"])) { + throw new Error("dev must retain live-node descriptor generation"); +} + +for (const name of [ + "zombienet_staking", + ...stakingShardNames, + "zombienet_coldkey_swap", + "zombienet_evm", + ...evmShardNames, + "zombienet_subnets", +]) { const configPath = environments.get(name)?.foundation?.zombieSpec?.configPath; if (configPath !== singleNodeConfig) { throw new Error(`${name} must use ${singleNodeConfig}; found ${configPath ?? "no config"}`); @@ -70,11 +301,21 @@ const canonicalShieldEnvironment = environments.get("zombienet_shield"); if (!canonicalShieldEnvironment) { throw new Error("Missing Moonwall environment: zombienet_shield"); } -const sharedShieldSettings = ({ name: _name, include: _include, ...settings }) => settings; +const sharedShieldSettings = ({ + name: _name, + include: _include, + envVars: _envVars, + vitestArgs: _vitestArgs, + ...settings +}) => settings; const canonicalShieldSettings = sharedShieldSettings(canonicalShieldEnvironment); +if (!isDeepStrictEqual(canonicalShieldEnvironment.vitestArgs, shieldDefaultVitestArgs)) { + throw new Error("zombienet_shield must retain the default fail-fast Vitest configuration"); +} for (const name of ["zombienet_shield", ...shieldShardNames]) { const environment = environments.get(name); + const expectedRuntime = name === "zombienet_shield" ? "release" : expectedShieldBinaries.get(name); const configPath = environment?.foundation?.zombieSpec?.configPath; const connectionNames = new Set((environment?.connections ?? []).map((connection) => connection.name)); if (configPath !== shieldConfig) { @@ -83,6 +324,9 @@ for (const name of ["zombienet_shield", ...shieldShardNames]) { if (!connectionNames.has("Node") || !connectionNames.has("NodeFull")) { throw new Error(`${name} must expose authority and full-node connections`); } + if (!isDeepStrictEqual(environment?.envVars, [`SHIELD_RUNTIME=${expectedRuntime}`])) { + throw new Error(`${name} must declare SHIELD_RUNTIME=${expectedRuntime}`); + } if (name !== "zombienet_shield") { if (!isDeepStrictEqual(sharedShieldSettings(environment), canonicalShieldSettings)) { throw new Error(`${name} settings must match zombienet_shield except for name and include`); @@ -91,5 +335,5 @@ for (const name of ["zombienet_shield", ...shieldShardNames]) { } console.log( - `Validated ${shieldFiles.length} Shield files, ${shieldShardNames.length + 1} multi-node Shield environments, and four single-node state suites.` + `Validated ${shieldFiles.length} Shield files across ${shieldShardNames.length} production-runtime shards, ${evmFiles.length} EVM files across ${evmShardNames.length} shards, ${stakingFiles.length} staking files across ${stakingShardNames.length} shards, ${shieldShardNames.length + 1} multi-node Shield environments, and eight single-node state environments.` ); diff --git a/ts-tests/suites/zombienet_shield/00.01-basic.test.ts b/ts-tests/suites/zombienet_shield/00.01-basic.test.ts index 7801019e79..25df54c992 100644 --- a/ts-tests/suites/zombienet_shield/00.01-basic.test.ts +++ b/ts-tests/suites/zombienet_shield/00.01-basic.test.ts @@ -26,6 +26,12 @@ describeSuite({ let alice: KeyringPair; let bob: KeyringPair; let charlie: KeyringPair; + let dave: KeyringPair; + let eve: KeyringPair; + let ferdie: KeyringPair; + let one: KeyringPair; + let two: KeyringPair; + let t04Recipient: KeyringPair; beforeAll(async () => { client = context.papi("Node"); @@ -35,12 +41,22 @@ describeSuite({ alice = keyring.addFromUri("//Alice"); bob = keyring.addFromUri("//Bob"); charlie = keyring.addFromUri("//Charlie"); + dave = keyring.addFromUri("//Dave"); + eve = keyring.addFromUri("//Eve"); + ferdie = keyring.addFromUri("//Ferdie"); + one = keyring.addFromUri("//One"); + two = keyring.addFromUri("//Two"); + t04Recipient = keyring.addFromUri("//ShieldT04Recipient"); await checkRuntime(api); await waitForFinalizedBlocks(api, 3); }, 120000); + // The environment runs these cases concurrently. Each case has a + // distinct sender nonce and successful recipient; the rejected T05/T06 + // calls share Dave only because neither is allowed to change his balance. + it({ id: "T01", title: "Happy path: wrapper and inner tx are included in the same block", @@ -70,21 +86,21 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, dave.address); - // Encrypt a transfer of more than Alice has. + // Encrypt a transfer of more than Charlie has. // The wrapper is valid (correct key_hash, valid encryption), but the // inner transfer should fail at dispatch with InsufficientBalance. - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, charlie.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(dave.address), value: 9_000_000_000_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(charlie), { nonce: nonce + 1 }); - await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); + await submitEncrypted(api, charlie, hexToU8a(innerTxHex), nextKey, nonce); - // The inner transfer failed, so bob's balance should not increase. - const balanceAfter = await getBalance(api, bob.address); + // The inner transfer failed, so Dave's balance should not increase. + const balanceAfter = await getBalance(api, dave.address); expect(balanceAfter).toBe(balanceBefore); }, }); @@ -93,7 +109,7 @@ describeSuite({ id: "T03", title: "Malformed ciphertext is rejected at pool level", test: async () => { - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, eve.address); // 5 bytes of garbage — not valid ciphertext at all. const garbage = new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05]); @@ -104,7 +120,7 @@ describeSuite({ // Pool validation rejects with FailedShieldedTxParsing (Custom code 23). await expect( - tx.signAndSubmit(getSignerFromKeypair(alice), { nonce, mortality: { mortal: true, period: 8 } }) + tx.signAndSubmit(getSignerFromKeypair(eve), { nonce, mortality: { mortal: true, period: 8 } }) ).rejects.toThrow(); }, }); @@ -118,9 +134,9 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, charlie.address); + const balanceBefore = await getBalance(api, t04Recipient.address); - const senders = [alice, bob]; + const senders = [ferdie, one]; const amount = 1_000_000_000n; const txPromises = []; @@ -128,17 +144,17 @@ describeSuite({ const nonce = await getAccountNonce(api, sender.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(charlie.address), + dest: MultiAddress.Id(t04Recipient.address), value: amount, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(sender), { nonce: nonce + 1 }); txPromises.push(submitEncrypted(api, sender, hexToU8a(innerTxHex), nextKey, nonce)); } await Promise.all(txPromises); - const balanceAfter = await getBalance(api, charlie.address); - expect(balanceAfter).toBeGreaterThan(balanceBefore); + const balanceAfter = await getBalance(api, t04Recipient.address); + expect(balanceAfter).toBe(balanceBefore + amount * BigInt(senders.length)); }, }); @@ -149,13 +165,13 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, dave.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, bob.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(dave.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(bob), { nonce: nonce + 1 }); const ciphertext = await encryptTransaction(hexToU8a(innerTxHex), nextKey!); @@ -166,7 +182,7 @@ describeSuite({ const tx = api.tx.MevShield.submit_encrypted({ ciphertext: Binary.fromBytes(tampered), }); - const signedHex = await tx.sign(getSignerFromKeypair(alice), { + const signedHex = await tx.sign(getSignerFromKeypair(bob), { nonce, mortality: { mortal: true, period: 8 }, }); @@ -177,7 +193,7 @@ describeSuite({ await waitForFinalizedBlocks(api, 3); // The inner transfer should NOT have executed. - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, dave.address); expect(balanceAfter).toBe(balanceBefore); }, }); @@ -193,20 +209,20 @@ describeSuite({ // currentKey and nextKey positions. await waitForFinalizedBlocks(api, 3); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, dave.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, two.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(dave.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(two), { nonce: nonce + 1 }); const ciphertext = await encryptTransaction(hexToU8a(innerTxHex), staleKey!); const tx = api.tx.MevShield.submit_encrypted({ ciphertext: Binary.fromBytes(ciphertext), }); - const signedHex = await tx.sign(getSignerFromKeypair(alice), { + const signedHex = await tx.sign(getSignerFromKeypair(two), { nonce, mortality: { mortal: true, period: 8 }, }); @@ -217,7 +233,7 @@ describeSuite({ await waitForFinalizedBlocks(api, 3); // The inner transfer should NOT have executed. - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, dave.address); expect(balanceAfter).toBe(balanceBefore); }, }); diff --git a/ts-tests/suites/zombienet_shield/01-scaling.test.ts b/ts-tests/suites/zombienet_shield/01-scaling.test.ts index aa12f50d46..87a57f5925 100644 --- a/ts-tests/suites/zombienet_shield/01-scaling.test.ts +++ b/ts-tests/suites/zombienet_shield/01-scaling.test.ts @@ -52,12 +52,16 @@ describeSuite({ let alice: KeyringPair; let bob: KeyringPair; let charlie: KeyringPair; + let t03Sender: KeyringPair; + let t03Recipient: KeyringPair; beforeAll(async () => { const keyring = new Keyring({ type: "sr25519" }); alice = keyring.addFromUri("//Alice"); bob = keyring.addFromUri("//Bob"); charlie = keyring.addFromUri("//Charlie"); + t03Sender = keyring.addFromUri("//Dave"); + t03Recipient = keyring.addFromUri("//Eve"); client = context.papi("Node"); api = client.getTypedApi(subtensor); @@ -67,6 +71,11 @@ describeSuite({ await checkRuntime(api); }, 120000); + // This shard runs all six read/write cases concurrently. T03 uses + // Dave/Eve while T04 uses Alice/Bob/Charlie, so the writers never + // share a nonce or a recipient balance. The key and topology cases + // are read-only. + it({ id: "T01", title: "Network scales to 6 nodes with full peering", @@ -101,18 +110,19 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const amount = 5_000_000_000n; + const balanceBefore = await getBalance(api, t03Recipient.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, t03Sender.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), - value: 5_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + dest: MultiAddress.Id(t03Recipient.address), + value: amount, + }).sign(getSignerFromKeypair(t03Sender), { nonce: nonce + 1 }); - await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); + await submitEncrypted(api, t03Sender, hexToU8a(innerTxHex), nextKey, nonce); - const balanceAfter = await getBalance(api, bob.address); - expect(balanceAfter).toBeGreaterThan(balanceBefore); + const balanceAfter = await getBalance(api, t03Recipient.address); + expect(balanceAfter).toBe(balanceBefore + amount); // The state-oriented suites run one immediately-finalized node, // so retain explicit GRANDPA propagation and Frontier indexing @@ -121,13 +131,15 @@ describeSuite({ // advancing latest-state views. const finalizedHash = (await client._request("chain_getFinalizedHead", [])) as string; const finalizedNumber = await api.query.System.Number.getValue({ at: finalizedHash }); - const authorityAccount = await api.query.System.Account.getValue(bob.address, { at: finalizedHash }); + const authorityAccount = await api.query.System.Account.getValue(t03Recipient.address, { + at: finalizedHash, + }); expect(authorityAccount.data.free).toBe(balanceAfter); const deadline = Date.now() + 60_000; let fullNodeBalance: bigint | undefined; while (Date.now() < deadline) { try { - const fullAccount = await apiFull.query.System.Account.getValue(bob.address, { + const fullAccount = await apiFull.query.System.Account.getValue(t03Recipient.address, { at: finalizedHash, }); fullNodeBalance = fullAccount.data.free; @@ -166,7 +178,7 @@ describeSuite({ const innerTxHex = await api.tx.Balances.transfer_keep_alive({ dest: MultiAddress.Id(charlie.address), value: amount, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(sender), { nonce: nonce + 1 }); txPromises.push(submitEncrypted(api, sender, hexToU8a(innerTxHex), nextKey, nonce)); } @@ -174,7 +186,7 @@ describeSuite({ await Promise.all(txPromises); const balanceAfter = await getBalance(api, charlie.address); - expect(balanceAfter).toBeGreaterThan(balanceBefore); + expect(balanceAfter).toBe(balanceBefore + BigInt(senders.length) * amount); }, }); }, diff --git a/ts-tests/suites/zombienet_shield/02-edge-cases.test.ts b/ts-tests/suites/zombienet_shield/02-edge-cases.test.ts index 951c939b97..69ca4698de 100644 --- a/ts-tests/suites/zombienet_shield/02-edge-cases.test.ts +++ b/ts-tests/suites/zombienet_shield/02-edge-cases.test.ts @@ -24,11 +24,15 @@ describeSuite({ let alice: KeyringPair; let bob: KeyringPair; + let charlie: KeyringPair; + let dave: KeyringPair; beforeAll(async () => { const keyring = new Keyring({ type: "sr25519" }); alice = keyring.addFromUri("//Alice"); bob = keyring.addFromUri("//Bob"); + charlie = keyring.addFromUri("//Charlie"); + dave = keyring.addFromUri("//Dave"); api = context.papi("Node").getTypedApi(subtensor); @@ -37,6 +41,10 @@ describeSuite({ await waitForFinalizedBlocks(api, 2); }, 120000); + // T01 and T02 run concurrently in this shard. Each case has a distinct + // funded sender and recipient, so nonce and balance state remain + // independent while production-time finality waits overlap. + it({ id: "T01", title: "Encrypted tx persists across blocks (CurrentKey fallback)", @@ -47,12 +55,13 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); + const amount = 2_000_000_000n; const balanceBefore = await getBalance(api, bob.address); const nonce = await getAccountNonce(api, alice.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ dest: MultiAddress.Id(bob.address), - value: 2_000_000_000n, + value: amount, }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); // Submit and wait for finalization — the tx may land in the next block @@ -60,7 +69,7 @@ describeSuite({ await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); const balanceAfter = await getBalance(api, bob.address); - expect(balanceAfter).toBeGreaterThan(balanceBefore); + expect(balanceAfter).toBe(balanceBefore + amount); }, }); @@ -77,18 +86,18 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, dave.address); // Garbage "inner transaction" bytes — not a valid extrinsic at all. const garbageInner = new Uint8Array(64); for (let i = 0; i < 64; i++) garbageInner[i] = (i * 7 + 13) & 0xff; - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, charlie.address); - await submitEncrypted(api, alice, garbageInner, nextKey, nonce); + await submitEncrypted(api, charlie, garbageInner, nextKey, nonce); // No balance change — the garbage inner call could not have been a valid transfer. - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, dave.address); expect(balanceAfter).toBe(balanceBefore); }, }); diff --git a/ts-tests/suites/zombienet_shield/03-timing.test.ts b/ts-tests/suites/zombienet_shield/03-timing.test.ts index b100e02e85..4bd655b148 100644 --- a/ts-tests/suites/zombienet_shield/03-timing.test.ts +++ b/ts-tests/suites/zombienet_shield/03-timing.test.ts @@ -25,17 +25,33 @@ describeSuite({ let alice: KeyringPair; let bob: KeyringPair; + let charlie: KeyringPair; + let dave: KeyringPair; + let eve: KeyringPair; + let ferdie: KeyringPair; + let one: KeyringPair; + let two: KeyringPair; beforeAll(async () => { const keyring = new Keyring({ type: "sr25519" }); alice = keyring.addFromUri("//Alice"); bob = keyring.addFromUri("//Bob"); + charlie = keyring.addFromUri("//Charlie"); + dave = keyring.addFromUri("//Dave"); + eve = keyring.addFromUri("//Eve"); + ferdie = keyring.addFromUri("//Ferdie"); + one = keyring.addFromUri("//One"); + two = keyring.addFromUri("//Two"); api = context.papi("Node").getTypedApi(subtensor); await checkRuntime(api); }, 120000); + // This environment runs the four cases concurrently. Every case uses a + // distinct pre-funded sender and recipient so nonce and balance state + // remain independent while their production-time finality waits overlap. + it({ id: "T01", title: "Submit immediately after a new block", @@ -74,17 +90,17 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, dave.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, charlie.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(dave.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(charlie), { nonce: nonce + 1 }); - await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); + await submitEncrypted(api, charlie, hexToU8a(innerTxHex), nextKey, nonce); - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, dave.address); expect(balanceAfter).toBeGreaterThan(balanceBefore); }, }); @@ -103,17 +119,17 @@ describeSuite({ const nextKey = await getNextKey(api); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, ferdie.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, eve.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(ferdie.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(eve), { nonce: nonce + 1 }); - await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); + await submitEncrypted(api, eve, hexToU8a(innerTxHex), nextKey, nonce); - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, ferdie.address); expect(balanceAfter).toBeGreaterThan(balanceBefore); }, }); @@ -130,17 +146,17 @@ describeSuite({ await sleep(12_000); - const balanceBefore = await getBalance(api, bob.address); + const balanceBefore = await getBalance(api, two.address); - const nonce = await getAccountNonce(api, alice.address); + const nonce = await getAccountNonce(api, one.address); const innerTxHex = await api.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(two.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(one), { nonce: nonce + 1 }); - await submitEncrypted(api, alice, hexToU8a(innerTxHex), nextKey, nonce); + await submitEncrypted(api, one, hexToU8a(innerTxHex), nextKey, nonce); - const balanceAfter = await getBalance(api, bob.address); + const balanceAfter = await getBalance(api, two.address); expect(balanceAfter).toBeGreaterThan(balanceBefore); }, }); diff --git a/ts-tests/suites/zombienet_shield/04-mortality.test.ts b/ts-tests/suites/zombienet_shield/04-mortality.test.ts index c1fb88a161..44312501f9 100644 --- a/ts-tests/suites/zombienet_shield/04-mortality.test.ts +++ b/ts-tests/suites/zombienet_shield/04-mortality.test.ts @@ -11,16 +11,20 @@ import { getAccountNonce, getBalance, getNextKey, + getShieldSlotDurationMs, getSignerFromKeypair, waitForFinalizedBlocks, } from "../../utils"; import { describeSuite } from "@moonwall/cli"; import { sleep } from "@zombienet/utils"; -// MAX_SHIELD_ERA_PERIOD is 8 blocks. With 12s slots, that's ~96s. +// MAX_SHIELD_ERA_PERIOD is eight blocks. The assertion is block-based, so it +// remains identical under the fast runtime while avoiding a production-time wait. const MAX_ERA_BLOCKS = 8; -const SLOT_DURATION_MS = 12_000; -const POLL_INTERVAL_MS = 3_000; +const SLOT_DURATION_MS = getShieldSlotDurationMs(); +const POLL_INTERVAL_MS = Math.min(3_000, SLOT_DURATION_MS); +const STARTUP_TIMEOUT_MS = Math.max(30_000, (MAX_ERA_BLOCKS + 8) * SLOT_DURATION_MS); +const EVICTION_TIMEOUT_MS = Math.max(30_000, MAX_ERA_BLOCKS * 3 * SLOT_DURATION_MS); describeSuite({ id: "04_mortality", @@ -32,28 +36,25 @@ describeSuite({ let apiFull: TypedApi; let clientFull: PolkadotClient; - let alice: KeyringPair; - let bob: KeyringPair; + let mortalitySender: KeyringPair; + let mortalityRecipient: KeyringPair; - beforeAll( - async () => { - const keyring = new Keyring({ type: "sr25519" }); - alice = keyring.addFromUri("//Alice"); - bob = keyring.addFromUri("//Bob"); + beforeAll(async () => { + const keyring = new Keyring({ type: "sr25519" }); + mortalitySender = keyring.addFromUri("//Eve"); + mortalityRecipient = keyring.addFromUri("//Ferdie"); - apiAuthority = context.papi("Node").getTypedApi(subtensor); + apiAuthority = context.papi("Node").getTypedApi(subtensor); - clientFull = context.papi("NodeFull"); - apiFull = clientFull.getTypedApi(subtensor); + clientFull = context.papi("NodeFull"); + apiFull = clientFull.getTypedApi(subtensor); - await checkRuntime(apiAuthority); + await checkRuntime(apiAuthority); - // Wait for a fresh finalized block, then immediately read NextKey and submit. - // This tests the "just after block" boundary where keys just rotated. - await waitForFinalizedBlocks(apiAuthority, 1); - }, - (MAX_ERA_BLOCKS + 8) * SLOT_DURATION_MS - ); + // Wait for a fresh finalized block, then immediately read NextKey and submit. + // This tests the "just after block" boundary where keys just rotated. + await waitForFinalizedBlocks(apiAuthority, 1); + }, STARTUP_TIMEOUT_MS); it({ id: "T01", @@ -63,13 +64,13 @@ describeSuite({ const nextKey = await getNextKey(apiAuthority); expect(nextKey).toBeDefined(); - const balanceBefore = await getBalance(apiFull, bob.address); + const balanceBefore = await getBalance(apiFull, mortalityRecipient.address); - const nonce = await getAccountNonce(apiFull, alice.address); + const nonce = await getAccountNonce(apiFull, mortalitySender.address); const innerTxHex = await apiFull.tx.Balances.transfer_keep_alive({ - dest: MultiAddress.Id(bob.address), + dest: MultiAddress.Id(mortalityRecipient.address), value: 1_000_000_000n, - }).sign(getSignerFromKeypair(alice), { nonce: nonce + 1 }); + }).sign(getSignerFromKeypair(mortalitySender), { nonce: nonce + 1 }); // Encrypt with valid key, then tamper the key_hash so no proposer will include it. const ciphertext = await encryptTransaction(hexToU8a(innerTxHex), nextKey); @@ -83,7 +84,7 @@ describeSuite({ // Sign with short mortality (must be ≤ MAX_SHIELD_ERA_PERIOD=8 to pass // CheckMortality validation). The tx enters the pool but no proposer // will include it (tampered key_hash doesn't match PendingKey). - const signedHex = await tx.sign(getSignerFromKeypair(alice), { + const signedHex = await tx.sign(getSignerFromKeypair(mortalitySender), { nonce, mortality: { mortal: true, period: 8 }, }); @@ -98,7 +99,7 @@ describeSuite({ } // Verify it's in the pool. - await sleep(1_000); + await sleep(Math.min(1_000, SLOT_DURATION_MS)); const normalizedTx = signedHex.toLowerCase(); const pending: string[] = await clientFull._request("author_pendingExtrinsics", []); const inPool = pending.some((hex) => hex.toLowerCase() === normalizedTx); @@ -107,9 +108,9 @@ describeSuite({ // Now poll until our specific tx disappears (mortality eviction). // Use a generous timeout — CI zombienet nodes can miss AURA slots, - // so N blocks may take significantly longer than N * 12s. + // so N blocks may take significantly longer than N nominal slots. const start = Date.now(); - const maxPollMs = MAX_ERA_BLOCKS * 3 * SLOT_DURATION_MS; + const maxPollMs = EVICTION_TIMEOUT_MS; let evicted = false; log(`Waiting for mortality eviction (up to ${maxPollMs / 1000}s)...`); @@ -132,7 +133,7 @@ describeSuite({ expect(evicted).toBe(true); // The inner transfer should NOT have executed. - const balanceAfter = await getBalance(apiFull, bob.address); + const balanceAfter = await getBalance(apiFull, mortalityRecipient.address); expect(balanceAfter).toBe(balanceBefore); }, }); diff --git a/ts-tests/utils/shield_helpers.ts b/ts-tests/utils/shield_helpers.ts index a437e6591c..c952e1f693 100644 --- a/ts-tests/utils/shield_helpers.ts +++ b/ts-tests/utils/shield_helpers.ts @@ -9,6 +9,20 @@ import { type TypedApi, Binary } from "polkadot-api"; import { getSignerFromKeypair } from "./account.ts"; import { waitForFinalizedBlocks } from "./transactions.ts"; +export type ShieldRuntimeMode = "release" | "fast"; + +const FAST_RUNTIME_THRESHOLD_MS = 6_000; + +export const getShieldRuntimeMode = (): ShieldRuntimeMode => { + const mode = process.env.SHIELD_RUNTIME ?? "release"; + if (mode !== "release" && mode !== "fast") { + throw new Error(`Unsupported SHIELD_RUNTIME value: ${mode}`); + } + return mode; +}; + +export const getShieldSlotDurationMs = (): number => (getShieldRuntimeMode() === "fast" ? 250 : 12_000); + const keyToBytes = (key: unknown): Uint8Array => { if (key instanceof Uint8Array) { return key; @@ -41,11 +55,11 @@ export const checkRuntime = async (api: TypedApi) => { const blockTimeMs = ts2 - ts1; - const MIN_BLOCK_TIME_MS = 6000; - // We check at least half of the block time length - if (blockTimeMs < MIN_BLOCK_TIME_MS) { + const expectedMode = getShieldRuntimeMode(); + const detectedMode: ShieldRuntimeMode = blockTimeMs < FAST_RUNTIME_THRESHOLD_MS ? "fast" : "release"; + if (detectedMode !== expectedMode) { throw new Error( - `Fast runtime detected (block time ~${blockTimeMs}ms < ${MIN_BLOCK_TIME_MS}ms). Rebuild with normal runtime before running MEV Shield tests.` + `Expected the ${expectedMode} Shield runtime, but detected ${detectedMode} from a ~${blockTimeMs}ms block-time delta.` ); } };