Add clone block latency monitoring and epoch soak

.github/scripts/classify-runtime-changes.sh

@@ -29,6 +29,10 @@ while IFS= read -r path; do
       runtime=true
       sdk_drift=true
       ;;
+    .github/workflows/clone-epoch-soak.yml|.github/scripts/test-clone-epoch-soak-workflow.sh|clones/scripts/clone-process-supervision.sh|clones/scripts/run-clone-block-monitor.sh|clones/scripts/run-clone-epoch-soak.sh|clones/js-tests/lib/clone-invariants.ts|clones/js-tests/lib/clone-performance.ts|clones/js-tests/lib/clone-readiness.ts|clones/js-tests/scripts/monitor-clone-blocks.ts|clones/js-tests/scripts/run-clone-epoch-soak.ts|clones/js-tests/scripts/wait-clone-readiness.ts|clones/js-tests/tests/clone-performance.test.ts|clones/js-tests/package.json)
+      runtime=true
+      snapshot_ci=true
+      ;;
     clones/*|website/apps/bittensor-website/scripts/*)
       runtime=true
       ;;
@@ -63,7 +67,7 @@ while IFS= read -r path; do
   esac
 
   case "$path" in
-    .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|clones/scripts/run-clone-regression-phase.sh)
+    .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|.github/workflows/clone-epoch-soak.yml|clones/scripts/clone-process-supervision.sh|clones/scripts/run-clone-block-monitor.sh|clones/scripts/run-clone-regression-phase.sh|clones/scripts/run-clone-epoch-soak.sh)
       snapshot_ci=true
       ;;
   esac

.github/scripts/download-artifact.sh

@@ -19,14 +19,16 @@ output_file="$6"
 : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}"
 : "${GITHUB_REPOSITORY_ID:?GITHUB_REPOSITORY_ID must be set}"
 
+release_sha="${EXPECTED_RELEASE_SHA:-${GITHUB_SHA:-}}"
+
 [[ "$artifact_id" =~ ^[1-9][0-9]*$ ]] || usage
 [[ "$expected_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || usage
 [[ "$expected_size" =~ ^[1-9][0-9]*$ ]] || usage
 [[ -n "$destination" && "$destination" != / ]] || usage
 case "$artifact_name" in
   mainnet-snapshot|try-runtime-snap-v0.10.1-mainnet|try-runtime-snap-v0.10.1-testnet|try-runtime-snap-v0.10.1-devnet) ;;
-  "node-subtensor-release-${GITHUB_SHA:-invalid}")
-    [[ "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || usage
+  "node-subtensor-release-${release_sha:-invalid}")
+    [[ "$release_sha" =~ ^[0-9a-f]{40}$ ]] || usage
     ;;
   *) echo "artifact is outside the host-cache allowlist: $artifact_name" >&2; exit 2 ;;
 esac

.github/scripts/select-shared-release-artifact.sh

@@ -17,12 +17,38 @@ max_wait_seconds="${2:-360}"
 : "${GITHUB_SHA:?GITHUB_SHA must be set}"
 : "${GITHUB_PR_HEAD_SHA:?GITHUB_PR_HEAD_SHA must be set}"
 
+event_name="${GITHUB_EVENT_NAME:-pull_request}"
 [[ "$GITHUB_REPOSITORY_ID" =~ ^[1-9][0-9]*$ ]] || usage
 [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] || usage
 [[ "$GITHUB_PR_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || usage
 [[ "$max_wait_seconds" =~ ^[0-9]+$ ]] || usage
+[[ "$event_name" == pull_request || "$event_name" == workflow_dispatch ]] || usage
 
-artifact_name="node-subtensor-release-$GITHUB_SHA"
+artifact_sha="$GITHUB_SHA"
+if [[ "$event_name" == workflow_dispatch ]]; then
+  pulls=$(gh api \
+    -H 'Accept: application/vnd.github+json' \
+    "repos/$GITHUB_REPOSITORY/commits/$GITHUB_PR_HEAD_SHA/pulls")
+  candidates=$(jq -cer \
+    --arg repository_id "$GITHUB_REPOSITORY_ID" \
+    --arg head_sha "$GITHUB_PR_HEAD_SHA" '
+      [.[]
+        | select(.state == "open")
+        | select(.head.sha == $head_sha)
+        | select((.head.repo.id | tostring) == $repository_id)
+        | select((.base.repo.id | tostring) == $repository_id)
+        | select(.merge_commit_sha | type == "string")
+        | select(.merge_commit_sha | test("^[0-9a-f]{40}$"))]
+    ' <<< "$pulls")
+  candidate_count=$(jq -er 'length' <<< "$candidates")
+  if [[ "$candidate_count" != 1 ]]; then
+    echo "workflow_dispatch requires exactly one open same-repository PR for $GITHUB_PR_HEAD_SHA; found $candidate_count" >&2
+    exit 1
+  fi
+  artifact_sha=$(jq -er '.[0].merge_commit_sha' <<< "$candidates")
+fi
+
+artifact_name="node-subtensor-release-$artifact_sha"
 workflow_path=.github/workflows/runtime-checks.yml
 started=$(date -u +%s)
 
@@ -68,6 +94,8 @@ while true; do
       {
         echo "found=true"
         echo "artifact_id=$artifact_id"
+        echo "artifact_name=$artifact_name"
+        echo "artifact_sha=$artifact_sha"
         echo "digest=$digest"
         echo "size=$size"
         echo "run_id=$run_id"
@@ -97,7 +125,7 @@ while true; do
   elapsed=$(($(date -u +%s) - started))
   if (( elapsed >= max_wait_seconds )); then
     write_miss
-    echo "No exact-commit TypeScript release artifact appeared after ${elapsed}s; using the local build fallback."
+    echo "No exact-merge Runtime Checks release artifact appeared after ${elapsed}s."
     exit 0
   fi
   remaining=$((max_wait_seconds - elapsed))

.github/scripts/test-clone-epoch-soak-workflow.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)
+workflow="$repo_root/.github/workflows/clone-epoch-soak.yml"
+soak_script="$repo_root/clones/scripts/run-clone-epoch-soak.sh"
+monitor_script="$repo_root/clones/scripts/run-clone-block-monitor.sh"
+supervisor_script="$repo_root/clones/scripts/clone-process-supervision.sh"
+package="$repo_root/clones/js-tests/package.json"
+epoch_script="$repo_root/clones/js-tests/scripts/run-clone-epoch-soak.ts"
+
+ruby -e 'require "yaml"; YAML.parse_file(ARGV.fetch(0))' "$workflow"
+bash -n "$soak_script"
+bash -n "$monitor_script"
+bash -n "$supervisor_script"
+
+grep -Fq 'workflow_dispatch:' "$workflow"
+grep -Fq 'types: [labeled, synchronize]' "$workflow"
+grep -Fq "github.event.label.name == 'run-clone-epoch-soak'" "$workflow"
+grep -Fq 'github.event.pull_request.head.repo.id || github.repository_id' "$workflow"
+grep -Fq 'github.event.pull_request.head.ref || github.ref_name' "$workflow"
+grep -Fq 'cancel-in-progress: true' "$workflow"
+if grep -Fq 'cancel-in-progress: false' "$workflow"; then
+  echo "clone epoch soak must cancel superseded branch runs" >&2
+  exit 1
+fi
+if grep -Eq 'uses: actions/(checkout|setup-node|upload-artifact)@v[0-9]+' "$workflow"; then
+  echo "clone epoch soak actions must be pinned to full commit SHAs" >&2
+  exit 1
+fi
+grep -Fq 'github.event.pull_request.head.repo.fork == false' "$workflow"
+if grep -Eq '^[[:space:]]*(push|schedule):' "$workflow"; then
+  echo "clone epoch soak must remain manually triggered" >&2
+  exit 1
+fi
+grep -Fq 'default: "2"' "$workflow"
+dispatch=$(sed -n '/^  workflow_dispatch:/,/^concurrency:/p' "$workflow")
+[[ $(grep -Ec '^          - "[123]"$' <<< "$dispatch") -eq 3 ]]
+[[ $(grep -Ec '^      (epoch_cycles|fresh_state):$' <<< "$dispatch") -eq 2 ]]
+grep -Fq 'type: boolean' <<< "$dispatch"
+grep -Fq 'default: false' <<< "$dispatch"
+grep -Fq 'runs-on: ubuntu-latest' "$workflow"
+grep -Fq 'runs-on: [self-hosted, fireactions-turbo-8]' "$workflow"
+if grep -Fq 'fireactions-validatorbench' "$workflow"; then
+  echo "clone epoch soak must use the turbo-8 runner" >&2
+  exit 1
+fi
+grep -Fq 'select-shared-release-artifact.sh "$GITHUB_OUTPUT" 1200' "$workflow"
+grep -Fq 'needs.select-node-release.outputs.artifact_name' "$workflow"
+grep -Fq 'EXPECTED_RELEASE_SHA: ${{ needs.select-node-release.outputs.artifact_sha }}' "$workflow"
+grep -Fq 'needs.select-node-release.outputs.digest' "$workflow"
+if grep -Fq 'cargo build --release -p node-subtensor' "$workflow"; then
+  echo "clone epoch soak must reuse the exact Runtime Checks release artifact" >&2
+  exit 1
+fi
+grep -Fq 'timeout-minutes: 150' "$workflow"
+grep -Fq 'deadline_epoch_ms: ${{ steps.deadline.outputs.deadline_epoch_ms }}' "$workflow"
+grep -Fq 'DEADLINE_EPOCH_MS: ${{ needs.select-node-release.outputs.deadline_epoch_ms }}' "$workflow"
+grep -Fq 'retention-days: 14' "$workflow"
+if grep -Fq 'clone-node.log.gz' "$workflow"; then
+  echo "the raw node log must not be retained as a soak artifact" >&2
+  exit 1
+fi
+grep -Fq 'inputs.fresh_state != true' "$workflow"
+grep -Fq 'snapshot-artifact.sh select' "$workflow"
+grep -Fq 'run-clone-epoch-soak.sh "${{ inputs.epoch_cycles || '\''2'\'' }}"' "$workflow"
+grep -Fq 'SOAK_CHECKPOINT=' "$workflow"
+grep -Fq 'clone-block-diagnostics-*.log' "$workflow"
+
+grep -Fq 'start-local-clone-and-wait.sh" accelerated' "$soak_script"
+grep -Fq 'run-clone-block-monitor.sh" "$policy" "$label"' "$soak_script"
+grep -Fq 'minimum_post_upgrade_blocks=${MINIMUM_POST_UPGRADE_BLOCKS:-7200}' "$soak_script"
+if grep -Fq -- '--migration' "$soak_script"; then
+  echo "epoch soak must not permit bypassing its migration gate" >&2
+  exit 1
+fi
+grep -Fq 'waitForBetaBasketV2ReleaseReadiness(api' "$epoch_script"
+if grep -Fq 'getFinalizedHead' "$epoch_script"; then
+  echo "single-node epoch coverage must use the same best-head state as readiness monitoring" >&2
+  exit 1
+fi
+jq -e '.scripts["monitor:block-latency"] and .scripts["wait:beta-basket-v2-readiness"] and .scripts["soak:epochs"] and .scripts["test:clone-performance"]' \
+  "$package" >/dev/null
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p \
+  "$tmp/repo/clones/scripts" \
+  "$tmp/repo/clones/js-tests/temp" \
+  "$tmp/bin"
+cp "$soak_script" "$supervisor_script" "$tmp/repo/clones/scripts/"
+
+cat > "$tmp/repo/clones/scripts/run-clone-block-monitor.sh" <<'EOF'
+#!/usr/bin/env bash
+printf 'monitor start %s\n' "$*" >> "$HARNESS_LOG"
+if [[ "${MOCK_MONITOR_FAIL:-false}" == true ]]; then
+  exit 1
+fi
+[[ -z "${CLONE_MONITOR_READY_FILE:-}" ]] || printf '{}\n' > "$CLONE_MONITOR_READY_FILE"
+trap 'printf "monitor stop\n" >> "$HARNESS_LOG"; exit 0' TERM INT
+while true; do
+  /bin/sleep 0.02
+done
+EOF
+cat > "$tmp/repo/clones/scripts/start-local-clone-and-wait.sh" <<'EOF'
+#!/usr/bin/env bash
+repo_root=$(cd -- "$(dirname -- "$0")/../.." && pwd)
+printf 'start %s\n' "$*" >> "$HARNESS_LOG"
+: > "$repo_root/clone-node.log"
+EOF
+cat > "$tmp/repo/clones/scripts/stop-local-clone.sh" <<'EOF'
+#!/usr/bin/env bash
+printf 'stop\n' >> "$HARNESS_LOG"
+EOF
+cat > "$tmp/repo/clones/scripts/local-clone-checkpoint.sh" <<'EOF'
+#!/usr/bin/env bash
+printf 'checkpoint %s\n' "$*" >> "$HARNESS_LOG"
+EOF
+cat > "$tmp/bin/npm" <<'EOF'
+#!/usr/bin/env bash
+printf 'npm %s\n' "$*" >> "$HARNESS_LOG"
+if [[ "$*" == *"soak:epochs"* ]]; then
+  exit "${MOCK_EPOCH_STATUS:-0}"
+fi
+if [[ "$*" == *"runtime:update:alice"* ]]; then
+  previous=
+  for argument in "$@"; do
+    if [[ "$previous" == --report ]]; then
+      printf '{"upgradeBlock":25,"finalizedAtEpochMs":123456}\n' > "$argument"
+    fi
+    previous=$argument
+  done
+fi
+EOF
+chmod +x "$tmp/repo/clones/scripts/"*.sh "$tmp/bin/npm"
+
+export PATH="$tmp/bin:$PATH"
+export HARNESS_LOG="$tmp/harness.log"
+assert_before() {
+  local first=$1 second=$2 first_line second_line
+  first_line=$(grep -Fn -- "$first" "$HARNESS_LOG" | head -n 1 | cut -d: -f1)
+  second_line=$(grep -Fn -- "$second" "$HARNESS_LOG" | head -n 1 | cut -d: -f1)
+  [[ -n "$first_line" && -n "$second_line" && "$first_line" -lt "$second_line" ]]
+}
+deadline=$(( $(date +%s) * 1000 + 60000 ))
+checkpoint="$tmp/checkpoint.tar.gz"
+: > "$checkpoint"
+SOAK_CHECKPOINT="$checkpoint" SOAK_DEADLINE_EPOCH_MS="$deadline" \
+  "$tmp/repo/clones/scripts/run-clone-epoch-soak.sh" 2
+[[ $(grep -Fc 'start accelerated' "$HARNESS_LOG") -eq 2 ]]
+grep -Fq 'monitor start baseline baseline ' "$HARNESS_LOG"
+grep -Fq 'monitor start collect soak ' "$HARNESS_LOG"
+[[ $(grep -Fc 'npm run soak:epochs -- --epoch-cycles 2' "$HARNESS_LOG") -eq 2 ]]
+grep -Fq 'checkpoint restore' "$HARNESS_LOG"
+grep -Fq -- '--release-gate beta-basket-v2 --upgrade-block 25 --minimum-post-upgrade-blocks 7200' "$HARNESS_LOG"
+assert_before 'monitor start baseline baseline' '--release-gate none'
+assert_before 'monitor start collect soak' 'npm run runtime:update:alice'
+assert_before 'npm run runtime:update:alice' '--release-gate beta-basket-v2'
+grep -Fq 'stop' "$HARNESS_LOG"
+
+: > "$HARNESS_LOG"
+if MOCK_EPOCH_STATUS=1 SOAK_CHECKPOINT="$checkpoint" SOAK_DEADLINE_EPOCH_MS="$deadline" \
+  "$tmp/repo/clones/scripts/run-clone-epoch-soak.sh" 2; then
+  echo "failed epoch coverage unexpectedly succeeded" >&2
+  exit 1
+fi
+grep -Fq 'stop' "$HARNESS_LOG"
+
+: > "$HARNESS_LOG"
+if MOCK_MONITOR_FAIL=true SOAK_CHECKPOINT="$checkpoint" SOAK_DEADLINE_EPOCH_MS="$deadline" \
+  "$tmp/repo/clones/scripts/run-clone-epoch-soak.sh" 2; then
+  echo "failed soak block monitor unexpectedly succeeded" >&2
+  exit 1
+fi
+grep -Fq 'stop' "$HARNESS_LOG"
+
+echo "clone epoch soak workflow contract tests passed"