chore: re-enable v5 backwards compatibility e2e CI

.github/ci3.sh

@@ -48,6 +48,13 @@ function check_cache {
   local cache_name="ci-success-${CI_MODE}-${tree_hash}.tar.gz"
   # Export for use by ci3_success.sh
   echo "CI_CACHE_NAME=$cache_name" >> $GITHUB_ENV
+  # Compat-gated runs must not read the marker: its key is (mode, tree hash), which encodes neither
+  # the historical version matrix nor whether the compat gate ran, so a marker from a non-gated run
+  # or from before a new stable tag was published would skip the gate.
+  if [ "${RUN_COMPAT_E2E:-0}" = 1 ]; then
+    echo "Compat-gated run: not reading the CI success cache (write still enabled)."
+    return
+  fi
   # Only whitelist some ci modes for cache.
   # E.g. we skip cache for release builds - they must always produce versioned images
   cached_ci_modes=(

.github/ci3_labels_to_env.sh

@@ -57,11 +57,6 @@ function main {
     echo "NO_FAIL_FAST=1" >> $GITHUB_ENV
   fi
 
-  # Handle skip-compat-e2e label (escape hatch for backwards compat test failures on release PRs)
-  if has_label "ci-skip-compat-e2e"; then
-    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
-  fi
-
   local chonk_input_update=0
   local chonk_input_update_requested=0
   if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && { has_label "ci-refresh-chonk" || head_commit_has_marker "--ci-refresh-chonk"; }; then
@@ -143,6 +138,10 @@ function main {
     ci_mode="barretenberg-full"
   elif has_label "ci-barretenberg" || [ "$target_branch" == "merge-train/barretenberg" ]; then
     ci_mode="barretenberg"
+  elif [ "$target_branch" == "v5-next" ]; then
+    # Runs targeting v5-next default to the full suite so the backwards-compat e2e gate applies to
+    # every ingress.
+    ci_mode="full"
   elif [[ "${GITHUB_REF:-}" == refs/tags/v* ]]; then
     # A pushed semver tag is a release; REF_NAME is the tag (see ci3/source_refname). In the private
     # repo this is the nightly path (nightly-release-tag*.yml push v<ver>-nightly.<date> tags on next and
@@ -155,6 +154,14 @@ function main {
   echo "CI_MODE=$ci_mode" >> $GITHUB_ENV
   echo "CI mode: $ci_mode"
 
+  # Runs targeting v5-next execute the backwards-compat e2e matrix after their normal suite.
+  # .github/ci3.sh (success-cache read) and bootstrap.sh (the compat hook) derive their behavior
+  # from this flag.
+  if [ "$target_branch" == "v5-next" ] &&
+     { [ "$ci_mode" == "full" ] || [ "$ci_mode" == "full-no-test-cache" ]; }; then
+    echo "RUN_COMPAT_E2E=1" >> $GITHUB_ENV
+  fi
+
   # Private-repo safety gate. The release flow can publish to DockerHub/npmjs/crates.io/github; that
   # MUST NEVER run in the private fork. So whenever this repo would release — for ANY trigger (a pushed
   # nightly tag, a ci-release-pr tag, anything future) — force the private path: publish only the docker
@@ -163,7 +170,9 @@ function main {
   if [ "$ci_mode" = "release" ] &&
      [ "$(printf '%s' "${GITHUB_REPOSITORY:-}" | tr 'A-Z' 'a-z')" = "aztecprotocol/aztec-packages-private" ]; then
     echo "PRIVATE_RELEASE=1" >> $GITHUB_ENV
-    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
+    # Private releases publish only to the internal Artifact Registry; the compat gate runs on the
+    # public release of the same tag.
+    echo "RUN_COMPAT_E2E=0" >> $GITHUB_ENV
   fi
 
   # Determine if benchmarks should be uploaded (merge-queue, full, or full-no-test-cache modes)

.github/workflows/ci3.yml

@@ -127,7 +127,7 @@ jobs:
           PR_NUMBER: ${{ github.event.pull_request.number }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
           GITHUB_ACTOR: ${{ github.actor }}
-          # Forwarded to EC2 so release_compat_e2e's nightly-failure Slack alert can build a real run URL.
+          # Forwarded to EC2 so compat_e2e's nightly-failure Slack alert can build a real run URL.
           RUN_ID: ${{ github.run_id }}
           CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
           CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}

bootstrap.sh

@@ -642,13 +642,16 @@ function private_release {
   done
 }
 
-function release_compat_e2e {
-  # Runs e2e tests with contract artifacts from every prior stable release since 4.2.0 (the version
+function compat_e2e {
+  # Runs e2e tests with contract artifacts from every prior stable release since 5.0.1 (the version
   # where we committed to backwards compatibility). Validates that old contract artifacts work on the
-  # current release. Blocking for stable/RC releases; observational (non-blocking) for nightlies.
-  # Set SKIP_COMPAT_E2E=1 to bypass (escape hatch via the ci-skip-compat-e2e label).
-  if [ "${SKIP_COMPAT_E2E:-0}" = "1" ]; then
-    echo "SKIP_COMPAT_E2E=1, skipping backwards compatibility e2e tests."
+  # current checkout.
+  # RUN_COMPAT_E2E is tri-state: unset runs (releases and manual invocations include the gate by
+  # default), 1 is the PR opt-in consumed by compat_e2e_if_requested, and 0 is the explicit
+  # opt-out (private releases via the ci3_labels_to_env.sh gate, local release dry-runs,
+  # emergencies).
+  if [ "${RUN_COMPAT_E2E:-1}" = 0 ]; then
+    echo "RUN_COMPAT_E2E=0, skipping backwards compatibility e2e tests."
     return 0
   fi
 
@@ -658,12 +661,10 @@ function release_compat_e2e {
     return 0
   fi
 
-  # TODO: bump when v5 commits to backwards-compatible contract artifacts.
   #   compat_major:       major version that has compat guarantees today.
-  #   compat_min_version: earliest stable tag of that major to test against
-  #                       (artifacts before this are incompatible due to oracle interface changes).
-  local compat_major="4"
-  local compat_min_version="4.2.0"
+  #   compat_min_version: earliest stable tag covered by that guarantee.
+  local compat_major="5"
+  local compat_min_version="5.0.1"
 
   local current_version major
   current_version=$(jq -r '."."' .release-please-manifest.json)
@@ -716,13 +717,13 @@ function release_compat_e2e {
     node yarn-project/end-to-end/src/install_legacy_contracts.cjs "$ver"
   done
 
-  # Build and run the compat test commands in an isolated subshell so the bespoke test settings
-  # (no test cache, no fast-fail short-circuit) don't leak into the release build/publish that follows.
-  # set -e re-enables errexit inside this subshell: the caller invokes release_compat_e2e with errexit
-  # disabled (to capture its exit code), so without this a failed build/install would be masked.
+  # Compat command cache keys include both the end-to-end build hash and CONTRACT_ARTIFACTS_VERSION,
+  # so successful results can only be reused for the same build inputs and historical artifacts.
+  # Keep the bespoke settings isolated from the surrounding run.
+  # set -e keeps failures fatal when the release caller temporarily disables errexit to capture them.
   (
     set -e
-    export USE_TEST_CACHE=0
+    export USE_TEST_CACHE=1
     export CI_FULL=0
     export NO_FAIL_FAST=1
     build
@@ -732,6 +733,14 @@ function release_compat_e2e {
   )
 }
 
+# PRs targeting v5-next run the compat matrix after their normal suite, on the same instance
+# (RUN_COMPAT_E2E is set by .github/ci3_labels_to_env.sh and forwarded by bootstrap_ec2).
+function compat_e2e_if_requested {
+  if [ "${RUN_COMPAT_E2E:-0}" = 1 ]; then
+    compat_e2e
+  fi
+}
+
 ### SELF TESTING #######################################################################################################
 function test_bootstrap_linux {
   local name=linux-bootstrap-test-ubuntu
@@ -831,13 +840,15 @@ case "$cmd" in
     export CI_FULL=1
     build_and_test full
     bench
+    compat_e2e_if_requested
     ;;
   "ci-full-no-test-cache")
     export CI=1
     export USE_TEST_CACHE=0
     export CI_FULL=1
     build_and_test full
     bench
+    compat_e2e_if_requested
     ;;
   "ci-chonk-input-update")
     export CI=1
@@ -1060,34 +1071,37 @@ case "$cmd" in
   ############
   # RELEASES #
   ############
+  "ci-compat-e2e")
+    # Standalone entrypoint for running the compat matrix on its own (manual runs and debugging);
+    # CI runs it via compat_e2e_if_requested at the end of the PR modes above.
+    export CI=1
+    compat_e2e
+    ;;
   "ci-release")
     # Single command that tests and publishes a release. Runs the backwards-compatibility e2e
-    # checks (blocking for stable/RC, observational for nightlies), then builds and publishes.
+    # checks before building and publishing.
     # DRY_RUN=1 exercises the whole flow without publishing — this is how releases are tested in CI.
     export CI=1
     export USE_TEST_CACHE=1
     if ! semver check $REF_NAME; then
       exit 1
     fi
 
-    # Backwards-compatibility e2e checks. A failure blocks stable/RC releases, but only warns on
-    # nightlies (where compat coverage is observational) so the nightly publish still proceeds.
-    # Toggle errexit explicitly rather than `release_compat_e2e || compat_rc=$?`: calling under `||`
+    # Backwards-compatibility e2e checks must pass before any release is published.
+    # Toggle errexit explicitly rather than `compat_e2e || compat_rc=$?`: calling under `||`
     # suspends errexit for the whole function (and its subshell), masking build/setup failures there.
     compat_rc=0
     set +e
-    release_compat_e2e
+    compat_e2e
     compat_rc=$?
     set -e
     if [ "$compat_rc" -ne 0 ]; then
       if [[ "${REF_NAME:-}" == *-nightly.* ]]; then
         run_url="https://github.com/${GITHUB_REPOSITORY:-AztecProtocol/aztec-packages}/actions/runs/${RUN_ID:-unknown}"
         "$ci3/slack_notify" "Backwards compatibility e2e tests FAILED on nightly tag <${run_url}|${REF_NAME}>" "#team-fairies" || true
-        echo "Compat e2e failed on nightly tag — continuing (non-blocking)."
-      else
-        echo "ERROR: backwards compatibility e2e tests failed — blocking release." >&2
-        exit 1
       fi
+      echo "ERROR: backwards compatibility e2e tests failed — blocking release." >&2
+      exit 1
     fi
 
     if [[ "$(semver prerelease $REF_NAME)" == private* ]]; then

ci.sh

@@ -131,7 +131,7 @@ case "$cmd" in
   full|full-no-test-cache)
     export CI_DASHBOARD="prs"
     export JOB_ID="x-$cmd"
-    export AWS_SHUTDOWN_TIME=75
+    export AWS_SHUTDOWN_TIME=90
     bootstrap_ec2 "./bootstrap.sh ci-$cmd"
     ;;
   chonk-input-update)

ci3/bootstrap_ec2

@@ -402,7 +402,7 @@ start_build() {
     -e DRY_RUN=${DRY_RUN:-} \
     -e PRIVATE_RELEASE=${PRIVATE_RELEASE:-} \
     -e INTERNAL_DOCKER_REGISTRY=${INTERNAL_DOCKER_REGISTRY:-} \
-    -e SKIP_COMPAT_E2E=${SKIP_COMPAT_E2E:-} \
+    -e RUN_COMPAT_E2E=${RUN_COMPAT_E2E:-} \
     --pids-limit=65536 \
     --shm-size=2g \
     --ulimit nofile=1048576:1048576 \