Repository navigation
Sync fork #1415
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - 'auto-cadence-upgrade/**' | |
| - staging | |
| - trying | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| - 'v[0-9]+.[0-9]+-rc' | |
| pull_request: | |
| branches: | |
| - master* | |
| - 'auto-cadence-upgrade/**' | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| - 'v[0-9]+.[0-9]+-rc' | |
| merge_group: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| env: | |
| GO_VERSION: "1.26" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-linter: | |
| name: Build Custom Linter | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Cache custom linter binary | |
| id: cache-linter | |
| uses: actions/cache@v5 | |
| with: | |
| # The cache is only a build accelerator: on a hit we skip rebuilding the custom linter | |
| # binary (which takes ~30 minutes). The binary is passed to the lint jobs via an | |
| # artifact (see below), so a failed cache save or restore only costs a rebuild on a | |
| # future run - it never fails the workflow. | |
| # Key changes whenever the implementation (tools/structwrite) or the compilation | |
| # config (.custom-gcl.yml) changes, causing a miss and a recompilation. | |
| key: custom-linter-${{ env.GO_VERSION }}-${{ runner.os }}-${{ hashFiles('.custom-gcl.yml', 'tools/structwrite/**') }} | |
| path: tools/custom-gcl # path defined in .custom-gcl.yml | |
| # The golangci-lint action detects .custom-gcl.yml and builds the custom linter binary. | |
| - name: Install golangci-lint and build custom linter binary | |
| if: steps.cache-linter.outputs.cache-hit != 'true' | |
| uses: golangci/golangci-lint-action@v9 | |
| with: | |
| install-only: true | |
| version: v2.7.1 | |
| # The artifact (not the cache) is how the binary reaches the lint jobs: artifacts are | |
| # guaranteed within a run (a failed upload fails this job), while cache saves are | |
| # best-effort and can fail silently, which previously hard-failed the lint jobs. | |
| - name: Upload custom linter binary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: custom-gcl | |
| path: tools/custom-gcl | |
| if-no-files-found: error | |
| # Allow "Re-run all jobs": the previous attempt's artifact still exists on the run, | |
| # and without this the upload would fail with a name conflict. | |
| overwrite: true | |
| golangci: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dir: [./, ./integration/, ./insecure/] | |
| name: Lint | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| needs: build-linter # must wait for custom linter binary to be available | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| # The build-linter job uploads the custom linter binary as an artifact, which is guaranteed | |
| # to exist within this run (unlike a cache entry, which is best-effort). | |
| - name: Download custom linter binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: custom-gcl | |
| path: tools/ | |
| - name: Run go generate | |
| run: go generate ./... | |
| working-directory: ${{ matrix.dir }} | |
| # The golangci-lint action has a configuration where it searches for a binary named | |
| # "golangci-lint" in the path rather than downloading. | |
| # Below we rename our binary to this expected canonical name, and add it to the path. | |
| - name: Rename custom linter binary | |
| run: mv ./tools/custom-gcl ./tools/golangci-lint | |
| - name: Make custom linter binary executable | |
| run: chmod +x ./tools/golangci-lint | |
| - name: Add custom linter binary to path | |
| run: echo "$(pwd)/tools" >> $GITHUB_PATH | |
| - name: Run golangci-lint | |
| uses: golangci/golangci-lint-action@v9 | |
| with: | |
| install-mode: 'none' # looks for binary in path rather than downloading | |
| args: "-v" | |
| working-directory: ${{ matrix.dir }} | |
| tidy: | |
| name: Tidy | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run tidy | |
| run: make tidy | |
| create-dynamic-test-matrix: | |
| name: Create Dynamic Test Matrix | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go | |
| create-insecure-dynamic-test-matrix: | |
| name: Create Dynamic Unit Test Insecure Package Matrix | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go -c insecure | |
| create-integration-dynamic-test-matrix: | |
| name: Create Dynamic Integration Test Package Matrix | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go -c integration | |
| unit-test: | |
| name: Unit Tests (${{ matrix.targets.name }}) | |
| needs: create-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: make install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v4 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v5 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| files: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| unit-test-insecure: | |
| name: Unit Tests Insecure (${{ matrix.targets.name }}) | |
| needs: create-insecure-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-insecure-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: make install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v4 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -C ./insecure -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v5 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| # the insecure module's Makefile writes its coverage profile to cover.out | |
| files: ./insecure/cover.out | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| docker-build: | |
| name: Docker Build | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| env: | |
| CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v4 | |
| with: | |
| username: ${{ vars.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: ${{ (github.event_name == 'merge_group' || (github.event.pull_request && (github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR'))) }} | |
| # this docker auth is exclusively for higher rate limits. continue unauthenticated if it fails | |
| continue-on-error: true | |
| - name: Docker build | |
| env: | |
| CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| run: make docker-native-build-flow docker-native-build-flow-corrupt | |
| - name: Save Docker images | |
| run: | | |
| docker save \ | |
| gcr.io/flow-container-registry/access:latest \ | |
| gcr.io/flow-container-registry/collection:latest \ | |
| gcr.io/flow-container-registry/consensus:latest \ | |
| gcr.io/flow-container-registry/execution:latest \ | |
| gcr.io/flow-container-registry/execution-ledger:latest \ | |
| gcr.io/flow-container-registry/ghost:latest \ | |
| gcr.io/flow-container-registry/observer:latest \ | |
| gcr.io/flow-container-registry/verification:latest \ | |
| gcr.io/flow-container-registry/access-corrupted:latest \ | |
| gcr.io/flow-container-registry/execution-corrupted:latest \ | |
| gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar | |
| - name: Cache Docker images | |
| uses: actions/cache@v5 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| integration-test-others: | |
| name: Integration Tests Others (${{ matrix.targets.name }}) | |
| needs: create-integration-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-integration-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: make install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v4 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -C ./integration -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v5 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| # the integration module's Makefile writes its coverage profile to cover.out | |
| files: ./integration/cover.out | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| integration-test: | |
| name: Integration Tests | |
| needs: docker-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Access Cohort1 Integration Tests | |
| make: make -C integration access-cohort1-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Access Cohort2 Integration Tests | |
| make: make -C integration access-cohort2-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Access Cohort3 Integration Tests | |
| make: make -C integration access-cohort3-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Access Cohort4 Integration Tests | |
| make: make -C integration access-cohort4-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed | |
| # - name: BFT (Framework) Integration Tests | |
| # make: make -C integration bft-framework-tests | |
| # runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: BFT (Protocol) Integration Tests | |
| make: make -C integration bft-protocol-tests | |
| runner: blacksmith-8vcpu-ubuntu-2404 | |
| - name: BFT (Gossipsub) Integration Tests | |
| make: make -C integration bft-gossipsub-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Collection Integration Tests | |
| make: make -C integration collection-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Consensus Integration Tests | |
| make: make -C integration consensus-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Epoch Cohort1 Integration Tests | |
| make: make -C integration epochs-cohort1-tests | |
| runner: blacksmith-8vcpu-ubuntu-2404 | |
| - name: Epoch Cohort2 Integration Tests | |
| make: make -C integration epochs-cohort2-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Execution Integration Tests | |
| make: make -C integration execution-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Ghost Integration Tests | |
| make: make -C integration ghost-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: MVP Integration Tests | |
| make: make -C integration mvp-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Network Integration Tests | |
| make: make -C integration network-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Verification Integration Tests | |
| make: make -C integration verification-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| - name: Upgrade Integration Tests | |
| make: make -C integration upgrades-tests | |
| runner: blacksmith-4vcpu-ubuntu-2404 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| - name: Setup private build environment | |
| if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} | |
| uses: ./actions/private-setup | |
| with: | |
| cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Load cached Docker images | |
| uses: actions/cache@v5 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the same cache key as the docker-build job | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| - name: Load Docker images | |
| run: docker load -i flow-docker-images.tar | |
| - name: Run tests (${{ matrix.name }}) | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| uses: nick-fields/retry@v4 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 ${{ matrix.make }} |