diff --git a/.github/actions/build-crypto-deps/action.yml b/.github/actions/build-crypto-deps/action.yml new file mode 100644 index 0000000000..2a8b7f3842 --- /dev/null +++ b/.github/actions/build-crypto-deps/action.yml @@ -0,0 +1,44 @@ +name: Build crypto deps +description: Cache and build Harmony mcl/bls dependencies. + +inputs: + harmony-path: + description: Path to the harmony checkout. + required: true + mcl-path: + description: Path to the mcl checkout. + required: true + bls-path: + description: Path to the bls checkout. + required: true + jobs: + description: Parallel make jobs. + required: false + default: "4" + static: + description: Build static BLS outputs. + required: false + default: "false" + +runs: + using: composite + steps: + - name: Cache crypto dependency build outputs + uses: actions/cache@v4 + with: + path: | + ${{ inputs['mcl-path'] }}/bin + ${{ inputs['mcl-path'] }}/lib + ${{ inputs['bls-path'] }}/bin + ${{ inputs['bls-path'] }}/lib + key: ${{ runner.os }}-${{ runner.arch }}-crypto-deps-static-${{ inputs.static }}-${{ hashFiles(format('{0}/**', inputs['mcl-path']), format('{0}/**', inputs['bls-path'])) }} + + - name: Build crypto deps + shell: bash + working-directory: ${{ inputs['harmony-path'] }} + run: | + if [[ "${{ inputs.static }}" == "true" ]]; then + JOBS="${{ inputs.jobs }}" make deps-static + else + JOBS="${{ inputs.jobs }}" make deps + fi diff --git a/.github/workflows/ci-pr.yaml b/.github/workflows/ci-pr.yaml index e6dec2104b..6bdc717447 100644 --- a/.github/workflows/ci-pr.yaml +++ b/.github/workflows/ci-pr.yaml @@ -67,6 +67,15 @@ jobs: go-version: '1.24.2' cache-dependency-path: go/src/github.com/harmony-one/harmony/go.sum + - &build-crypto-deps + name: Build crypto deps + uses: ./go/src/github.com/harmony-one/harmony/.github/actions/build-crypto-deps + with: + harmony-path: go/src/github.com/harmony-one/harmony + mcl-path: go/src/github.com/harmony-one/mcl + bls-path: go/src/github.com/harmony-one/bls + jobs: "4" + - name: Build harmony run: make linux_static working-directory: go/src/github.com/harmony-one/harmony @@ -95,12 +104,7 @@ jobs: - *checkout-mcl - *checkout-bls - *setup-go - - name: Build mcl - run: make -j4 - working-directory: go/src/github.com/harmony-one/mcl - - name: Build bls - run: make BLS_SWAP_G=1 -j4 - working-directory: go/src/github.com/harmony-one/bls + - *build-crypto-deps - name: Install tools run: | go install golang.org/x/tools/cmd/goimports@v0.30.0 diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index ba14129919..0bbf0246d1 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -116,6 +116,15 @@ jobs: # Release artifacts should not restore mutable CI/PR caches. cache: false + - name: Build crypto deps + uses: ./harmony/.github/actions/build-crypto-deps + with: + harmony-path: harmony + mcl-path: mcl + bls-path: bls + jobs: "4" + static: "true" + - name: Build harmony binary for Linux id: build-binary run: | @@ -134,7 +143,7 @@ jobs: BINARY_NAME="harmony-${ARCH}" - make linux_static + make linux_static_quick mv ./bin/harmony "./bin/${BINARY_NAME}" file "./bin/${BINARY_NAME}" diff --git a/Makefile b/Makefile index 844f6c5099..e2957f587d 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,16 @@ SHELL := bash EPOCH_TO_WAIT ?=5 EXTRA_NODES_FILE ?="./test/configs/local-extra-nodes.txt" -.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log +.PHONY: all help deps deps-static libs build exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log all: libs bash ./scripts/go_executable_build.sh -S help: @echo "all - build the harmony binary & bootnode along with the MCL & BLS libs (if necessary)" + @echo "deps - build only the MCL & BLS libs through the shared dependency bootstrap" + @echo "deps-static - build only the static MCL & BLS libs through the shared dependency bootstrap" + @echo "build - build the harmony binary & bootnode after dependency bootstrap" @echo "libs - build only the MCL & BLS libs (if necessary)" @echo "exe - build the harmony binary & bootnode" @echo "race - build the harmony binary & bootnode with race condition checks" @@ -69,9 +72,15 @@ help: @echo "debug-multi-bls-multi-ext-node - start a localnet with multiple external nodes and multi-BLS configuration" @echo "protofiles - generate Go code from protobuf files" -libs: - make -C $(TOP)/mcl -j8 - make -C $(TOP)/bls BLS_SWAP_G=1 -j8 +deps: + bash ./scripts/build_crypto_deps.sh + +deps-static: + STATIC_BLS=true bash ./scripts/build_crypto_deps.sh + +libs: deps + +build: deps exe exe: bash ./scripts/go_executable_build.sh -S @@ -148,8 +157,7 @@ distclean: clean make -C $(TOP)/bls clean go-get: - source ./scripts/setup_bls_build_flags.sh - go get -v ./... + source ./scripts/setup_bls_build_flags.sh && go get -v ./... test: bash ./test/all.sh @@ -170,8 +178,7 @@ test-rosetta-attach: bash ./test/rosetta.sh attach linux_static: - make -C $(TOP)/mcl -j8 - make -C $(TOP)/bls minimised_static BLS_SWAP_G=1 -j8 + STATIC_BLS=true bash ./scripts/build_crypto_deps.sh bash ./scripts/go_executable_build.sh -s linux_static_quick: @@ -271,4 +278,4 @@ debug-delete-log: docker-go-test: - docker run --rm -it -v "$PWD":/go/src/github.com/harmony-one/harmony frozen621/harmony-test bash -c 'make go-test' \ No newline at end of file + docker run --rm -it -v "$PWD":/go/src/github.com/harmony-one/harmony frozen621/harmony-test bash -c 'make go-test' diff --git a/scripts/build_crypto_deps.sh b/scripts/build_crypto_deps.sh new file mode 100755 index 0000000000..7b9589094c --- /dev/null +++ b/scripts/build_crypto_deps.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +# shellcheck source=scripts/setup_bls_build_flags.sh +. "${ROOT}/scripts/setup_bls_build_flags.sh" + +jobs="${JOBS:-}" +if [[ -z "${jobs}" ]]; then + if command -v nproc >/dev/null 2>&1; then + jobs=$(nproc) + elif command -v sysctl >/dev/null 2>&1; then + jobs=$(sysctl -n hw.ncpu) + else + jobs=4 + fi +fi + +if [[ ! -d "${MCL_DIR}" ]]; then + echo "missing mcl repository: ${MCL_DIR}" >&2 + exit 1 +fi + +if [[ ! -d "${BLS_DIR}" ]]; then + echo "missing bls repository: ${BLS_DIR}" >&2 + exit 1 +fi + +echo "building mcl (${MCL_DIR}) with -j${jobs}" +make -C "${MCL_DIR}" -j"${jobs}" + +if [[ "${STATIC_BLS:-false}" == "true" ]]; then + echo "building bls minimised_static (${BLS_DIR}) with -j${jobs}" + make -C "${BLS_DIR}" minimised_static BLS_SWAP_G=1 -j"${jobs}" +else + echo "building bls (${BLS_DIR}) with -j${jobs}" + make -C "${BLS_DIR}" BLS_SWAP_G=1 -j"${jobs}" +fi diff --git a/scripts/build_harmony_test_image.sh b/scripts/build_harmony_test_image.sh new file mode 100755 index 0000000000..e134e3a61c --- /dev/null +++ b/scripts/build_harmony_test_image.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail + +main_repo_branch=${1:?missing main repo branch} +main_repo_org=${2:?missing main repo org} +image_tag=${3:-harmonyone/localnet-test} + +cache_scope=${HARMONY_TEST_DOCKER_CACHE_SCOPE:-harmony-test-localnet} + +if [[ "${HARMONY_TEST_DOCKER_CACHE:-false}" == "true" ]]; then + echo "Building ${image_tag} with Docker Buildx cache scope: ${cache_scope}" + docker buildx build \ + --build-arg MAIN_REPO_BRANCH="${main_repo_branch}" \ + --build-arg MAIN_REPO_ORG="${main_repo_org}" \ + --progress plain \ + --cache-from "type=gha,scope=${cache_scope}" \ + --cache-to "type=gha,mode=max,scope=${cache_scope}" \ + --load \ + -t "${image_tag}" . +else + echo "Building ${image_tag} without Docker Buildx cache" + docker build \ + --build-arg MAIN_REPO_BRANCH="${main_repo_branch}" \ + --build-arg MAIN_REPO_ORG="${main_repo_org}" \ + --progress plain \ + -t "${image_tag}" . +fi diff --git a/scripts/run_harmony_test_image.sh b/scripts/run_harmony_test_image.sh new file mode 100755 index 0000000000..9c268dceef --- /dev/null +++ b/scripts/run_harmony_test_image.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +test_mode=${1:?missing localnet test mode, expected -n, -p, or -r} +image_tag=${HARMONY_TEST_IMAGE:-harmonyone/localnet-test} +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +echo "Pulling ${image_tag}" +docker pull "${image_tag}" + +echo "Running ${image_tag} ${test_mode}" +docker run \ + -v "${root}:/go/src/github.com/harmony-one/harmony" \ + "${image_tag}" "${test_mode}" diff --git a/scripts/setup_bls_build_flags.sh b/scripts/setup_bls_build_flags.sh index 92ee20579e..3ffe8355dc 100644 --- a/scripts/setup_bls_build_flags.sh +++ b/scripts/setup_bls_build_flags.sh @@ -36,7 +36,7 @@ case $OS in ;; esac -if [ "$1" = "-v" ]; then +if [ "${1:-}" = "-v" ]; then echo "{ \"CGO_CFLAGS\" : \"$CGO_CFLAGS\", \"CGO_LDFLAGS\" : \"$CGO_LDFLAGS\", \"LD_LIBRARY_PATH\" : \"$LD_LIBRARY_PATH\", diff --git a/scripts/travis_pyhmy_checker.sh b/scripts/travis_pyhmy_checker.sh index 557c82049d..c5b7d16ade 100755 --- a/scripts/travis_pyhmy_checker.sh +++ b/scripts/travis_pyhmy_checker.sh @@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}" git checkout "${TEST_REPO_BRANCH}" git pull --rebase=true cd localnet -docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \ - --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . +"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}" # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo docker run -e HARMONY_PREBUILT_STATIC=true -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -p diff --git a/scripts/travis_rosetta_checker.sh b/scripts/travis_rosetta_checker.sh index acf23609ab..8d9ee4b927 100755 --- a/scripts/travis_rosetta_checker.sh +++ b/scripts/travis_rosetta_checker.sh @@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}" git checkout "${TEST_REPO_BRANCH}" git pull --rebase=true cd localnet -docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \ - --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . +"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}" # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -r diff --git a/scripts/travis_rpc_checker.sh b/scripts/travis_rpc_checker.sh index 9fab9201eb..2dacb5db49 100755 --- a/scripts/travis_rpc_checker.sh +++ b/scripts/travis_rpc_checker.sh @@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}" git checkout "${TEST_REPO_BRANCH}" git pull --rebase=true cd localnet -docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \ - --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test . +"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}" # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -n