Enhance build process and caching for crypto dependencies

.github/actions/build-crypto-deps/action.yml

@@ -0,0 +1,44 @@
+name: Build crypto deps
+description: Cache and build Harmony mcl/bls dependencies.
+
+inputs:
+  harmony-path:
+    description: Path to the harmony checkout.
+    required: true
+  mcl-path:
+    description: Path to the mcl checkout.
+    required: true
+  bls-path:
+    description: Path to the bls checkout.
+    required: true
+  jobs:
+    description: Parallel make jobs.
+    required: false
+    default: "4"
+  static:
+    description: Build static BLS outputs.
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    - name: Cache crypto dependency build outputs
+      uses: actions/cache@v4
+      with:
+        path: |
+          ${{ inputs['mcl-path'] }}/bin
+          ${{ inputs['mcl-path'] }}/lib
+          ${{ inputs['bls-path'] }}/bin
+          ${{ inputs['bls-path'] }}/lib
+        key: ${{ runner.os }}-${{ runner.arch }}-crypto-deps-static-${{ inputs.static }}-${{ hashFiles(format('{0}/**', inputs['mcl-path']), format('{0}/**', inputs['bls-path'])) }}
+
+    - name: Build crypto deps
+      shell: bash
+      working-directory: ${{ inputs['harmony-path'] }}
+      run: |
+        if [[ "${{ inputs.static }}" == "true" ]]; then
+          JOBS="${{ inputs.jobs }}" make deps-static
+        else
+          JOBS="${{ inputs.jobs }}" make deps
+        fi

.github/workflows/ci-pr.yaml

@@ -67,6 +67,15 @@ jobs:
           go-version: '1.24.2'
           cache-dependency-path: go/src/github.com/harmony-one/harmony/go.sum
 
+      - &build-crypto-deps
+        name: Build crypto deps
+        uses: ./go/src/github.com/harmony-one/harmony/.github/actions/build-crypto-deps
+        with:
+          harmony-path: go/src/github.com/harmony-one/harmony
+          mcl-path: go/src/github.com/harmony-one/mcl
+          bls-path: go/src/github.com/harmony-one/bls
+          jobs: "4"
+
       - name: Build harmony
         run: make linux_static
         working-directory: go/src/github.com/harmony-one/harmony
@@ -95,12 +104,7 @@ jobs:
       - *checkout-mcl
       - *checkout-bls
       - *setup-go
-      - name: Build mcl
-        run: make -j4
-        working-directory: go/src/github.com/harmony-one/mcl
-      - name: Build bls
-        run: make BLS_SWAP_G=1 -j4
-        working-directory: go/src/github.com/harmony-one/bls
+      - *build-crypto-deps
       - name: Install tools
         run: |
           go install golang.org/x/tools/cmd/goimports@v0.30.0

.github/workflows/ci-release.yaml

@@ -116,6 +116,15 @@ jobs:
           # Release artifacts should not restore mutable CI/PR caches.
           cache: false
 
+      - name: Build crypto deps
+        uses: ./harmony/.github/actions/build-crypto-deps
+        with:
+          harmony-path: harmony
+          mcl-path: mcl
+          bls-path: bls
+          jobs: "4"
+          static: "true"
+
       - name: Build harmony binary for Linux
         id: build-binary
         run: |
@@ -134,7 +143,7 @@ jobs:
 
           BINARY_NAME="harmony-${ARCH}"
 
-          make linux_static
+          make linux_static_quick
           mv ./bin/harmony "./bin/${BINARY_NAME}"
 
           file "./bin/${BINARY_NAME}"

Makefile

@@ -14,13 +14,16 @@ SHELL := bash
 EPOCH_TO_WAIT ?=5
 EXTRA_NODES_FILE ?="./test/configs/local-extra-nodes.txt"
 
-.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log
+.PHONY: all help deps deps-static libs build exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log
 
 all: libs
 	bash ./scripts/go_executable_build.sh -S
 
 help:
 	@echo "all - build the harmony binary & bootnode along with the MCL & BLS libs (if necessary)"
+	@echo "deps - build only the MCL & BLS libs through the shared dependency bootstrap"
+	@echo "deps-static - build only the static MCL & BLS libs through the shared dependency bootstrap"
+	@echo "build - build the harmony binary & bootnode after dependency bootstrap"
 	@echo "libs - build only the MCL & BLS libs (if necessary)"
 	@echo "exe - build the harmony binary & bootnode"
 	@echo "race - build the harmony binary & bootnode with race condition checks"
@@ -69,9 +72,15 @@ help:
 	@echo "debug-multi-bls-multi-ext-node - start a localnet with multiple external nodes and multi-BLS configuration"
 	@echo "protofiles - generate Go code from protobuf files"
 
-libs:
-	make -C $(TOP)/mcl -j8
-	make -C $(TOP)/bls BLS_SWAP_G=1 -j8
+deps:
+	bash ./scripts/build_crypto_deps.sh
+
+deps-static:
+	STATIC_BLS=true bash ./scripts/build_crypto_deps.sh
+
+libs: deps
+
+build: deps exe
 
 exe:
 	bash ./scripts/go_executable_build.sh -S
@@ -148,8 +157,7 @@ distclean: clean
 	make -C $(TOP)/bls clean
 
 go-get:
-	source ./scripts/setup_bls_build_flags.sh
-	go get -v ./...
+	source ./scripts/setup_bls_build_flags.sh && go get -v ./...
 
 test:
 	bash ./test/all.sh
@@ -170,8 +178,7 @@ test-rosetta-attach:
 	bash ./test/rosetta.sh attach
 
 linux_static:
-	make -C $(TOP)/mcl -j8
-	make -C $(TOP)/bls minimised_static BLS_SWAP_G=1 -j8
+	STATIC_BLS=true bash ./scripts/build_crypto_deps.sh
 	bash ./scripts/go_executable_build.sh -s
 
 linux_static_quick:
@@ -271,4 +278,4 @@ debug-delete-log:
 
 
 docker-go-test:
-	docker run --rm -it -v "$PWD":/go/src/github.com/harmony-one/harmony frozen621/harmony-test bash -c 'make go-test'
+	docker run --rm -it -v "$PWD":/go/src/github.com/harmony-one/harmony frozen621/harmony-test bash -c 'make go-test'

scripts/build_crypto_deps.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+# shellcheck source=scripts/setup_bls_build_flags.sh
+. "${ROOT}/scripts/setup_bls_build_flags.sh"
+
+jobs="${JOBS:-}"
+if [[ -z "${jobs}" ]]; then
+	if command -v nproc >/dev/null 2>&1; then
+		jobs=$(nproc)
+	elif command -v sysctl >/dev/null 2>&1; then
+		jobs=$(sysctl -n hw.ncpu)
+	else
+		jobs=4
+	fi
+fi
+
+if [[ ! -d "${MCL_DIR}" ]]; then
+	echo "missing mcl repository: ${MCL_DIR}" >&2
+	exit 1
+fi
+
+if [[ ! -d "${BLS_DIR}" ]]; then
+	echo "missing bls repository: ${BLS_DIR}" >&2
+	exit 1
+fi
+
+echo "building mcl (${MCL_DIR}) with -j${jobs}"
+make -C "${MCL_DIR}" -j"${jobs}"
+
+if [[ "${STATIC_BLS:-false}" == "true" ]]; then
+	echo "building bls minimised_static (${BLS_DIR}) with -j${jobs}"
+	make -C "${BLS_DIR}" minimised_static BLS_SWAP_G=1 -j"${jobs}"
+else
+	echo "building bls (${BLS_DIR}) with -j${jobs}"
+	make -C "${BLS_DIR}" BLS_SWAP_G=1 -j"${jobs}"
+fi

scripts/build_harmony_test_image.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+main_repo_branch=${1:?missing main repo branch}
+main_repo_org=${2:?missing main repo org}
+image_tag=${3:-harmonyone/localnet-test}
+
+cache_scope=${HARMONY_TEST_DOCKER_CACHE_SCOPE:-harmony-test-localnet}
+
+if [[ "${HARMONY_TEST_DOCKER_CACHE:-false}" == "true" ]]; then
+	echo "Building ${image_tag} with Docker Buildx cache scope: ${cache_scope}"
+	docker buildx build \
+		--build-arg MAIN_REPO_BRANCH="${main_repo_branch}" \
+		--build-arg MAIN_REPO_ORG="${main_repo_org}" \
+		--progress plain \
+		--cache-from "type=gha,scope=${cache_scope}" \
+		--cache-to "type=gha,mode=max,scope=${cache_scope}" \
+		--load \
+		-t "${image_tag}" .
+else
+	echo "Building ${image_tag} without Docker Buildx cache"
+	docker build \
+		--build-arg MAIN_REPO_BRANCH="${main_repo_branch}" \
+		--build-arg MAIN_REPO_ORG="${main_repo_org}" \
+		--progress plain \
+		-t "${image_tag}" .
+fi

scripts/run_harmony_test_image.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+test_mode=${1:?missing localnet test mode, expected -n, -p, or -r}
+image_tag=${HARMONY_TEST_IMAGE:-harmonyone/localnet-test}
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+echo "Pulling ${image_tag}"
+docker pull "${image_tag}"
+
+echo "Running ${image_tag} ${test_mode}"
+docker run \
+	-v "${root}:/go/src/github.com/harmony-one/harmony" \
+	"${image_tag}" "${test_mode}"

scripts/setup_bls_build_flags.sh

@@ -36,7 +36,7 @@ case $OS in
       ;;
 esac
 
-if [ "$1" = "-v" ]; then
+if [ "${1:-}" = "-v" ]; then
    echo "{ \"CGO_CFLAGS\" : \"$CGO_CFLAGS\",
             \"CGO_LDFLAGS\" : \"$CGO_LDFLAGS\",
             \"LD_LIBRARY_PATH\" : \"$LD_LIBRARY_PATH\",

scripts/travis_pyhmy_checker.sh

@@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \
-    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test .
+"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}"
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -e HARMONY_PREBUILT_STATIC=true -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -p

scripts/travis_rosetta_checker.sh

@@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \
-    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test .
+"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}"
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -r

scripts/travis_rpc_checker.sh

@@ -45,7 +45,6 @@ git fetch origin "${TEST_REPO_BRANCH}"
 git checkout "${TEST_REPO_BRANCH}"
 git pull --rebase=true
 cd localnet
-docker build --build-arg MAIN_REPO_BRANCH="${MAIN_REPO_BRANCH}" --progress plain \
-    --build-arg MAIN_REPO_ORG="${MAIN_REPO_ORG}" -t harmonyone/localnet-test .
+"${DIR}/build_harmony_test_image.sh" "${MAIN_REPO_BRANCH}" "${MAIN_REPO_ORG}"
 # WARN: this is the place where LOCAL repository is provided to the harmony-tests repo
 docker run -v "$DIR/../:/go/src/github.com/harmony-one/harmony" harmonyone/localnet-test -B -n