diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index fb73619ec13..21d4f1247ac 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -24,13 +24,13 @@ echo "Configuring pnpm store directory..." pnpm config set store-dir /tmp/.pnpm-store # Install the binary form of golangci-lint into the Go bin directory (on PATH in -# the dev container). Pinned version + checksums live in build/tools.mk. +# the dev container). Pinned version + checksums live in build/tools.yaml. echo "Installing golangci-lint..." GOLANGCI_LINT_INSTALL_DIR="$(go env GOPATH)/bin" make install-golangci-lint # Install the binary form of shellcheck into the Go bin directory (on PATH in the # dev container) so 'make lint-shell' works out of the box. Pinned version + -# checksums live in build/tools.mk. +# checksums live in build/tools.yaml. echo "Installing shellcheck..." SHELLCHECK_INSTALL_DIR="$(go env GOPATH)/bin" make install-shellcheck diff --git a/.github/actions/create-kind-cluster/action.yaml b/.github/actions/create-kind-cluster/action.yaml index 7b904a4dcf9..40314c85c9a 100644 --- a/.github/actions/create-kind-cluster/action.yaml +++ b/.github/actions/create-kind-cluster/action.yaml @@ -32,7 +32,7 @@ runs: using: composite steps: - name: Install KinD - # Installs the version pinned in build/tools.mk (KIND_VERSION) and adds the + # Installs the version pinned in build/tools.yaml (KIND_VERSION) and adds the # install dir to the job PATH, so the cluster-create steps below find kind. shell: bash run: make install-kind diff --git a/.github/scripts/update-tools-pr.sh b/.github/scripts/update-tools-pr.sh new file mode 100644 index 00000000000..5220e77ab84 --- /dev/null +++ b/.github/scripts/update-tools-pr.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# ------------------------------------------------------------ +# Copyright 2026 The Radius Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------ + +set -euo pipefail + +readonly REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +readonly PR_BASE="${PR_BASE:-main}" + +require_environment() { + local name + for name in GH_TOKEN GIT_USER_EMAIL GIT_USER_NAME PR_BRANCH PR_TITLE; do + if [[ -z "${!name:-}" ]]; then + echo "Error: ${name} is required" >&2 + return 1 + fi + done +} + +pull_request_body() { + cat <<'EOF' +This automated PR refreshes the pinned command-line tool versions and SHA-256 +checksums from the release sources declared in `build/tools.yaml`. + +`build/tools.generated.mk` is generated from the manifest and committed with +it. Bicep remains intentionally held at its compatibility-pinned version until +local `br:localhost` functional tests support a newer release. +EOF +} + +# The workflow checks out only the base branch, so there is no remote-tracking +# ref to lease against. Read the destination SHA from the remote instead; an +# empty value leases on the branch not existing yet. +push_branch() { + local expected + expected="$( + git ls-remote origin "refs/heads/${PR_BRANCH}" | awk '{print $1}' + )" + git push --force-with-lease="refs/heads/${PR_BRANCH}:${expected}" \ + origin "HEAD:${PR_BRANCH}" +} + +require_environment +cd "${REPO_ROOT}" + +git config user.name "${GIT_USER_NAME}" +git config user.email "${GIT_USER_EMAIL}" +git checkout -B "${PR_BRANCH}" +git add -A +git commit --signoff -m "${PR_TITLE}" +gh auth setup-git +push_branch + +existing_pr="$( + gh pr list --state open --head "${PR_BRANCH}" \ + --json number --jq '.[0].number' +)" +body="$(pull_request_body)" +if [[ -n "${existing_pr}" ]]; then + gh pr edit "${existing_pr}" --title "${PR_TITLE}" --body "${body}" + echo "Updated PR #${existing_pr}" +else + gh pr create --base "${PR_BASE}" --head "${PR_BRANCH}" \ + --title "${PR_TITLE}" --body "${body}" +fi diff --git a/.github/scripts/update-tools-pr_test.sh b/.github/scripts/update-tools-pr_test.sh new file mode 100644 index 00000000000..4fe83e9e2db --- /dev/null +++ b/.github/scripts/update-tools-pr_test.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# ------------------------------------------------------------ +# Copyright 2026 The Radius Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +TEST_ROOT="$(mktemp -d)" +readonly TEST_ROOT +trap 'rm -rf "${TEST_ROOT}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local actual="$1" + local expected="$2" + [[ "${actual}" == *"${expected}"* ]] || + fail "expected '${actual}' to contain '${expected}'" +} + +readonly REPOSITORY="${TEST_ROOT}/repository" +readonly REMOTE="${TEST_ROOT}/remote.git" +readonly SEED="${TEST_ROOT}/seed" +readonly FAKE_BIN="${TEST_ROOT}/bin" +readonly GH_LOG_PATH="${TEST_ROOT}/gh.log" + +git init -q --bare "${REMOTE}" +git init -q -b main "${SEED}" +git -C "${SEED}" config user.name "Test User" +git -C "${SEED}" config user.email "test@example.com" +# A contributor's global commit.gpgsign makes git commit block on a signing +# prompt. DCO --signoff is unrelated and still applies. +git -C "${SEED}" config commit.gpgsign false +echo "original" >"${SEED}/tools.yaml" +git -C "${SEED}" add tools.yaml +git -C "${SEED}" commit -q -m "initial" +git -C "${SEED}" remote add origin "${REMOTE}" +git -C "${SEED}" push -q origin main + +# The workflow checks out the base branch only, so the automation branch never +# gains a remote-tracking ref. Mirror that here or the pushes below lease +# against a ref that CI does not have. +git clone -q --single-branch --branch main "${REMOTE}" "${REPOSITORY}" +# The script under test commits here, so it needs the same protection. +git -C "${REPOSITORY}" config commit.gpgsign false + +mkdir -p "${FAKE_BIN}" +cat >"${FAKE_BIN}/gh" <<'EOF' +#!/bin/bash +set -euo pipefail +printf '%s\n' "$*" >>"${GH_LOG}" +if [[ "$1 $2" == "pr list" ]]; then + printf '%s\n' "${EXISTING_PR:-}" +fi +EOF +chmod +x "${FAKE_BIN}/gh" + +run_script() { + local existing_pr="$1" + PATH="${FAKE_BIN}:${PATH}" \ + REPO_ROOT="${REPOSITORY}" \ + GH_LOG="${GH_LOG_PATH}" \ + EXISTING_PR="${existing_pr}" \ + GH_TOKEN="test-token" \ + GIT_USER_EMAIL="automation@example.com" \ + GIT_USER_NAME="Automation Bot" \ + PR_BRANCH="automation/update-tools" \ + PR_TITLE="chore: update pinned tool versions" \ + bash "${SCRIPT_DIR}/update-tools-pr.sh" >/dev/null +} + +echo "first update" >>"${REPOSITORY}/tools.yaml" +run_script "" + +commit_message="$( + git --git-dir="${REMOTE}" log -1 --format=%B \ + refs/heads/automation/update-tools +)" +assert_contains "${commit_message}" "chore: update pinned tool versions" +assert_contains \ + "${commit_message}" \ + "Signed-off-by: Automation Bot " +assert_contains "$(cat "${GH_LOG_PATH}")" "pr create" + +if git -C "${REPOSITORY}" show-ref --quiet \ + refs/remotes/origin/automation/update-tools; then + fail "fixture drifted from CI: the automation branch is now tracked" +fi + +before="$( + git --git-dir="${REMOTE}" rev-parse refs/heads/automation/update-tools +)" +git -C "${REPOSITORY}" checkout -q main +echo "second update" >>"${REPOSITORY}/tools.yaml" +run_script "42" +after="$( + git --git-dir="${REMOTE}" rev-parse refs/heads/automation/update-tools +)" +assert_contains "$(cat "${GH_LOG_PATH}")" "pr edit 42" +[[ "${before}" != "${after}" ]] || + fail "expected the remote automation branch to be updated" + +echo "update-tools-pr tests passed" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b993a574e24..e205badcc88 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -159,7 +159,7 @@ jobs: - name: Install oras if: needs.changes.outputs.only_changed != 'true' - # Version pinned in build/tools.mk (ORAS_VERSION). + # Version pinned in build/tools.yaml (ORAS_VERSION). run: make install-oras - name: Push latest rad cli binary to GHCR (unix-like) @@ -328,7 +328,7 @@ jobs: persist-credentials: false - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm - name: Setup Python diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index b9becbb5861..5418b583ccf 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -26,7 +26,7 @@ # functional-test-noncloud.yaml). Language runtime versions are read from the # repo's version files (go.mod, .node-version, .python-version, .terraform-version) # so a version bump flows here automatically; CLI tool versions and checksums are -# pinned in build/tools.mk and installed via the shared `make install-` +# pinned in build/tools.yaml and installed via the shared `make install-` # targets. Versions are NOT declared in a workflow-level `env:` # block because the Copilot coding agent replays these steps in a harness that # does not provide that context. @@ -41,7 +41,10 @@ on: - .node-version - .python-version - .terraform-version + - build/tools.yaml - build/tools.mk + - cmd/tool-updater/** + - internal/tooling/** - build/scripts/install-*.sh pull_request: paths: @@ -49,7 +52,10 @@ on: - .node-version - .python-version - .terraform-version + - build/tools.yaml - build/tools.mk + - cmd/tool-updater/** + - internal/tooling/** - build/scripts/install-*.sh permissions: {} @@ -64,7 +70,7 @@ permissions: {} # See: https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/customize-the-agent-environment # Language runtime versions read from repo version files (go.mod, .node-version, # .python-version, .terraform-version) still work because the steps read the files -# directly; CLI tool versions come from build/tools.mk via `make install-`. +# directly; CLI tool versions come from build/tools.yaml via `make install-`. jobs: # The job MUST be named `copilot-setup-steps` for the Copilot coding agent to @@ -103,11 +109,11 @@ jobs: python-version-file: .python-version - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm - name: Install kubectl - # Version pinned in build/tools.mk (KUBECTL_VERSION). + # Version pinned in build/tools.yaml (KUBECTL_VERSION). run: make install-kubectl - name: Install pnpm and TypeSpec tooling @@ -136,14 +142,14 @@ jobs: cspell --version - name: Install golangci-lint - # Pinned version + checksums live in build/tools.mk (GOLANGCI_LINT_VERSION). + # Pinned version + checksums live in build/tools.yaml (GOLANGCI_LINT_VERSION). run: make install-golangci-lint - name: Install Delve (dlv) # controller-gen, mockgen, and gotestsum are managed as Go tool dependencies # (the 'tool' directives in go.mod) and run via `go tool`, so they need no # separate install. dlv (Delve) is used by the process-debugging workflow - # (make debug-*). Version pinned in build/tools.mk (DLV_VERSION). + # (make debug-*). Version pinned in build/tools.yaml (DLV_VERSION). run: make install-dlv - name: Install yq @@ -152,31 +158,31 @@ jobs: - name: Install KinD # Lets the agent create a local Kubernetes cluster for integration and - # functional tests. Version pinned in build/tools.mk (KIND_VERSION). + # functional tests. Version pinned in build/tools.yaml (KIND_VERSION). run: make install-kind - name: Install k3d # Used by the local debug environment (make debug-start) to create a - # lightweight Kubernetes cluster. Version pinned in build/tools.mk (K3D_VERSION). + # lightweight Kubernetes cluster. Version pinned in build/tools.yaml (K3D_VERSION). run: make install-k3d - name: Install stern # Multi-pod log tailing for debugging Kubernetes output across the - # radius-system components. Version pinned in build/tools.mk (STERN_VERSION). + # radius-system components. Version pinned in build/tools.yaml (STERN_VERSION). run: make install-stern - name: Install Dapr CLI - # Version pinned in build/tools.mk (DAPR_VERSION). + # Version pinned in build/tools.yaml (DAPR_VERSION). run: make install-dapr - name: Install Terraform # Version defaults to .terraform-version (overridable via TERRAFORM_VERSION) - # and is compiled into the rad binary; checksums pinned in build/tools.mk. + # and is compiled into the rad binary; checksums pinned in build/tools.yaml. run: make install-terraform - name: Install Bicep CLI # Required by Bicep type publishing (make publish-bicep-extension) and - # rad bicep flows. Pinned version + checksum live in build/tools.mk. + # rad bicep flows. Pinned version + checksum live in build/tools.yaml. run: make install-bicep - name: Install PostgreSQL client diff --git a/.github/workflows/functional-test-cloud.yaml b/.github/workflows/functional-test-cloud.yaml index 7e90407006a..b318f42a851 100644 --- a/.github/workflows/functional-test-cloud.yaml +++ b/.github/workflows/functional-test-cloud.yaml @@ -507,7 +507,7 @@ jobs: make generate-bicep-types VERSION="${BICEP_TYPES_VERSION}" - name: Install bicep CLI - # Pinned version + checksum live in build/tools.mk (previously installed + # Pinned version + checksum live in build/tools.yaml (previously installed # the unpinned 'latest', which could pull v0.43+). run: make install-bicep @@ -745,7 +745,7 @@ jobs: AZURE_SUBSCRIPTIONID_TESTS: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm # The role-to-assume is the role that the github action will assume to execute aws commands and @@ -758,7 +758,7 @@ jobs: aws-region: ${{ env.AWS_REGION }} - name: Install KinD - # Installs the version pinned in build/tools.mk (KIND_VERSION) and adds + # Installs the version pinned in build/tools.yaml (KIND_VERSION) and adds # the install dir to the job PATH for the cluster-create step below. run: make install-kind diff --git a/.github/workflows/functional-test-noncloud.yaml b/.github/workflows/functional-test-noncloud.yaml index 6999a8c6600..6a1becb40a7 100644 --- a/.github/workflows/functional-test-noncloud.yaml +++ b/.github/workflows/functional-test-noncloud.yaml @@ -315,7 +315,7 @@ jobs: registry-port: ${{ env.LOCAL_REGISTRY_PORT }} - name: Install bicep CLI - # Pinned version + checksum live in build/tools.mk. + # Pinned version + checksum live in build/tools.yaml. run: make install-bicep - name: Publish bicep types @@ -366,7 +366,7 @@ jobs: rad version - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm - name: Create a KinD cluster with a local registry @@ -583,7 +583,7 @@ jobs: - name: Install Dapr CLI if: matrix.name == 'daprrp-noncloud' - # Version pinned in build/tools.mk (DAPR_VERSION). + # Version pinned in build/tools.yaml (DAPR_VERSION). run: make install-dapr - name: Install Dapr control plane diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c552ce55490..ef1d18d5b2e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -83,15 +83,15 @@ jobs: run: make install-yq - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm - name: Install golangci-lint - # Pinned version + checksums live in build/tools.mk (GOLANGCI_LINT_VERSION). + # Pinned version + checksums live in build/tools.yaml (GOLANGCI_LINT_VERSION). run: make install-golangci-lint - name: Install shellcheck - # Pinned version + checksums live in build/tools.mk (SHELLCHECK_VERSION). + # Pinned version + checksums live in build/tools.yaml (SHELLCHECK_VERSION). run: make install-shellcheck # Run the independent, read-only linters concurrently. Each parallel step is diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index bf50a79c5f6..0e27c715e39 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -67,7 +67,7 @@ jobs: # Helm is required because `make test` depends on the `test-helm` target, # which installs the helm-unittest plugin and runs the chart unit tests. - name: Install helm - # Version pinned in build/tools.mk (HELM_VERSION). + # Version pinned in build/tools.yaml (HELM_VERSION). run: make install-helm - name: Run make test (unit tests) diff --git a/.github/workflows/update-tools.yaml b/.github/workflows/update-tools.yaml new file mode 100644 index 00000000000..2bcd4f596e4 --- /dev/null +++ b/.github/workflows/update-tools.yaml @@ -0,0 +1,100 @@ +# yaml-language-server: $schema=https://www.schemastore.org/github-workflow.json +--- +name: update-tools + +on: + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + +concurrency: + group: update-tools + cancel-in-progress: false + +permissions: {} + +jobs: + update: + if: >- + github.repository == 'radius-project/radius' || + github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + env: + PR_BRANCH: automation/update-tools + PR_TITLE: "chore: update pinned tool versions" + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + cache: true + + - name: Check and update tool metadata + env: + GITHUB_TOKEN: ${{ github.token }} + run: make update-tools + + - name: Detect changes + id: changes + shell: bash + run: | + set -euo pipefail + if [ -n "$(git status --porcelain)" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + git status --short + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Tool metadata is current." + fi + + - name: Generate Dependabot-manager App token + if: >- + steps.changes.outputs.changed == 'true' && + github.repository == 'radius-project/radius' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.DEPENDABOT_MANAGER_BOT_CLIENT_ID }} + private-key: ${{ secrets.DEPENDABOT_MANAGER_BOT_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + radius + permission-contents: write + permission-pull-requests: write + + - name: Get Dependabot-manager bot details + if: >- + steps.changes.outputs.changed == 'true' && + github.repository == 'radius-project/radius' + id: bot-details + uses: raven-actions/bot-details@ee8966a9ff6e7e42cbfc4a56b4ddb60a9d1b40a6 # v1.2.0 + with: + bot-slug-name: ${{ steps.app-token.outputs.app-slug }} + + - name: Commit updates and create or update pull request + if: >- + steps.changes.outputs.changed == 'true' && + github.repository == 'radius-project/radius' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + GIT_USER_EMAIL: ${{ steps.bot-details.outputs.email }} + GIT_USER_NAME: ${{ steps.bot-details.outputs.name }} + run: make workflow-update-tools-pr + + - name: Report fork changes + if: >- + steps.changes.outputs.changed == 'true' && + github.repository != 'radius-project/radius' + run: | + echo "Tool updates were found." + echo "PR creation is skipped outside radius-project/radius because" + echo "the Dependabot-manager App secrets are unavailable to forks." diff --git a/.github/workflows/validate-bicep.yaml b/.github/workflows/validate-bicep.yaml index 4fd0d417d4b..ffe08704df6 100644 --- a/.github/workflows/validate-bicep.yaml +++ b/.github/workflows/validate-bicep.yaml @@ -77,7 +77,7 @@ jobs: - name: Install bicep CLI # Install into ~/.rad/bin so the rad CLI finds it; the script also adds # the dir to PATH for subsequent steps. Pinned version + checksum live in - # build/tools.mk. + # build/tools.yaml. run: make install-bicep BICEP_INSTALL_DIR="$HOME/.rad/bin" - name: Create a temporary file system @@ -101,7 +101,7 @@ jobs: go run ./cmd/rad/main.go bicep publish-extension -f ./test/functional-portable/dynamicrp/noncloud/resources/testdata/testresourcetypes.yaml --target ./tmp-testresources-bicep-extension/testresources.tgz --force - name: Install jq - # Pinned version + checksums live in build/tools.mk (JQ_VERSION). + # Pinned version + checksums live in build/tools.yaml (JQ_VERSION). run: make install-jq - name: Modify bicepconfig.json diff --git a/Makefile b/Makefile index 726390e1793..7a4963f6b86 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,4 @@ ARROW := \033[34;1m=>\033[0m # order matters for these -include build/help.mk build/version.mk build/build.mk build/generate.mk build/test.mk build/docker.mk build/artifacts.mk build/recipes.mk build/install.mk build/db.mk build/lint.mk build/debug.mk build/workflow.mk build/resource-types.mk build/tools.mk +include build/tools.mk build/help.mk build/version.mk build/build.mk build/generate.mk build/test.mk build/docker.mk build/artifacts.mk build/recipes.mk build/install.mk build/db.mk build/lint.mk build/debug.mk build/workflow.mk build/resource-types.mk diff --git a/build/build.mk b/build/build.mk index 47dadd5e4ae..3da0f91edb2 100644 --- a/build/build.mk +++ b/build/build.mk @@ -49,12 +49,8 @@ else endif # Linker flags: https://cmake.org/cmake/help/latest/envvar/LDFLAGS.html. -# TERRAFORM_VERSION defaults to the .terraform-version file (the canonical source, -# matching the .node-version / .python-version convention) but can be overridden, -# e.g. `make build TERRAFORM_VERSION=1.15.0`. It is embedded into the rad binary so -# the recipe engine downloads a matching Terraform, and is consumed by the -# install-terraform target in build/tools.mk. -TERRAFORM_VERSION ?= $(shell cat .terraform-version) +# TERRAFORM_VERSION is loaded from the generated tool metadata include before this +# file is parsed. It can still be overridden, e.g. `make build TERRAFORM_VERSION=1.15.0`. LDFLAGS := "-s -w -X $(BASE_PACKAGE_NAME)/pkg/version.channel=$(REL_CHANNEL) -X $(BASE_PACKAGE_NAME)/pkg/version.release=$(REL_VERSION) -X $(BASE_PACKAGE_NAME)/pkg/version.commit=$(GIT_COMMIT) -X $(BASE_PACKAGE_NAME)/pkg/version.version=$(GIT_VERSION) -X $(BASE_PACKAGE_NAME)/pkg/version.chartVersion=$(CHART_VERSION) -X $(BASE_PACKAGE_NAME)/pkg/recipes/terraform.terraformVersion=$(TERRAFORM_VERSION)" # Combination of flags into GOARGS. diff --git a/build/generate.mk b/build/generate.mk index f35af74511f..8f5f7d97cf3 100644 --- a/build/generate.mk +++ b/build/generate.mk @@ -17,7 +17,6 @@ ##@ Generate (Code and Schema Generation) GOOS ?= $(shell go env GOOS) -CONTROLLER_TOOLS_VERSION ?= v0.12.0 ifeq ($(GOOS),windows) CMD_EXT = .cmd diff --git a/build/scripts/install-bicep.sh b/build/scripts/install-bicep.sh index cae4ce0e58f..97133d926a9 100755 --- a/build/scripts/install-bicep.sh +++ b/build/scripts/install-bicep.sh @@ -8,7 +8,7 @@ set -euo pipefail # later steps can run bicep. # # The pinned version and per-platform SHA-256 checksums are normally provided by -# build/tools.mk through the environment. The script is generic, so when a value +# build/tools.yaml through the generated Make include. The script is generic, so when a value # is not supplied it is resolved at runtime: # * empty BICEP_VERSION -> the latest published release # * missing checksum for platform -> install without verification (a warning is @@ -168,7 +168,7 @@ main() { || fail "could not download ${asset} ${version}" # Azure/bicep does not publish checksums, so verification only happens when a - # pinned checksum is supplied (the common case via build/tools.mk). + # pinned checksum is supplied (the common case via build/tools.yaml). if [ -n "$checksum" ]; then verify_checksum "$checksum" "${WORKDIR}/bicep" else diff --git a/build/scripts/install-dapr.sh b/build/scripts/install-dapr.sh index 78a90dd2c8a..c9cac5119a9 100755 --- a/build/scripts/install-dapr.sh +++ b/build/scripts/install-dapr.sh @@ -9,7 +9,7 @@ set -euo pipefail # # The Dapr CLI is published as a per-platform tarball on dapr/cli's GitHub # releases. The pinned version and per-platform SHA-256 checksums (of the -# tarball) are normally provided by build/tools.mk through the environment. The +# tarball) are normally provided by build/tools.yaml through the generated Make include. The # script is generic, so when a value is not supplied it is resolved at runtime: # * empty DAPR_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own diff --git a/build/scripts/install-dlv.sh b/build/scripts/install-dlv.sh index aa1c5cfbf48..4c982770025 100755 --- a/build/scripts/install-dlv.sh +++ b/build/scripts/install-dlv.sh @@ -11,7 +11,7 @@ set -euo pipefail # GitHub Actions the Go bin dir is added to the job PATH so later steps can run # dlv. # -# The pinned version is normally provided by build/tools.mk through the +# The pinned version is normally provided by build/tools.yaml through the # environment. When it is not supplied it defaults to the latest release. # # Usage: install-dlv.sh [install_dir] diff --git a/build/scripts/install-golangci-lint.sh b/build/scripts/install-golangci-lint.sh index bc4fe1bcd18..333cf9511a6 100755 --- a/build/scripts/install-golangci-lint.sh +++ b/build/scripts/install-golangci-lint.sh @@ -9,7 +9,7 @@ set -euo pipefail # # golangci-lint is published as a per-platform tarball on golangci/golangci-lint's # GitHub releases. The pinned version and per-platform SHA-256 checksums (of the -# tarball) are normally provided by build/tools.mk through the environment. The +# tarball) are normally provided by build/tools.yaml through the generated Make include. The # script is generic, so when a value is not supplied it is resolved at runtime: # * empty GOLANGCI_LINT_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own diff --git a/build/scripts/install-helm.sh b/build/scripts/install-helm.sh index fc3889fa7b4..22ea2af1dae 100755 --- a/build/scripts/install-helm.sh +++ b/build/scripts/install-helm.sh @@ -9,7 +9,7 @@ set -euo pipefail # # Helm is published as a per-platform tarball on the Helm release CDN # (get.helm.sh), not GitHub. The pinned version and per-platform SHA-256 checksums -# (of the tarball) are normally provided by build/tools.mk through the +# (of the tarball) are normally provided by build/tools.yaml through the # environment. The script is generic, so when a value is not supplied it is # resolved at runtime: # * empty HELM_VERSION -> the latest published release diff --git a/build/scripts/install-jq.sh b/build/scripts/install-jq.sh index c7b07dbe34d..35228f58ad0 100755 --- a/build/scripts/install-jq.sh +++ b/build/scripts/install-jq.sh @@ -10,7 +10,7 @@ set -euo pipefail # jq is published as a single per-platform binary on jqlang/jq's GitHub releases, # whose release tags are 'jq-' (e.g. jq-1.8.2) and whose darwin assets # are named 'macos'. The pinned version and per-platform SHA-256 checksums are -# normally provided by build/tools.mk through the environment. The script is +# normally provided by build/tools.yaml through the generated Make include. The script is # generic, so when a value is not supplied it is resolved at runtime: # * empty JQ_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own 'sha256sum.txt' diff --git a/build/scripts/install-k3d.sh b/build/scripts/install-k3d.sh index 09b3d778202..40b0cfae06d 100755 --- a/build/scripts/install-k3d.sh +++ b/build/scripts/install-k3d.sh @@ -9,7 +9,7 @@ set -euo pipefail # # k3d is published as a per-platform single binary on k3d-io/k3d's GitHub # releases. The pinned version and per-platform SHA-256 checksums are normally -# provided by build/tools.mk through the environment. The script is generic, so +# provided by build/tools.yaml through the generated Make include. The script is generic, so # when a value is not supplied it is resolved at runtime: # * empty K3D_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own combined diff --git a/build/scripts/install-kind.sh b/build/scripts/install-kind.sh index c1b9317cc21..2870d0cfe99 100755 --- a/build/scripts/install-kind.sh +++ b/build/scripts/install-kind.sh @@ -8,7 +8,7 @@ set -euo pipefail # install dir is added to the job PATH so later steps can run kind. # # The pinned version and per-platform SHA-256 checksums are normally provided by -# build/tools.mk through the environment. The script is generic, so when a value +# build/tools.yaml through the generated Make include. The script is generic, so when a value # is not supplied it is resolved at runtime: # * empty KIND_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own diff --git a/build/scripts/install-kubectl.sh b/build/scripts/install-kubectl.sh index 5913c80fa05..e0c0263e5ee 100755 --- a/build/scripts/install-kubectl.sh +++ b/build/scripts/install-kubectl.sh @@ -9,7 +9,7 @@ set -euo pipefail # # kubectl is published on the Kubernetes release CDN (dl.k8s.io), not GitHub. The # pinned version and per-platform SHA-256 checksums are normally provided by -# build/tools.mk through the environment. The script is generic, so when a value +# build/tools.yaml through the generated Make include. The script is generic, so when a value # is not supplied it is resolved at runtime: # * empty KUBECTL_VERSION -> the latest stable release (stable.txt) # * missing checksum for platform -> read from the release's own diff --git a/build/scripts/install-oras.sh b/build/scripts/install-oras.sh index 4ffc9a8b71d..204dfe39423 100755 --- a/build/scripts/install-oras.sh +++ b/build/scripts/install-oras.sh @@ -9,7 +9,7 @@ set -euo pipefail # # ORAS is published as a per-platform tarball on oras-project/oras's GitHub # releases. The pinned version and per-platform SHA-256 checksums (of the -# tarball) are normally provided by build/tools.mk through the environment. The +# tarball) are normally provided by build/tools.yaml through the generated Make include. The # script is generic, so when a value is not supplied it is resolved at runtime: # * empty ORAS_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own diff --git a/build/scripts/install-shellcheck.sh b/build/scripts/install-shellcheck.sh index b5b63921aa9..eb819e0ed38 100755 --- a/build/scripts/install-shellcheck.sh +++ b/build/scripts/install-shellcheck.sh @@ -9,7 +9,7 @@ set -euo pipefail # # ShellCheck is published as a per-platform '.tar.xz' archive on # koalaman/shellcheck's GitHub releases. The pinned version and per-platform -# SHA-256 checksums (of the archive) are normally provided by build/tools.mk +# SHA-256 checksums (of the archive) are normally provided by build/tools.yaml # through the environment. The script is generic, so when a value is not supplied # it is resolved at runtime: # * empty SHELLCHECK_VERSION -> the latest published release diff --git a/build/scripts/install-stern.sh b/build/scripts/install-stern.sh index 0432b506cf6..1e3a5fa1971 100755 --- a/build/scripts/install-stern.sh +++ b/build/scripts/install-stern.sh @@ -9,7 +9,7 @@ set -euo pipefail # # stern is published as a per-platform tarball on stern/stern's GitHub releases. # The pinned version and per-platform SHA-256 checksums (of the tarball) are -# normally provided by build/tools.mk through the environment. The script is +# normally provided by build/tools.yaml through the generated Make include. The script is # generic, so when a value is not supplied it is resolved at runtime: # * empty STERN_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own combined diff --git a/build/scripts/install-terraform.sh b/build/scripts/install-terraform.sh index 94af96a41fa..4607c3072ba 100755 --- a/build/scripts/install-terraform.sh +++ b/build/scripts/install-terraform.sh @@ -9,7 +9,7 @@ set -euo pipefail # # Terraform is published as a per-platform zip on the HashiCorp release CDN # (releases.hashicorp.com), not GitHub. The pinned version and per-platform -# SHA-256 checksums (of the zip) are normally provided by build/tools.mk through +# SHA-256 checksums (of the zip) are normally provided by build/tools.yaml through # the environment. The script is generic, so when a value is not supplied it is # resolved at runtime: # * empty TERRAFORM_VERSION -> the latest published release diff --git a/build/scripts/install-yq.sh b/build/scripts/install-yq.sh index 9196fa928e7..9e0681428f7 100755 --- a/build/scripts/install-yq.sh +++ b/build/scripts/install-yq.sh @@ -8,7 +8,7 @@ set -euo pipefail # PATH so later steps can run yq. # # The pinned version and per-platform SHA-256 checksums are normally provided by -# build/tools.mk through the environment. The script is generic, so when a value +# build/tools.yaml through the generated Make include. The script is generic, so when a value # is not supplied it is resolved at runtime: # * empty YQ_VERSION -> the latest published release # * missing checksum for platform -> read from the release's own checksums file diff --git a/build/test.mk b/build/test.mk index d75feed4a39..1cf3aef4225 100644 --- a/build/test.mk +++ b/build/test.mk @@ -53,13 +53,17 @@ GOTEST_OPTS ?= GOTEST_TOOL ?= go tool gotestsum $(GOTESTSUM_OPTS) -- .PHONY: test -test: test-get-envtools test-helm test-manage-radius-installation ## Runs unit tests, excluding kubernetes controller tests +test: test-get-envtools test-helm test-manage-radius-installation test-update-tools-pr ## Runs unit tests, excluding kubernetes controller tests KUBEBUILDER_ASSETS="$(shell $(ENV_SETUP) use -p path ${K8S_VERSION} --arch amd64)" CGO_ENABLED=1 $(GOTEST_TOOL) ./pkg/... $(GOTEST_OPTS) .PHONY: test-manage-radius-installation test-manage-radius-installation: ## Tests Radius installation lifecycle reconciliation @bash ./.github/scripts/manage-radius-installation_test.sh +.PHONY: test-update-tools-pr +test-update-tools-pr: ## Tests the automated tool-update pull request workflow + @bash ./.github/scripts/update-tools-pr_test.sh + .PHONY: test-compile test-compile: test-get-envtools ## Compiles all tests without running them @echo "$(ARROW) Compiling unit tests..." diff --git a/build/tools.generated.mk b/build/tools.generated.mk new file mode 100644 index 00000000000..fd9ab90ac56 --- /dev/null +++ b/build/tools.generated.mk @@ -0,0 +1,82 @@ +# Code generated by cmd/tool-updater; DO NOT EDIT. + +YQ_VERSION ?= v4.53.3 +YQ_CHECKSUM_LINUX_AMD64 ?= fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4 +YQ_CHECKSUM_LINUX_ARM64 ?= 578648e463a11c1b6db6010cbf41eafed6bee79466fcffa1bb446672cf7945ea +YQ_CHECKSUM_DARWIN_AMD64 ?= b4ba1ecce3c47f00803f4f964de38394326c7a32eb6540616e04fb2935a0f08d +YQ_CHECKSUM_DARWIN_ARM64 ?= 877de31753a4dd2401aa048937aa9a7fc4d5f6ce858cf31508c5802954297213 + +BICEP_VERSION ?= v0.42.1 +BICEP_CHECKSUM_LINUX_AMD64 ?= aed90eb2c69a6ee2bd70dc0d4354408ac4d04fd9911d3ec8e0cd74ad173e7139 +BICEP_CHECKSUM_LINUX_ARM64 ?= b01ac3bb5259096dfbe548138a538d1c4e4a55e6f87f3827e2299fbc2d4e6796 +BICEP_CHECKSUM_DARWIN_AMD64 ?= 8219bfd0601a514cc0a814b4b194aed588f4efa68b7c7ac7c9b64f3d84713dd7 +BICEP_CHECKSUM_DARWIN_ARM64 ?= 1c66533af4d4d47f875623d88074d28ca7fe7e9dc1f783a62570e8724700aca1 + +KIND_VERSION ?= v0.32.0 +KIND_CHECKSUM_LINUX_AMD64 ?= 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 +KIND_CHECKSUM_LINUX_ARM64 ?= b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d +KIND_CHECKSUM_DARWIN_AMD64 ?= 295ac6d0d634c9819c9907df45e3017d1f13166bd13c3404c45e79f7faa47498 +KIND_CHECKSUM_DARWIN_ARM64 ?= dca67911095a110c2b5c36e26df6cac860c602033e456c0db47be498cdef1ebb + +KUBECTL_VERSION ?= v1.36.2 +KUBECTL_CHECKSUM_LINUX_AMD64 ?= 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 +KUBECTL_CHECKSUM_LINUX_ARM64 ?= c957eb8c4bea27a3bb35b269edd9082e27f027f7b76b20b5bf4afebc726c6d3e +KUBECTL_CHECKSUM_DARWIN_AMD64 ?= ce6c5e55cd17559e87e4fb5e73ebbbc2511bcf2b695d7a40c1b1461a9817d4b3 +KUBECTL_CHECKSUM_DARWIN_ARM64 ?= 4408c85c83fd3a31adaa555bdf3c7a6c81f74b19449a9060ba31ab91926f023d + +DAPR_VERSION ?= v1.15.2 +DAPR_CHECKSUM_LINUX_AMD64 ?= 09328bc0e4353036b824c2ec9cf7cabf4d75b4fc00ca02d80ae3e4374ee27eda +DAPR_CHECKSUM_LINUX_ARM64 ?= b49244701a191c1e843211383703be9f2cd086a1db259c9789672f7e4e82ad55 +DAPR_CHECKSUM_DARWIN_AMD64 ?= 42a36e667559aef0fb6357fbe8f0fdbf1a6d9ea0ba8484c32e90ea61ddf15ba0 +DAPR_CHECKSUM_DARWIN_ARM64 ?= 176f455ea1961cdb59ab0e9ec3e4900b877576a9a2178d3b4b2619bfe947643f + +HELM_VERSION ?= v4.2.2 +HELM_CHECKSUM_LINUX_AMD64 ?= 9adafecab4d406853bba163a70e9f104f47dbbf65ce24b7653bae7e36150bcb6 +HELM_CHECKSUM_LINUX_ARM64 ?= 78803142087a0069fa4b50d3f32a84d3ef25c14d1ee8a40fbccf86a6216d2f36 +HELM_CHECKSUM_DARWIN_AMD64 ?= 10c1e36ee8c5f2e2ee25a16599cb03ab74c0953cd889cacb980a49ba4b6574ba +HELM_CHECKSUM_DARWIN_ARM64 ?= 5410a0dae3d5d91f45653b161260d9301aabc4ae80ae50a6605d66884b6df8ea + +K3D_VERSION ?= v5.9.0 +K3D_CHECKSUM_LINUX_AMD64 ?= 06d8f25bc3a971c4eb29e0ff08429b180402db0f4dec838c9eac427e296800a0 +K3D_CHECKSUM_LINUX_ARM64 ?= 03cde5cf23e6e8e67de5a039ecf26e5b85aca82fba3e5d13dadf904cd218a250 +K3D_CHECKSUM_DARWIN_AMD64 ?= b4aabc37534f95b9c764e7823f2df923f50d57600837aa60a06266cce47db732 +K3D_CHECKSUM_DARWIN_ARM64 ?= fe106541d5d0a3f18debcd4d432a16f8c0ce3e6ddc06f8fbb6f696a122313e00 + +GOLANGCI_LINT_VERSION ?= v2.12.2 +GOLANGCI_LINT_CHECKSUM_LINUX_AMD64 ?= 8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 +GOLANGCI_LINT_CHECKSUM_LINUX_ARM64 ?= 44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a +GOLANGCI_LINT_CHECKSUM_DARWIN_AMD64 ?= f6f06d94b6241521c53d15450c5209b028270bf966f842afb11c030c79f5bc16 +GOLANGCI_LINT_CHECKSUM_DARWIN_ARM64 ?= a9c54498731b3128f79e090be6110f3e5fffccc617b08142ed244d4126c73f29 + +TERRAFORM_VERSION ?= 1.14.9 +TERRAFORM_CHECKSUM_LINUX_AMD64 ?= 2e5cffc20a0b48a67a76268723bd5a10b8666f69b2aa4f04906e206726bedd63 +TERRAFORM_CHECKSUM_LINUX_ARM64 ?= 863002085b886453795d9ff4b8989b8468784478150b70ba8a1df3e3ad66da99 +TERRAFORM_CHECKSUM_DARWIN_AMD64 ?= c15326e1af102d2767d40208a0157d1402057f80192991f56803b66457304cf3 +TERRAFORM_CHECKSUM_DARWIN_ARM64 ?= 5bc0b11b7a63c8984a41d82523356df46f7833c2e9651a39a7f8919422de5cde + +STERN_VERSION ?= v1.34.0 +STERN_CHECKSUM_LINUX_AMD64 ?= 7754adfa653939240f7d20fff4ada9b69cda40c9e70732301f67bb8045f1ef3e +STERN_CHECKSUM_LINUX_ARM64 ?= e215cfc5e42d71e93b77d3fac8f0df7d736271f44b2d92a5b417eaa588edff3a +STERN_CHECKSUM_DARWIN_AMD64 ?= 153355317f21e565ea10bc710d4c2e3d98fd06f83cae5eb927e7031cc724a7a6 +STERN_CHECKSUM_DARWIN_ARM64 ?= 4014d84096e1e603ee115864e03a1e15fb9bae9876647bf7bb8031eee278dcd3 + +ORAS_VERSION ?= v1.3.2 +ORAS_CHECKSUM_LINUX_AMD64 ?= 9229ccc6d17bb282039ad4a69abb16dcb887a5bce567c075d731d9b3c7ad8eaf +ORAS_CHECKSUM_LINUX_ARM64 ?= 8db4a223bd6034deff198e791ea7cb3af0840df25b7e9f370e2f1f3fd20d389b +ORAS_CHECKSUM_DARWIN_AMD64 ?= 2621f6b252b222f6fbf4e114d2fcaa0cec6b632624ffaf73143f66e4e0994f86 +ORAS_CHECKSUM_DARWIN_ARM64 ?= 7929f792cf272268412375ecad6f0fb3c20f164368d5b57966e67ad6d36eca53 + +SHELLCHECK_VERSION ?= v0.11.0 +SHELLCHECK_CHECKSUM_LINUX_AMD64 ?= 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 +SHELLCHECK_CHECKSUM_LINUX_ARM64 ?= 12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588 +SHELLCHECK_CHECKSUM_DARWIN_AMD64 ?= 3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6 +SHELLCHECK_CHECKSUM_DARWIN_ARM64 ?= 56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79 + +JQ_VERSION ?= 1.8.2 +JQ_CHECKSUM_LINUX_AMD64 ?= b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f +JQ_CHECKSUM_LINUX_ARM64 ?= 8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309 +JQ_CHECKSUM_DARWIN_AMD64 ?= e94b266e3c26690550006abe63152b782280f4e14374accdf04cbde844f00bc0 +JQ_CHECKSUM_DARWIN_ARM64 ?= 2d75340ba57a4b4b4c8708a21c2dc8e958a48aaa8bba13b27f77f6e4c0eca07e + +DLV_VERSION ?= v1.27.0 + diff --git a/build/tools.mk b/build/tools.mk index 3b5fbf65b23..5f060432e6e 100644 --- a/build/tools.mk +++ b/build/tools.mk @@ -16,15 +16,25 @@ ##@ Tools -# yq - pinned version and per-platform SHA-256 checksums consumed by -# build/scripts/install-yq.sh. The script is generic: clear YQ_VERSION to install -# the latest release, and clear a checksum to have it read from the release's own -# published checksums file. Keep the checksums in sync when bumping YQ_VERSION. -YQ_VERSION ?= v4.53.3 -YQ_CHECKSUM_LINUX_AMD64 ?= fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4 -YQ_CHECKSUM_LINUX_ARM64 ?= 578648e463a11c1b6db6010cbf41eafed6bee79466fcffa1bb446672cf7945ea -YQ_CHECKSUM_DARWIN_AMD64 ?= b4ba1ecce3c47f00803f4f964de38394326c7a32eb6540616e04fb2935a0f08d -YQ_CHECKSUM_DARWIN_ARM64 ?= 877de31753a4dd2401aa048937aa9a7fc4d5f6ce858cf31508c5802954297213 +TOOL_MANIFEST := build/tools.yaml +TOOL_MAKE_INCLUDE := build/tools.generated.mk +TOOL_UPDATER_GOOS := $(shell go env GOHOSTOS) +TOOL_UPDATER_GOARCH := $(shell go env GOHOSTARCH) +TOOL_UPDATER_BINARY := bin/tool-updater$(if $(filter windows,$(TOOL_UPDATER_GOOS)),.exe,) +TOOL_UPDATER_SOURCES := $(wildcard cmd/tool-updater/*.go) $(wildcard internal/tooling/*.go) + +include $(TOOL_MAKE_INCLUDE) + +$(TOOL_UPDATER_BINARY): $(TOOL_UPDATER_SOURCES) go.mod go.sum + @mkdir -p bin + @GOOS="$(TOOL_UPDATER_GOOS)" GOARCH="$(TOOL_UPDATER_GOARCH)" go build -o "$@" ./cmd/tool-updater + +$(TOOL_MAKE_INCLUDE): $(TOOL_MANIFEST) $(TOOL_UPDATER_BINARY) + @"$(TOOL_UPDATER_BINARY)" generate-make --manifest "$(TOOL_MANIFEST)" --output "$@" + +.PHONY: update-tools +update-tools: $(TOOL_UPDATER_BINARY) ## Check tool releases and refresh versions and checksums in the manifest. + @"$(TOOL_UPDATER_BINARY)" update --manifest "$(TOOL_MANIFEST)" --makefile "$(TOOL_MAKE_INCLUDE)" .PHONY: install-yq install-yq: ## Install the pinned yq YAML processor into a user-owned bin dir (no sudo). @@ -36,18 +46,6 @@ install-yq: ## Install the pinned yq YAML processor into a user-owned bin dir (n YQ_INSTALL_DIR="$(YQ_INSTALL_DIR)" \ ./build/scripts/install-yq.sh -# bicep CLI - pinned version and per-platform SHA-256 checksums consumed by -# build/scripts/install-bicep.sh. Pinned to v0.42.1: v0.43+ rejects br:localhost -# registries used by the local functional tests; bump only after verifying -# localhost support. Azure/bicep publishes no checksums file, so these are -# computed from the pinned release - recompute them (download each -# bicep-- asset and sha256sum it) when bumping BICEP_VERSION. -BICEP_VERSION ?= v0.42.1 -BICEP_CHECKSUM_LINUX_AMD64 ?= aed90eb2c69a6ee2bd70dc0d4354408ac4d04fd9911d3ec8e0cd74ad173e7139 -BICEP_CHECKSUM_LINUX_ARM64 ?= b01ac3bb5259096dfbe548138a538d1c4e4a55e6f87f3827e2299fbc2d4e6796 -BICEP_CHECKSUM_DARWIN_AMD64 ?= 8219bfd0601a514cc0a814b4b194aed588f4efa68b7c7ac7c9b64f3d84713dd7 -BICEP_CHECKSUM_DARWIN_ARM64 ?= 1c66533af4d4d47f875623d88074d28ca7fe7e9dc1f783a62570e8724700aca1 - .PHONY: install-bicep install-bicep: ## Install the pinned Bicep CLI into a user-owned bin dir (no sudo). @BICEP_VERSION="$(BICEP_VERSION)" \ @@ -58,17 +56,6 @@ install-bicep: ## Install the pinned Bicep CLI into a user-owned bin dir (no sud BICEP_INSTALL_DIR="$(BICEP_INSTALL_DIR)" \ ./build/scripts/install-bicep.sh -# kind (Kubernetes IN Docker) - pinned version and per-platform SHA-256 checksums -# consumed by build/scripts/install-kind.sh. The script is generic: clear -# KIND_VERSION to install the latest release, and clear a checksum to have it read -# from the release's own '.sha256sum' file. Keep the checksums in sync when -# bumping KIND_VERSION. -KIND_VERSION ?= v0.32.0 -KIND_CHECKSUM_LINUX_AMD64 ?= 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 -KIND_CHECKSUM_LINUX_ARM64 ?= b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d -KIND_CHECKSUM_DARWIN_AMD64 ?= 295ac6d0d634c9819c9907df45e3017d1f13166bd13c3404c45e79f7faa47498 -KIND_CHECKSUM_DARWIN_ARM64 ?= dca67911095a110c2b5c36e26df6cac860c602033e456c0db47be498cdef1ebb - .PHONY: install-kind install-kind: ## Install the pinned kind cluster tool into a user-owned bin dir (no sudo). @KIND_VERSION="$(KIND_VERSION)" \ @@ -79,18 +66,6 @@ install-kind: ## Install the pinned kind cluster tool into a user-owned bin dir KIND_INSTALL_DIR="$(KIND_INSTALL_DIR)" \ ./build/scripts/install-kind.sh -# kubectl (Kubernetes CLI) - pinned version and per-platform SHA-256 checksums -# consumed by build/scripts/install-kubectl.sh. kubectl is published on the -# Kubernetes release CDN (dl.k8s.io), not GitHub. The script is generic: clear -# KUBECTL_VERSION to install the latest stable release, and clear a checksum to -# have it read from the release's own 'kubectl.sha256' file. Keep the checksums in -# sync when bumping KUBECTL_VERSION. -KUBECTL_VERSION ?= v1.36.2 -KUBECTL_CHECKSUM_LINUX_AMD64 ?= 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 -KUBECTL_CHECKSUM_LINUX_ARM64 ?= c957eb8c4bea27a3bb35b269edd9082e27f027f7b76b20b5bf4afebc726c6d3e -KUBECTL_CHECKSUM_DARWIN_AMD64 ?= ce6c5e55cd17559e87e4fb5e73ebbbc2511bcf2b695d7a40c1b1461a9817d4b3 -KUBECTL_CHECKSUM_DARWIN_ARM64 ?= 4408c85c83fd3a31adaa555bdf3c7a6c81f74b19449a9060ba31ab91926f023d - .PHONY: install-kubectl install-kubectl: ## Install the pinned kubectl Kubernetes CLI into a user-owned bin dir (no sudo). @KUBECTL_VERSION="$(KUBECTL_VERSION)" \ @@ -101,18 +76,6 @@ install-kubectl: ## Install the pinned kubectl Kubernetes CLI into a user-owned KUBECTL_INSTALL_DIR="$(KUBECTL_INSTALL_DIR)" \ ./build/scripts/install-kubectl.sh -# Dapr CLI - pinned version and per-platform SHA-256 checksums (of the release -# tarball) consumed by build/scripts/install-dapr.sh. The script is generic: clear -# DAPR_VERSION to install the latest release, and clear a checksum to have it -# read from the release's own '.sha256' file. Keep the checksums in sync -# when bumping DAPR_VERSION. This pins the Dapr CLI only; the Dapr runtime and -# dashboard versions used by `dapr init -k` are pinned in the workflows. -DAPR_VERSION ?= v1.15.2 -DAPR_CHECKSUM_LINUX_AMD64 ?= 09328bc0e4353036b824c2ec9cf7cabf4d75b4fc00ca02d80ae3e4374ee27eda -DAPR_CHECKSUM_LINUX_ARM64 ?= b49244701a191c1e843211383703be9f2cd086a1db259c9789672f7e4e82ad55 -DAPR_CHECKSUM_DARWIN_AMD64 ?= 42a36e667559aef0fb6357fbe8f0fdbf1a6d9ea0ba8484c32e90ea61ddf15ba0 -DAPR_CHECKSUM_DARWIN_ARM64 ?= 176f455ea1961cdb59ab0e9ec3e4900b877576a9a2178d3b4b2619bfe947643f - .PHONY: install-dapr install-dapr: ## Install the pinned Dapr CLI into a user-owned bin dir (no sudo). @DAPR_VERSION="$(DAPR_VERSION)" \ @@ -123,18 +86,6 @@ install-dapr: ## Install the pinned Dapr CLI into a user-owned bin dir (no sudo) DAPR_INSTALL_DIR="$(DAPR_INSTALL_DIR)" \ ./build/scripts/install-dapr.sh -# Helm CLI - pinned version and per-platform SHA-256 checksums (of the release -# tarball) consumed by build/scripts/install-helm.sh. Helm is published on the -# Helm release CDN (get.helm.sh), not GitHub. The script is generic: clear -# HELM_VERSION to install the latest release, and clear a checksum to have it read -# from the release's own '.sha256sum' file. Keep the checksums in sync -# when bumping HELM_VERSION. -HELM_VERSION ?= v4.2.2 -HELM_CHECKSUM_LINUX_AMD64 ?= 9adafecab4d406853bba163a70e9f104f47dbbf65ce24b7653bae7e36150bcb6 -HELM_CHECKSUM_LINUX_ARM64 ?= 78803142087a0069fa4b50d3f32a84d3ef25c14d1ee8a40fbccf86a6216d2f36 -HELM_CHECKSUM_DARWIN_AMD64 ?= 10c1e36ee8c5f2e2ee25a16599cb03ab74c0953cd889cacb980a49ba4b6574ba -HELM_CHECKSUM_DARWIN_ARM64 ?= 5410a0dae3d5d91f45653b161260d9301aabc4ae80ae50a6605d66884b6df8ea - .PHONY: install-helm install-helm: ## Install the pinned Helm CLI into a user-owned bin dir (no sudo). @HELM_VERSION="$(HELM_VERSION)" \ @@ -145,17 +96,6 @@ install-helm: ## Install the pinned Helm CLI into a user-owned bin dir (no sudo) HELM_INSTALL_DIR="$(HELM_INSTALL_DIR)" \ ./build/scripts/install-helm.sh -# k3d (k3s in Docker) - pinned version and per-platform SHA-256 checksums consumed -# by build/scripts/install-k3d.sh. The script is generic: clear K3D_VERSION to -# install the latest release, and clear a checksum to have it read from the -# release's own combined 'checksums.txt' file. Keep the checksums in sync when -# bumping K3D_VERSION. -K3D_VERSION ?= v5.9.0 -K3D_CHECKSUM_LINUX_AMD64 ?= 06d8f25bc3a971c4eb29e0ff08429b180402db0f4dec838c9eac427e296800a0 -K3D_CHECKSUM_LINUX_ARM64 ?= 03cde5cf23e6e8e67de5a039ecf26e5b85aca82fba3e5d13dadf904cd218a250 -K3D_CHECKSUM_DARWIN_AMD64 ?= b4aabc37534f95b9c764e7823f2df923f50d57600837aa60a06266cce47db732 -K3D_CHECKSUM_DARWIN_ARM64 ?= fe106541d5d0a3f18debcd4d432a16f8c0ce3e6ddc06f8fbb6f696a122313e00 - .PHONY: install-k3d install-k3d: ## Install the pinned k3d cluster tool into a user-owned bin dir (no sudo). @K3D_VERSION="$(K3D_VERSION)" \ @@ -166,18 +106,6 @@ install-k3d: ## Install the pinned k3d cluster tool into a user-owned bin dir (n K3D_INSTALL_DIR="$(K3D_INSTALL_DIR)" \ ./build/scripts/install-k3d.sh -# golangci-lint - pinned version and per-platform SHA-256 checksums (of the -# release tarball) consumed by build/scripts/install-golangci-lint.sh. The script -# is generic: clear GOLANGCI_LINT_VERSION to install the latest release, and clear -# a checksum to have it read from the release's own -# 'golangci-lint--checksums.txt' file. Keep the checksums in sync when -# bumping GOLANGCI_LINT_VERSION. -GOLANGCI_LINT_VERSION ?= v2.12.2 -GOLANGCI_LINT_CHECKSUM_LINUX_AMD64 ?= 8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 -GOLANGCI_LINT_CHECKSUM_LINUX_ARM64 ?= 44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a -GOLANGCI_LINT_CHECKSUM_DARWIN_AMD64 ?= f6f06d94b6241521c53d15450c5209b028270bf966f842afb11c030c79f5bc16 -GOLANGCI_LINT_CHECKSUM_DARWIN_ARM64 ?= a9c54498731b3128f79e090be6110f3e5fffccc617b08142ed244d4126c73f29 - .PHONY: install-golangci-lint install-golangci-lint: ## Install the pinned golangci-lint into a user-owned bin dir (no sudo). @GOLANGCI_LINT_VERSION="$(GOLANGCI_LINT_VERSION)" \ @@ -188,18 +116,6 @@ install-golangci-lint: ## Install the pinned golangci-lint into a user-owned bin GOLANGCI_LINT_INSTALL_DIR="$(GOLANGCI_LINT_INSTALL_DIR)" \ ./build/scripts/install-golangci-lint.sh -# Terraform CLI - per-platform SHA-256 checksums (of the release zip) consumed by -# build/scripts/install-terraform.sh. TERRAFORM_VERSION itself is defined in -# build/build.mk, where it defaults to the .terraform-version file (overridable, -# e.g. `make install-terraform TERRAFORM_VERSION=1.15.0`) and is also embedded into -# the rad binary. Keep these checksums in sync when bumping .terraform-version. The -# script is generic: clear a checksum to have it read from the release's own -# 'terraform__SHA256SUMS' file. -TERRAFORM_CHECKSUM_LINUX_AMD64 ?= 2e5cffc20a0b48a67a76268723bd5a10b8666f69b2aa4f04906e206726bedd63 -TERRAFORM_CHECKSUM_LINUX_ARM64 ?= 863002085b886453795d9ff4b8989b8468784478150b70ba8a1df3e3ad66da99 -TERRAFORM_CHECKSUM_DARWIN_AMD64 ?= c15326e1af102d2767d40208a0157d1402057f80192991f56803b66457304cf3 -TERRAFORM_CHECKSUM_DARWIN_ARM64 ?= 5bc0b11b7a63c8984a41d82523356df46f7833c2e9651a39a7f8919422de5cde - .PHONY: install-terraform install-terraform: ## Install the pinned Terraform CLI into a user-owned bin dir (no sudo). @TERRAFORM_VERSION="$(TERRAFORM_VERSION)" \ @@ -210,18 +126,6 @@ install-terraform: ## Install the pinned Terraform CLI into a user-owned bin dir TERRAFORM_INSTALL_DIR="$(TERRAFORM_INSTALL_DIR)" \ ./build/scripts/install-terraform.sh -# stern (multi-pod Kubernetes log tailing) - pinned version and per-platform -# SHA-256 checksums (of the release tarball) consumed by -# build/scripts/install-stern.sh. The script is generic: clear STERN_VERSION to -# install the latest release, and clear a checksum to have it read from the -# release's own combined 'checksums.txt' file. Keep the checksums in sync when -# bumping STERN_VERSION. -STERN_VERSION ?= v1.34.0 -STERN_CHECKSUM_LINUX_AMD64 ?= 7754adfa653939240f7d20fff4ada9b69cda40c9e70732301f67bb8045f1ef3e -STERN_CHECKSUM_LINUX_ARM64 ?= e215cfc5e42d71e93b77d3fac8f0df7d736271f44b2d92a5b417eaa588edff3a -STERN_CHECKSUM_DARWIN_AMD64 ?= 153355317f21e565ea10bc710d4c2e3d98fd06f83cae5eb927e7031cc724a7a6 -STERN_CHECKSUM_DARWIN_ARM64 ?= 4014d84096e1e603ee115864e03a1e15fb9bae9876647bf7bb8031eee278dcd3 - .PHONY: install-stern install-stern: ## Install the pinned stern log tailing tool into a user-owned bin dir (no sudo). @STERN_VERSION="$(STERN_VERSION)" \ @@ -232,18 +136,6 @@ install-stern: ## Install the pinned stern log tailing tool into a user-owned bi STERN_INSTALL_DIR="$(STERN_INSTALL_DIR)" \ ./build/scripts/install-stern.sh -# ORAS (OCI Registry As Storage) CLI - pinned version and per-platform SHA-256 -# checksums (of the release tarball) consumed by build/scripts/install-oras.sh. -# The script is generic: clear ORAS_VERSION to install the latest release, and -# clear a checksum to have it read from the release's own -# 'oras__checksums.txt' file. Keep the checksums in sync when bumping -# ORAS_VERSION. -ORAS_VERSION ?= v1.3.2 -ORAS_CHECKSUM_LINUX_AMD64 ?= 9229ccc6d17bb282039ad4a69abb16dcb887a5bce567c075d731d9b3c7ad8eaf -ORAS_CHECKSUM_LINUX_ARM64 ?= 8db4a223bd6034deff198e791ea7cb3af0840df25b7e9f370e2f1f3fd20d389b -ORAS_CHECKSUM_DARWIN_AMD64 ?= 2621f6b252b222f6fbf4e114d2fcaa0cec6b632624ffaf73143f66e4e0994f86 -ORAS_CHECKSUM_DARWIN_ARM64 ?= 7929f792cf272268412375ecad6f0fb3c20f164368d5b57966e67ad6d36eca53 - .PHONY: install-oras install-oras: ## Install the pinned ORAS CLI into a user-owned bin dir (no sudo). @ORAS_VERSION="$(ORAS_VERSION)" \ @@ -254,21 +146,6 @@ install-oras: ## Install the pinned ORAS CLI into a user-owned bin dir (no sudo) ORAS_INSTALL_DIR="$(ORAS_INSTALL_DIR)" \ ./build/scripts/install-oras.sh -# ShellCheck (static analysis for shell scripts) - pinned version and per-platform -# SHA-256 checksums (of the release '.tar.xz' archive) consumed by -# build/scripts/install-shellcheck.sh. ShellCheck is published as a per-platform -# archive on koalaman/shellcheck's GitHub releases and publishes no checksums -# file, so these are computed from the pinned release - recompute them (download -# each shellcheck-...tar.xz asset and sha256sum it) when -# bumping SHELLCHECK_VERSION. The script is generic: clear SHELLCHECK_VERSION to -# install the latest release, and clear a checksum to install without -# verification. -SHELLCHECK_VERSION ?= v0.11.0 -SHELLCHECK_CHECKSUM_LINUX_AMD64 ?= 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 -SHELLCHECK_CHECKSUM_LINUX_ARM64 ?= 12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588 -SHELLCHECK_CHECKSUM_DARWIN_AMD64 ?= 3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6 -SHELLCHECK_CHECKSUM_DARWIN_ARM64 ?= 56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79 - .PHONY: install-shellcheck install-shellcheck: ## Install the pinned ShellCheck into a user-owned bin dir (no sudo). @SHELLCHECK_VERSION="$(SHELLCHECK_VERSION)" \ @@ -279,19 +156,6 @@ install-shellcheck: ## Install the pinned ShellCheck into a user-owned bin dir ( SHELLCHECK_INSTALL_DIR="$(SHELLCHECK_INSTALL_DIR)" \ ./build/scripts/install-shellcheck.sh -# jq (command-line JSON processor) - pinned version and per-platform SHA-256 -# checksums consumed by build/scripts/install-jq.sh. jq is published as a single -# per-platform binary on jqlang/jq's GitHub releases (release tags are -# 'jq-', and darwin assets are named 'macos'). The script is generic: -# clear JQ_VERSION to install the latest release, and clear a checksum to have it -# read from the release's own 'sha256sum.txt' file. Keep the checksums in sync -# when bumping JQ_VERSION. -JQ_VERSION ?= 1.8.2 -JQ_CHECKSUM_LINUX_AMD64 ?= b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f -JQ_CHECKSUM_LINUX_ARM64 ?= 8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309 -JQ_CHECKSUM_DARWIN_AMD64 ?= e94b266e3c26690550006abe63152b782280f4e14374accdf04cbde844f00bc0 -JQ_CHECKSUM_DARWIN_ARM64 ?= 2d75340ba57a4b4b4c8708a21c2dc8e958a48aaa8bba13b27f77f6e4c0eca07e - .PHONY: install-jq install-jq: ## Install the pinned jq JSON processor into a user-owned bin dir (no sudo). @JQ_VERSION="$(JQ_VERSION)" \ @@ -302,14 +166,6 @@ install-jq: ## Install the pinned jq JSON processor into a user-owned bin dir (n JQ_INSTALL_DIR="$(JQ_INSTALL_DIR)" \ ./build/scripts/install-jq.sh -# Delve (the 'dlv' Go debugger) - pinned module version consumed by -# build/scripts/install-dlv.sh. Delve publishes no prebuilt binaries, so it is -# installed with 'go install' and its integrity is guaranteed by the Go checksum -# database; there are no per-platform SHA-256 checksums to pin. Clear DLV_VERSION -# to install the latest release. Used by the process-debugging workflow (make -# debug-*). -DLV_VERSION ?= v1.27.0 - .PHONY: install-dlv install-dlv: ## Install the pinned Delve (dlv) Go debugger via 'go install'. @DLV_VERSION="$(DLV_VERSION)" \ diff --git a/build/tools.yaml b/build/tools.yaml new file mode 100644 index 00000000000..99e70d4f790 --- /dev/null +++ b/build/tools.yaml @@ -0,0 +1,366 @@ +--- +schemaVersion: 1 +platforms: + - linux_amd64 + - linux_arm64 + - darwin_amd64 + - darwin_arm64 + +tools: + - name: yq + makePrefix: YQ + version: v4.53.3 + source: + type: github-release + repository: mikefarah/yq + latestURL: https://api.github.com/repos/mikefarah/yq/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: yq_linux_amd64 + checksum: fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4 + linux_arm64: + asset: yq_linux_arm64 + checksum: 578648e463a11c1b6db6010cbf41eafed6bee79466fcffa1bb446672cf7945ea + darwin_amd64: + asset: yq_darwin_amd64 + checksum: b4ba1ecce3c47f00803f4f964de38394326c7a32eb6540616e04fb2935a0f08d + darwin_arm64: + asset: yq_darwin_arm64 + checksum: 877de31753a4dd2401aa048937aa9a7fc4d5f6ce858cf31508c5802954297213 + checksumSource: + type: github-release-file + fileTemplate: checksums + orderFileTemplate: checksums_hashes_order + format: yq + + - name: bicep + makePrefix: BICEP + version: v0.42.1 + update: false + notes: Bicep v0.43 and later reject the br:localhost registries used by local functional tests. + source: + type: github-release + repository: Azure/bicep + latestURL: https://api.github.com/repos/Azure/bicep/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: bicep-linux-x64 + checksum: aed90eb2c69a6ee2bd70dc0d4354408ac4d04fd9911d3ec8e0cd74ad173e7139 + linux_arm64: + asset: bicep-linux-arm64 + checksum: b01ac3bb5259096dfbe548138a538d1c4e4a55e6f87f3827e2299fbc2d4e6796 + darwin_amd64: + asset: bicep-osx-x64 + checksum: 8219bfd0601a514cc0a814b4b194aed588f4efa68b7c7ac7c9b64f3d84713dd7 + darwin_arm64: + asset: bicep-osx-arm64 + checksum: 1c66533af4d4d47f875623d88074d28ca7fe7e9dc1f783a62570e8724700aca1 + checksumSource: + type: download + + - name: kind + makePrefix: KIND + version: v0.32.0 + source: + type: github-release + repository: kubernetes-sigs/kind + latestURL: https://api.github.com/repos/kubernetes-sigs/kind/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: kind-linux-amd64 + checksum: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 + linux_arm64: + asset: kind-linux-arm64 + checksum: b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d + darwin_amd64: + asset: kind-darwin-amd64 + checksum: 295ac6d0d634c9819c9907df45e3017d1f13166bd13c3404c45e79f7faa47498 + darwin_arm64: + asset: kind-darwin-arm64 + checksum: dca67911095a110c2b5c36e26df6cac860c602033e456c0db47be498cdef1ebb + checksumSource: + type: github-release-file + fileTemplate: "{asset}.sha256sum" + format: first + + - name: kubectl + makePrefix: KUBECTL + version: v1.36.2 + source: + type: stable-text + latestURL: https://dl.k8s.io/release/stable.txt + downloadTemplate: https://dl.k8s.io/release/{version}/bin/{os}/{arch}/kubectl + platforms: + linux_amd64: + asset: kubectl + checksum: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + linux_arm64: + asset: kubectl + checksum: c957eb8c4bea27a3bb35b269edd9082e27f027f7b76b20b5bf4afebc726c6d3e + darwin_amd64: + asset: kubectl + checksum: ce6c5e55cd17559e87e4fb5e73ebbbc2511bcf2b695d7a40c1b1461a9817d4b3 + darwin_arm64: + asset: kubectl + checksum: 4408c85c83fd3a31adaa555bdf3c7a6c81f74b19449a9060ba31ab91926f023d + checksumSource: + type: url-file + urlTemplate: https://dl.k8s.io/release/{version}/bin/{os}/{arch}/kubectl.sha256 + format: first + + - name: dapr + makePrefix: DAPR + version: v1.15.2 + source: + type: github-release + repository: dapr/cli + latestURL: https://api.github.com/repos/dapr/cli/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: dapr_linux_amd64.tar.gz + checksum: 09328bc0e4353036b824c2ec9cf7cabf4d75b4fc00ca02d80ae3e4374ee27eda + linux_arm64: + asset: dapr_linux_arm64.tar.gz + checksum: b49244701a191c1e843211383703be9f2cd086a1db259c9789672f7e4e82ad55 + darwin_amd64: + asset: dapr_darwin_amd64.tar.gz + checksum: 42a36e667559aef0fb6357fbe8f0fdbf1a6d9ea0ba8484c32e90ea61ddf15ba0 + darwin_arm64: + asset: dapr_darwin_arm64.tar.gz + checksum: 176f455ea1961cdb59ab0e9ec3e4900b877576a9a2178d3b4b2619bfe947643f + checksumSource: + type: github-release-file + fileTemplate: "{asset}.sha256" + format: first + + - name: helm + makePrefix: HELM + version: v4.2.2 + source: + type: github-release + repository: helm/helm + latestURL: https://api.github.com/repos/helm/helm/releases/latest + downloadTemplate: https://get.helm.sh/helm-{version}-{os}-{arch}.tar.gz + platforms: + linux_amd64: + asset: helm-{version}-linux-amd64.tar.gz + checksum: 9adafecab4d406853bba163a70e9f104f47dbbf65ce24b7653bae7e36150bcb6 + linux_arm64: + asset: helm-{version}-linux-arm64.tar.gz + checksum: 78803142087a0069fa4b50d3f32a84d3ef25c14d1ee8a40fbccf86a6216d2f36 + darwin_amd64: + asset: helm-{version}-darwin-amd64.tar.gz + checksum: 10c1e36ee8c5f2e2ee25a16599cb03ab74c0953cd889cacb980a49ba4b6574ba + darwin_arm64: + asset: helm-{version}-darwin-arm64.tar.gz + checksum: 5410a0dae3d5d91f45653b161260d9301aabc4ae80ae50a6605d66884b6df8ea + checksumSource: + type: url-file + urlTemplate: https://get.helm.sh/{asset}.sha256sum + format: first + + - name: k3d + makePrefix: K3D + version: v5.9.0 + source: + type: github-release + repository: k3d-io/k3d + latestURL: https://api.github.com/repos/k3d-io/k3d/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: k3d-linux-amd64 + checksum: 06d8f25bc3a971c4eb29e0ff08429b180402db0f4dec838c9eac427e296800a0 + linux_arm64: + asset: k3d-linux-arm64 + checksum: 03cde5cf23e6e8e67de5a039ecf26e5b85aca82fba3e5d13dadf904cd218a250 + darwin_amd64: + asset: k3d-darwin-amd64 + checksum: b4aabc37534f95b9c764e7823f2df923f50d57600837aa60a06266cce47db732 + darwin_arm64: + asset: k3d-darwin-arm64 + checksum: fe106541d5d0a3f18debcd4d432a16f8c0ce3e6ddc06f8fbb6f696a122313e00 + checksumSource: + type: github-release-file + fileTemplate: checksums.txt + format: basename + + - name: golangci-lint + makePrefix: GOLANGCI_LINT + version: v2.12.2 + source: + type: github-release + repository: golangci/golangci-lint + latestURL: https://api.github.com/repos/golangci/golangci-lint/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: golangci-lint-{version_no_v}-linux-amd64.tar.gz + checksum: 8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 + linux_arm64: + asset: golangci-lint-{version_no_v}-linux-arm64.tar.gz + checksum: 44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a + darwin_amd64: + asset: golangci-lint-{version_no_v}-darwin-amd64.tar.gz + checksum: f6f06d94b6241521c53d15450c5209b028270bf966f842afb11c030c79f5bc16 + darwin_arm64: + asset: golangci-lint-{version_no_v}-darwin-arm64.tar.gz + checksum: a9c54498731b3128f79e090be6110f3e5fffccc617b08142ed244d4126c73f29 + checksumSource: + type: github-release-file + fileTemplate: golangci-lint-{version_no_v}-checksums.txt + format: standard + + - name: terraform + makePrefix: TERRAFORM + version: 1.14.9 + source: + type: hashicorp-checkpoint + latestURL: https://checkpoint-api.hashicorp.com/v1/check/terraform + downloadTemplate: https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{os}_{arch}.zip + platforms: + linux_amd64: + asset: terraform_{version}_linux_amd64.zip + checksum: 2e5cffc20a0b48a67a76268723bd5a10b8666f69b2aa4f04906e206726bedd63 + linux_arm64: + asset: terraform_{version}_linux_arm64.zip + checksum: 863002085b886453795d9ff4b8989b8468784478150b70ba8a1df3e3ad66da99 + darwin_amd64: + asset: terraform_{version}_darwin_amd64.zip + checksum: c15326e1af102d2767d40208a0157d1402057f80192991f56803b66457304cf3 + darwin_arm64: + asset: terraform_{version}_darwin_arm64.zip + checksum: 5bc0b11b7a63c8984a41d82523356df46f7833c2e9651a39a7f8919422de5cde + checksumSource: + type: url-file + urlTemplate: https://releases.hashicorp.com/terraform/{version}/terraform_{version}_SHA256SUMS + format: standard + versionFiles: + - path: .terraform-version + format: plain + - path: pkg/recipes/terraform/version.go + format: replace + prefix: 'var terraformVersion = "' + suffix: '"' + - path: deploy/Chart/values.yaml + format: replace + prefix: ' version: "' + suffix: '"' + + - name: stern + makePrefix: STERN + version: v1.34.0 + source: + type: github-release + repository: stern/stern + latestURL: https://api.github.com/repos/stern/stern/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: stern_{version_no_v}_linux_amd64.tar.gz + checksum: 7754adfa653939240f7d20fff4ada9b69cda40c9e70732301f67bb8045f1ef3e + linux_arm64: + asset: stern_{version_no_v}_linux_arm64.tar.gz + checksum: e215cfc5e42d71e93b77d3fac8f0df7d736271f44b2d92a5b417eaa588edff3a + darwin_amd64: + asset: stern_{version_no_v}_darwin_amd64.tar.gz + checksum: 153355317f21e565ea10bc710d4c2e3d98fd06f83cae5eb927e7031cc724a7a6 + darwin_arm64: + asset: stern_{version_no_v}_darwin_arm64.tar.gz + checksum: 4014d84096e1e603ee115864e03a1e15fb9bae9876647bf7bb8031eee278dcd3 + checksumSource: + type: github-release-file + fileTemplate: checksums.txt + format: standard + + - name: oras + makePrefix: ORAS + version: v1.3.2 + source: + type: github-release + repository: oras-project/oras + latestURL: https://api.github.com/repos/oras-project/oras/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: oras_{version_no_v}_linux_amd64.tar.gz + checksum: 9229ccc6d17bb282039ad4a69abb16dcb887a5bce567c075d731d9b3c7ad8eaf + linux_arm64: + asset: oras_{version_no_v}_linux_arm64.tar.gz + checksum: 8db4a223bd6034deff198e791ea7cb3af0840df25b7e9f370e2f1f3fd20d389b + darwin_amd64: + asset: oras_{version_no_v}_darwin_amd64.tar.gz + checksum: 2621f6b252b222f6fbf4e114d2fcaa0cec6b632624ffaf73143f66e4e0994f86 + darwin_arm64: + asset: oras_{version_no_v}_darwin_arm64.tar.gz + checksum: 7929f792cf272268412375ecad6f0fb3c20f164368d5b57966e67ad6d36eca53 + checksumSource: + type: github-release-file + fileTemplate: oras_{version_no_v}_checksums.txt + format: standard + + - name: shellcheck + makePrefix: SHELLCHECK + version: v0.11.0 + source: + type: github-release + repository: koalaman/shellcheck + latestURL: https://api.github.com/repos/koalaman/shellcheck/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: shellcheck-{version}.linux.x86_64.tar.xz + checksum: 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 + linux_arm64: + asset: shellcheck-{version}.linux.aarch64.tar.xz + checksum: 12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588 + darwin_amd64: + asset: shellcheck-{version}.darwin.x86_64.tar.xz + checksum: 3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6 + darwin_arm64: + asset: shellcheck-{version}.darwin.aarch64.tar.xz + checksum: 56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79 + checksumSource: + type: download + + - name: jq + makePrefix: JQ + version: 1.8.2 + source: + type: github-release + repository: jqlang/jq + tagPrefix: jq- + latestURL: https://api.github.com/repos/jqlang/jq/releases/latest + downloadTemplate: https://github.com/{repository}/releases/download/{tag}/{asset} + platforms: + linux_amd64: + asset: jq-linux-amd64 + checksum: b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f + linux_arm64: + asset: jq-linux-arm64 + checksum: 8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309 + darwin_amd64: + asset: jq-macos-amd64 + checksum: e94b266e3c26690550006abe63152b782280f4e14374accdf04cbde844f00bc0 + darwin_arm64: + asset: jq-macos-arm64 + checksum: 2d75340ba57a4b4b4c8708a21c2dc8e958a48aaa8bba13b27f77f6e4c0eca07e + checksumSource: + type: github-release-file + fileTemplate: sha256sum.txt + format: standard + + - name: dlv + makePrefix: DLV + version: v1.27.0 + source: + type: github-release + repository: go-delve/delve + latestURL: https://api.github.com/repos/go-delve/delve/releases/latest + checksumSource: + type: none + integrity: go-sumdb diff --git a/build/workflow.mk b/build/workflow.mk index 2adeaf1a0d1..f7d77cb8c8a 100644 --- a/build/workflow.mk +++ b/build/workflow.mk @@ -4,7 +4,7 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -29,3 +29,7 @@ workflow-enable-all: ## Enable all workflows in the current repo .PHONY: workflow-delete-all-runs workflow-delete-all-runs: ## Delete all workflow runs in the repository. NOTE: This is a destructive operation and cannot be undone. @bash $(WORKFLOW_SCRIPT) delete-all-runs + +.PHONY: workflow-update-tools-pr +workflow-update-tools-pr: ## Commit tool updates and create or update the automation pull request. + @bash ./.github/scripts/update-tools-pr.sh diff --git a/cmd/tool-updater/main.go b/cmd/tool-updater/main.go new file mode 100644 index 00000000000..8319f8f5e2c --- /dev/null +++ b/cmd/tool-updater/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + + "github.com/radius-project/radius/internal/tooling" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "tool-updater: %v\n", err) + os.Exit(1) //nolint:forbidigo // this is OK inside the main function. + } +} + +func run(args []string) error { + if len(args) == 0 { + return errors.New("a command is required: generate-make or update") + } + + switch args[0] { + case "generate-make": + return generateMake(args[1:]) + case "update": + return update(args[1:]) + default: + return fmt.Errorf("unknown command %q", args[0]) + } +} + +func generateMake(args []string) error { + flags := flag.NewFlagSet("generate-make", flag.ContinueOnError) + manifestPath := flags.String("manifest", "build/tools.yaml", "tool manifest path") + outputPath := flags.String("output", "build/tools.generated.mk", "generated Make include path") + err := flags.Parse(args) + if err != nil { + return fmt.Errorf("parse flags: %w", err) + } + + manifest, err := tooling.LoadManifest(*manifestPath) + if err != nil { + return fmt.Errorf("load manifest: %w", err) + } + changed, err := tooling.WriteMakeFile(*outputPath, manifest) + if err != nil { + return fmt.Errorf("write Make metadata: %w", err) + } + if changed { + fmt.Printf("generated %s\n", *outputPath) + } + return nil +} + +func update(args []string) error { + flags := flag.NewFlagSet("update", flag.ContinueOnError) + manifestPath := flags.String("manifest", "build/tools.yaml", "tool manifest path") + makePath := flags.String("makefile", "build/tools.generated.mk", "generated Make include path") + err := flags.Parse(args) + if err != nil { + return fmt.Errorf("parse flags: %w", err) + } + + manifest, err := tooling.LoadManifest(*manifestPath) + if err != nil { + return fmt.Errorf("load manifest: %w", err) + } + changes, err := tooling.UpdateManifest(context.Background(), &manifest, tooling.NewClient("")) + if err != nil { + return fmt.Errorf("update tool metadata: %w", err) + } + if len(changes) == 0 { + fmt.Println("tool metadata is current") + } else { + for _, change := range changes { + fmt.Printf("updated %s\n", change) + } + if _, err := tooling.WriteManifest(*manifestPath, manifest); err != nil { + return fmt.Errorf("write manifest: %w", err) + } + } + + if err := syncVersionFiles(".", manifest); err != nil { + return err + } + if _, err := tooling.WriteMakeFile(*makePath, manifest); err != nil { + return fmt.Errorf("write Make metadata: %w", err) + } + return nil +} + +func syncVersionFiles(root string, manifest tooling.Manifest) error { + for _, tool := range manifest.Tools { + if err := tooling.SyncVersionFiles(root, tool); err != nil { + return fmt.Errorf("sync %s version consumers: %w", tool.Name, err) + } + } + return nil +} diff --git a/cmd/tool-updater/main_test.go b/cmd/tool-updater/main_test.go new file mode 100644 index 00000000000..c631f0465d1 --- /dev/null +++ b/cmd/tool-updater/main_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/radius-project/radius/internal/tooling" +) + +func TestSyncVersionFilesUpdatesEveryTool(t *testing.T) { + root := t.TempDir() + manifest := tooling.Manifest{ + Tools: []tooling.Tool{ + { + Name: "first", + Version: "1.2.3", + VersionFiles: []tooling.VersionFile{{ + Path: "first.version", + Format: "plain", + }}, + }, + { + Name: "second", + Version: "v4.5.6", + VersionFiles: []tooling.VersionFile{{ + Path: "second.version", + Format: "plain", + }}, + }, + }, + } + for _, name := range []string{"first.version", "second.version"} { + if err := os.WriteFile(filepath.Join(root, name), nil, 0o644); err != nil { + t.Fatal(err) + } + } + + if err := syncVersionFiles(root, manifest); err != nil { + t.Fatalf("syncVersionFiles() error = %v", err) + } + for path, want := range map[string]string{ + "first.version": "1.2.3\n", + "second.version": "v4.5.6\n", + } { + contents, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + t.Fatal(err) + } + if got := string(contents); got != want { + t.Errorf("%s = %q, want %q", path, got, want) + } + } +} diff --git a/deploy/Chart/tests/terraform_init_test.yaml b/deploy/Chart/tests/terraform_init_test.yaml index 7df486d964f..edd4033aa9d 100644 --- a/deploy/Chart/tests/terraform_init_test.yaml +++ b/deploy/Chart/tests/terraform_init_test.yaml @@ -74,23 +74,27 @@ tests: errorMessage: 'dynamicrp.terraform.path must be "/terraform" -- the runtime configmap (deploy/Chart/templates/dynamic-rp/configmaps.yaml) and pkg/recipes/terraform/install.go both hard-code /terraform, so overriding the mount path produces an inconsistent deployment and silently breaks Terraform execution/caching.' template: dynamic-rp/deployment.yaml - # Regression test for the second half of #11880: after fixing the - # binary-path mismatch the runtime actually uses the pre-mounted - # binary, so its version MUST match terraformVersion in - # pkg/recipes/terraform/version.go (currently "1.14.9"). If these - # drift the runtime download fallback and the pre-mount install - # different versions, causing recipes to behave differently between - # cold and warm starts (see Test_TerraformRecipe_Context failure). - - it: rp init container pins terraform version to match pkg/recipes/terraform/version.go + # Regression test for the second half of #11880: the init container must + # install the same version the runtime falls back to, or recipes behave + # differently between cold and warm starts. The literal below is arbitrary + # -- these cases only prove the chart value reaches the command. Drift + # between values.yaml and pkg/recipes/terraform/version.go is covered by + # TestTerraformVersionMatchesHelmChart, so keeping the pinned version out + # of here lets the tool updater bump Terraform without editing this file. + - it: rp init container installs the version from global.terraform.version + set: + global.terraform.version: "9.9.9" asserts: - matchRegex: path: spec.template.spec.initContainers[?(@.name=='terraform-init')].command[2] - pattern: 'TERRAFORM_VERSION="1\.14\.9"' + pattern: 'TERRAFORM_VERSION="9\.9\.9"' template: rp/deployment.yaml - - it: dynamic-rp init container pins terraform version to match pkg/recipes/terraform/version.go + - it: dynamic-rp init container installs the version from global.terraform.version + set: + global.terraform.version: "9.9.9" asserts: - matchRegex: path: spec.template.spec.initContainers[?(@.name=='terraform-init')].command[2] - pattern: 'TERRAFORM_VERSION="1\.14\.9"' + pattern: 'TERRAFORM_VERSION="9\.9\.9"' template: dynamic-rp/deployment.yaml diff --git a/docs/contributing/contributing-code/contributing-code-shell-and-make/README.md b/docs/contributing/contributing-code/contributing-code-shell-and-make/README.md index dcfd2edff08..d8609ba21ee 100644 --- a/docs/contributing/contributing-code/contributing-code-shell-and-make/README.md +++ b/docs/contributing/contributing-code/contributing-code-shell-and-make/README.md @@ -17,6 +17,10 @@ Follow the [shell instruction file](../../../../.github/instructions/shell.instr - **Well-structured Make** — declare `.PHONY` targets, keep recipes small, and add a `##`-style help comment so `make help` stays complete. - **One home per target** — add a new target to the `build/*.mk` include that owns its topic rather than the root `Makefile`. +## Managing external CLI pins + +The canonical versions, release sources, platform assets, and SHA-256 checksums for downloaded CLI tools live in [`build/tools.yaml`](../../../../build/tools.yaml). Run `make update-tools` to check every source and refresh available versions and checksums. The command regenerates the committed [`build/tools.generated.mk`](../../../../build/tools.generated.mk) include that supplies metadata to the existing `make install-` targets. The updater is built into the ignored `bin/` directory before it runs, which avoids Windows security software blocking Go's temporary `go run` executable. See the [tool updater reference](../../../../internal/tooling/README.md) for the manifest schema and supported values. Bicep is checked but intentionally held at its compatibility-pinned version until newer releases support the local registries used by functional tests. + ## Linting shell scripts with ShellCheck Every tracked `.sh` script is validated with [ShellCheck](https://github.com/koalaman/shellcheck) as part of the pull request checks, so run it locally before you push. Install the pinned version (into a user-owned bin directory — no `sudo`) and lint every tracked script with: @@ -26,7 +30,7 @@ make install-shellcheck make lint-shell ``` -`make lint-shell` applies the shared configuration in [`.shellcheckrc`](../../../../.github/linters/.shellcheckrc) and skips the third-party Spec Kit tooling under `.specify/`. The version and per-platform checksums that `make install-shellcheck` pins live in [`build/tools.mk`](../../../../build/tools.mk) (`SHELLCHECK_VERSION`). +`make lint-shell` applies the shared configuration in [`.shellcheckrc`](../../../../.github/linters/.shellcheckrc) and skips the third-party Spec Kit tooling under `.specify/`. The version and per-platform checksums that `make install-shellcheck` pins live in [`build/tools.yaml`](../../../../build/tools.yaml) (`SHELLCHECK_VERSION`). The easy path is the [dev container](../../../../.devcontainer/), which installs the ShellCheck CLI on the `PATH` and the ShellCheck VS Code extension for you, so you can run `make lint-shell` (and see findings inline as you edit) without installing anything. diff --git a/internal/tooling/README.md b/internal/tooling/README.md new file mode 100644 index 00000000000..c737d16d4a5 --- /dev/null +++ b/internal/tooling/README.md @@ -0,0 +1,159 @@ +# External Tool Updater + +## Purpose + +The external tool updater keeps downloaded CLI versions and SHA-256 checksums in [`build/tools.yaml`](../../build/tools.yaml). [`manifest.go`](manifest.go) validates the manifest, [`updater.go`](updater.go) checks release sources, and [`cmd/tool-updater/main.go`](../../cmd/tool-updater/main.go) exposes the commands used by Make and CI. + +The generated [`build/tools.generated.mk`](../../build/tools.generated.mk) file is committed and supplies the metadata to the installer recipes in [`build/tools.mk`](../../build/tools.mk). Do not edit the generated file directly. + +## Commands + +### Update pinned versions + +```sh +make update-tools +``` + +This command checks every configured release source, advances enabled tools to newer releases when available, refreshes platform checksums, synchronizes declared `versionFiles`, and regenerates the committed [`build/tools.generated.mk`](../../build/tools.generated.mk) file. Tools with `update: false` remain pinned at their current version. + +`make update-tools` reaches the network and modifies repository files. It does not install tools on the local machine. + +### Install a pinned tool + +```sh +make install-yq +make install-kubectl +``` + +Use `make install-` to download and install the version pinned in [`build/tools.yaml`](../../build/tools.yaml). Installer targets read versions and checksums from [`build/tools.generated.mk`](../../build/tools.generated.mk) and install into a user-owned binary directory. They do not update the manifest. + +### Run the updater binary directly + +Make builds the updater into `bin/` before executing it. This stable path is important on Windows, where security software may block the temporary executable created by `go run`. + +To invoke the compiled updater directly: + +```sh +bin/tool-updater.exe update --manifest build/tools.yaml --makefile build/tools.generated.mk +``` + +Use `bin/tool-updater` instead of `bin/tool-updater.exe` on non-Windows systems. The other subcommand regenerates only the Make include: + +```sh +bin/tool-updater.exe generate-make --manifest build/tools.yaml --output build/tools.generated.mk +``` + +There is no separate Make target for regeneration only. The `generate-make` subcommand is the binary-level equivalent, and Make runs it automatically when the committed generated include is stale. + +## Manifest reference + +The manifest is YAML with the following top-level properties: + +| Property | Type | Allowed or required values | Description | +|-----------------|-----------------|----------------------------------------------------------------------------------------------------|--------------------------------------------------------| +| `schemaVersion` | integer | `1` | Manifest schema version. | +| `platforms` | list of strings | One or more unique values matching `linux_amd64`, `linux_arm64`, `darwin_amd64`, or `darwin_arm64` | Platforms for which checksums and assets are recorded. | +| `tools` | list of objects | One or more tools | Tool definitions described below. | + +### Tool properties + +| Property | Type | Allowed or required values | Description | +|--------------------|---------|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| +| `name` | string | Lowercase letters, numbers, and hyphens; must start with a lowercase letter or number | Stable tool identifier. | +| `makePrefix` | string | Uppercase letters, numbers, and underscores; must start with an uppercase letter | Prefix used for generated Make variables such as `YQ_VERSION`. | +| `version` | string | Non-empty | Currently pinned release version. The value may include a leading `v` when the upstream uses one. | +| `update` | boolean | `true` or `false`; omitted means `true` | Set to `false` to check the source while keeping the current version pinned. | +| `notes` | string | Optional | Human-readable compatibility or pinning rationale. | +| `source` | object | Required | Describes how the latest version is discovered. | +| `downloadTemplate` | string | Required unless `checksumSource.type` is `none` | URL template for an asset download. | +| `platforms` | map | Required for checksum-bearing tools; must contain every top-level platform | Asset and checksum data for each supported platform. | +| `checksumSource` | object | Required | Describes how the updater obtains or computes SHA-256 checksums. | +| `versionFiles` | list | Optional | Additional repository files whose embedded version must stay synchronized. | + +### `source` properties + +| Property | Type | Allowed or required values | Description | +|--------------|--------|------------------------------------------------------------|--------------------------------------------------------------------------| +| `type` | string | `github-release`, `stable-text`, or `hashicorp-checkpoint` | Version source parser selected by the updater. | +| `repository` | string | Required for `github-release`, in `owner/repository` form | GitHub repository containing the release. | +| `tagPrefix` | string | Optional | Prefix added to the pinned version to form a release tag, such as `jq-`. | +| `latestURL` | string | Non-empty HTTPS URL | Endpoint queried for the latest version. | + +For `github-release`, the endpoint must return a GitHub release object with `tag_name`. `stable-text` reads the trimmed response body, and `hashicorp-checkpoint` reads `current_version` from the JSON response. + +### `checksumSource` properties + +| `type` | Required properties | Behavior | +|-----------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| +| `github-release-file` | `fileTemplate`, `format`; also `orderFileTemplate` when `format` is `yq` | Reads a checksum file from the GitHub release. | +| `url-file` | `urlTemplate`, `format` | Reads a checksum file from an arbitrary URL. | +| `download` | None | Downloads the asset and hashes its bytes locally. | +| `none` | `integrity` | Records a non-SHA-256 integrity method, such as `go-sumdb`; the tool has no platform asset map. | + +Supported checksum formats are `standard` (filename in the second column), `basename` (compare only the filename after a path), `first` (use the first field), and `yq` (use `checksums_hashes_order` to locate the SHA-256 column). Every stored checksum must be exactly 64 lowercase hexadecimal characters. + +### Platform entries + +Each platform entry has these properties: + +| Property | Type | Allowed or required values | Description | +|------------|--------|-------------------------------------------|---------------------------------------------------------| +| `asset` | string | Non-empty; may contain template variables | Release asset name. | +| `checksum` | string | Lowercase 64-character SHA-256 value | Expected checksum for the asset. | +| `os` | string | Optional | Overrides the operating-system value used in templates. | +| `arch` | string | Optional | Overrides the architecture value used in templates. | + +### Templates + +Asset, download, checksum-file, and checksum-URL templates may use these variables: + +| Variable | Value | +|------------------|-----------------------------------------------------------| +| `{repository}` | `source.repository` | +| `{tag}` | `source.tagPrefix` plus the requested version | +| `{version}` | Requested version, including its leading `v` when present | +| `{version_no_v}` | Requested version without a leading `v` | +| `{asset}` | Expanded platform asset name | +| `{os}` | Platform OS, or the entry's `os` override | +| `{arch}` | Platform architecture, or the entry's `arch` override | + +Unknown or unterminated template variables are errors. + +### Version files + +A `versionFiles` entry keeps another repository file synchronized when a tool version changes: + +| Property | Type | Allowed or required values | Description | +|----------|--------|-------------------------------------------|---------------------------------------------------------------------------| +| `path` | string | Repository-relative path | File to update. Paths outside the repository are rejected. | +| `format` | string | `plain` or `replace` | Replace the whole file with the version, or replace text between markers. | +| `prefix` | string | Required for `replace`; empty for `plain` | Text immediately before the embedded version. | +| `suffix` | string | Required for `replace`; empty for `plain` | Text immediately after the embedded version. | + +The `replace` prefix must occur exactly once. Terraform uses `versionFiles` for `.terraform-version`, the Go fallback, and the Helm chart default. + +## Update behavior + +- The updater checks the latest version source for every tool. +- A version changes only when the source is a greater semantic version; downgrades are ignored. +- A tool with `update: false` remains pinned, but its source and current-version checks still run. +- Checksums are refreshed for every configured platform at the selected version. +- The manifest, generated Make include, Terraform compatibility file, and declared version consumers are updated only after source checks succeed. + +## Verification + +Run the focused tests and static checks after changing the updater or manifest: + +```sh +go test ./internal/tooling ./cmd/tool-updater +go vet ./internal/tooling ./cmd/tool-updater +make --no-print-directory -n update-tools +``` + +The Make dry run should invoke `bin/tool-updater` or `bin/tool-updater.exe`, not `go run`. + +## Troubleshooting + +- **Windows reports an elevation error for `go run`.** Use `make update-tools`, or build and run `bin/tool-updater.exe` directly. The Make target avoids Go's temporary executable directory. +- **A checksum is not found.** Check the release asset name, version/tag prefix, checksum-file template, and checksum format together. The updater matches the expanded asset name exactly. +- **Manifest validation fails.** Ensure every checksum-bearing tool defines every platform listed at the manifest's top level, every checksum is 64 lowercase hexadecimal characters, and any `versionFiles` path stays inside the repository. diff --git a/internal/tooling/manifest.go b/internal/tooling/manifest.go new file mode 100644 index 00000000000..f47bb46d17e --- /dev/null +++ b/internal/tooling/manifest.go @@ -0,0 +1,648 @@ +// Package tooling manages the pinned command-line tools used by Radius builds. +package tooling + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "go.yaml.in/yaml/v3" +) + +const manifestSchemaVersion = 1 + +var ( + platformPattern = regexp.MustCompile(`^(linux|darwin)_(amd64|arm64)$`) + namePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + makePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`) +) + +// Manifest is the source of truth for the versions, release sources, assets, +// and checksums of the external tools used by the repository. +type Manifest struct { + SchemaVersion int `yaml:"schemaVersion"` + Platforms []string `yaml:"platforms"` + Tools []Tool `yaml:"tools"` +} + +// Tool describes one pinned external tool. +type Tool struct { + Name string `yaml:"name"` + MakePrefix string `yaml:"makePrefix"` + Version string `yaml:"version"` + Update *bool `yaml:"update,omitempty"` + Notes string `yaml:"notes,omitempty"` + Source Source `yaml:"source"` + DownloadTemplate string `yaml:"downloadTemplate,omitempty"` + Platforms map[string]Platform `yaml:"platforms,omitempty"` + ChecksumSource ChecksumSource `yaml:"checksumSource"` + VersionFiles []VersionFile `yaml:"versionFiles,omitempty"` +} + +// Source describes how the updater discovers a tool's latest version. +type Source struct { + Type string `yaml:"type"` + Repository string `yaml:"repository,omitempty"` + TagPrefix string `yaml:"tagPrefix,omitempty"` + LatestURL string `yaml:"latestURL"` +} + +// Platform describes a release asset and its pinned checksum for one target +// platform. Asset names may contain manifest template variables. +type Platform struct { + Asset string `yaml:"asset"` + Checksum string `yaml:"checksum"` + OS string `yaml:"os,omitempty"` + Arch string `yaml:"arch,omitempty"` +} + +// VersionFile describes a repository file that must stay synchronized with a +// tool version when the tool is not only consumed through Make. +type VersionFile struct { + Path string `yaml:"path"` + Format string `yaml:"format"` + Prefix string `yaml:"prefix,omitempty"` + Suffix string `yaml:"suffix,omitempty"` +} + +// ChecksumSource describes where a release asset's SHA-256 checksum comes from. +type ChecksumSource struct { + Type string `yaml:"type"` + FileTemplate string `yaml:"fileTemplate,omitempty"` + OrderFileTemplate string `yaml:"orderFileTemplate,omitempty"` + URLTemplate string `yaml:"urlTemplate,omitempty"` + Format string `yaml:"format,omitempty"` + Integrity string `yaml:"integrity,omitempty"` +} + +// LoadManifest reads and validates a tool manifest. +func LoadManifest(path string) (Manifest, error) { + contents, err := os.ReadFile(path) + if err != nil { + return Manifest{}, fmt.Errorf("read tool manifest: %w", err) + } + + var manifest Manifest + decoder := yaml.NewDecoder(bytes.NewReader(contents)) + decoder.KnownFields(true) + if err := decoder.Decode(&manifest); err != nil { + return Manifest{}, fmt.Errorf("parse tool manifest: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return Manifest{}, fmt.Errorf("parse tool manifest: multiple YAML documents are not supported") + } + return Manifest{}, fmt.Errorf("parse trailing tool manifest content: %w", err) + } + if err := manifest.Validate(); err != nil { + return Manifest{}, fmt.Errorf("validate tool manifest: %w", err) + } + return manifest, nil +} + +// Validate checks that a manifest contains enough information for both Make +// generation and source verification. +func (m Manifest) Validate() error { + if m.SchemaVersion != manifestSchemaVersion { + return fmt.Errorf("unsupported schemaVersion %d", m.SchemaVersion) + } + if len(m.Platforms) == 0 { + return fmt.Errorf("at least one platform is required") + } + + seenPlatforms := make(map[string]struct{}, len(m.Platforms)) + for _, platform := range m.Platforms { + if !platformPattern.MatchString(platform) { + return fmt.Errorf("invalid platform %q", platform) + } + if _, exists := seenPlatforms[platform]; exists { + return fmt.Errorf("duplicate platform %q", platform) + } + seenPlatforms[platform] = struct{}{} + } + + if len(m.Tools) == 0 { + return fmt.Errorf("at least one tool is required") + } + seenNames := make(map[string]struct{}, len(m.Tools)) + seenPrefixes := make(map[string]struct{}, len(m.Tools)) + for _, tool := range m.Tools { + if !namePattern.MatchString(tool.Name) { + return fmt.Errorf("invalid tool name %q", tool.Name) + } + if !makePattern.MatchString(tool.MakePrefix) { + return fmt.Errorf("invalid Make prefix %q for %s", tool.MakePrefix, tool.Name) + } + if _, exists := seenNames[tool.Name]; exists { + return fmt.Errorf("duplicate tool %q", tool.Name) + } + if _, exists := seenPrefixes[tool.MakePrefix]; exists { + return fmt.Errorf("duplicate Make prefix %q", tool.MakePrefix) + } + seenNames[tool.Name] = struct{}{} + seenPrefixes[tool.MakePrefix] = struct{}{} + if strings.TrimSpace(tool.Version) == "" { + return fmt.Errorf("version is required for %s", tool.Name) + } + if strings.TrimSpace(tool.Source.Type) == "" || strings.TrimSpace(tool.Source.LatestURL) == "" { + return fmt.Errorf("source type and latestURL are required for %s", tool.Name) + } + if err := validateSource(tool); err != nil { + return err + } + if err := validateChecksumSource(tool); err != nil { + return err + } + if err := validateVersionFiles(tool); err != nil { + return err + } + + if tool.ChecksumSource.Type == "none" { + if len(tool.Platforms) != 0 { + return fmt.Errorf("tool %s has platforms but no checksum source", tool.Name) + } + continue + } + if strings.TrimSpace(tool.DownloadTemplate) == "" { + return fmt.Errorf("downloadTemplate is required for %s", tool.Name) + } + if len(tool.Platforms) != len(m.Platforms) { + return fmt.Errorf("tool %s must define every manifest platform", tool.Name) + } + for _, platform := range m.Platforms { + entry, exists := tool.Platforms[platform] + if !exists { + return fmt.Errorf("tool %s is missing platform %s", tool.Name, platform) + } + if strings.TrimSpace(entry.Asset) == "" { + return fmt.Errorf("tool %s has no asset for %s", tool.Name, platform) + } + if !hashPattern.MatchString(entry.Checksum) { + return fmt.Errorf("tool %s has invalid SHA-256 for %s", tool.Name, platform) + } + if err := validateToolURLs(tool, platform); err != nil { + return err + } + } + } + + return nil +} + +func validateToolURLs(tool Tool, platform string) error { + values, err := tool.TemplateValues(platform, tool.Version) + if err != nil { + return err + } + asset, err := ExpandTemplate(values["asset"], values) + if err != nil { + return fmt.Errorf("expand asset for %s %s: %w", tool.Name, platform, err) + } + values["asset"] = asset + + if err := validateHTTPSTemplate(tool.Name+" downloadTemplate", tool.DownloadTemplate, values); err != nil { + return err + } + if tool.ChecksumSource.Type == "url-file" { + if err := validateHTTPSTemplate(tool.Name+" checksumSource.urlTemplate", tool.ChecksumSource.URLTemplate, values); err != nil { + return err + } + } + return nil +} + +func validateHTTPSTemplate(name, template string, values map[string]string) error { + expanded, err := ExpandTemplate(template, values) + if err != nil { + return fmt.Errorf("expand %s: %w", name, err) + } + parsedURL, err := url.Parse(expanded) + if err != nil || parsedURL.Scheme != "https" || parsedURL.Host == "" { + return fmt.Errorf("%s must expand to a valid HTTPS URL", name) + } + return nil +} + +func validateSource(tool Tool) error { + switch tool.Source.Type { + case "github-release", "stable-text", "hashicorp-checkpoint": + default: + return fmt.Errorf("unsupported version source %q for %s", tool.Source.Type, tool.Name) + } + if tool.Source.Type == "github-release" && strings.TrimSpace(tool.Source.Repository) == "" { + return fmt.Errorf("repository is required for GitHub tool %s", tool.Name) + } + + parsedURL, err := url.Parse(tool.Source.LatestURL) + if err != nil || parsedURL.Scheme != "https" || parsedURL.Host == "" { + return fmt.Errorf("latestURL for %s must be a valid HTTPS URL", tool.Name) + } + return nil +} + +func validateChecksumSource(tool Tool) error { + source := tool.ChecksumSource + switch source.Type { + case "github-release-file": + if strings.TrimSpace(tool.Source.Repository) == "" { + return fmt.Errorf("repository is required for GitHub checksum source %s", tool.Name) + } + if source.FileTemplate == "" || source.Format == "" { + return fmt.Errorf("GitHub checksum file and format are required for %s", tool.Name) + } + if !isSupportedChecksumFormat(source.Format, true) { + return fmt.Errorf("unsupported checksum format %q for %s", source.Format, tool.Name) + } + if source.Format == "yq" && source.OrderFileTemplate == "" { + return fmt.Errorf("yq checksum order file is required for %s", tool.Name) + } + case "url-file": + if source.URLTemplate == "" || source.Format == "" { + return fmt.Errorf("checksum URL and format are required for %s", tool.Name) + } + if !isSupportedChecksumFormat(source.Format, false) { + return fmt.Errorf("unsupported checksum format %q for %s", source.Format, tool.Name) + } + case "download": + case "none": + if source.Integrity == "" { + return fmt.Errorf("integrity description is required for %s", tool.Name) + } + default: + return fmt.Errorf("unsupported checksum source %q for %s", source.Type, tool.Name) + } + return nil +} + +func isSupportedChecksumFormat(format string, allowYQ bool) bool { + switch format { + case "standard", "basename", "first": + return true + case "yq": + return allowYQ + default: + return false + } +} + +func validateVersionFiles(tool Tool) error { + for _, versionFile := range tool.VersionFiles { + cleanPath := filepath.ToSlash(filepath.Clean(versionFile.Path)) + if cleanPath == "." || cleanPath == ".." || strings.HasPrefix(cleanPath, "../") || filepath.IsAbs(versionFile.Path) { + return fmt.Errorf("version file path %q for %s must stay within the repository", versionFile.Path, tool.Name) + } + switch versionFile.Format { + case "plain": + if versionFile.Prefix != "" || versionFile.Suffix != "" { + return fmt.Errorf("plain version file %q for %s cannot have a prefix or suffix", versionFile.Path, tool.Name) + } + case "replace": + if versionFile.Prefix == "" || versionFile.Suffix == "" { + return fmt.Errorf("replace version file %q for %s needs a prefix and suffix", versionFile.Path, tool.Name) + } + default: + return fmt.Errorf("unsupported version file format %q for %s", versionFile.Format, tool.Name) + } + } + return nil +} + +// UpdatesEnabled reports whether the updater may advance the tool's version. +func (t Tool) UpdatesEnabled() bool { + return t.Update == nil || *t.Update +} + +// Platform returns a platform entry by name. +func (t Tool) Platform(name string) (Platform, bool) { + entry, ok := t.Platforms[name] + return entry, ok +} + +// TemplateValues builds the values used by asset and URL templates. +func (t Tool) TemplateValues(platform, version string) (map[string]string, error) { + entry, ok := t.Platform(platform) + if !ok { + return nil, fmt.Errorf("tool %s has no platform %s", t.Name, platform) + } + parts := strings.SplitN(platform, "_", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid platform %q", platform) + } + osName := entry.OS + if osName == "" { + osName = parts[0] + } + archName := entry.Arch + if archName == "" { + archName = parts[1] + } + return map[string]string{ + "repository": t.Source.Repository, + "tag": t.TagForVersion(version), + "version": version, + "version_no_v": strings.TrimPrefix(version, "v"), + "asset": entry.Asset, + "os": osName, + "arch": archName, + }, nil +} + +// TagForVersion converts a manifest version into its release tag. +func (t Tool) TagForVersion(version string) string { + return t.Source.TagPrefix + version +} + +// VersionFromTag converts a release tag into the manifest's version format. +func (t Tool) VersionFromTag(tag string) string { + return strings.TrimPrefix(tag, t.Source.TagPrefix) +} + +// ExpandTemplate expands the small, explicit template language used by the +// manifest and rejects unknown variables instead of silently producing a bad URL. +func ExpandTemplate(template string, values map[string]string) (string, error) { + var result strings.Builder + for index := 0; index < len(template); { + start := strings.IndexByte(template[index:], '{') + if start < 0 { + result.WriteString(template[index:]) + break + } + start += index + result.WriteString(template[index:start]) + end := strings.IndexByte(template[start+1:], '}') + if end < 0 { + return "", fmt.Errorf("unterminated template variable in %q", template) + } + end += start + 1 + key := template[start+1 : end] + value, ok := values[key] + if !ok { + return "", fmt.Errorf("unknown template variable %q", key) + } + result.WriteString(value) + index = end + 1 + } + return result.String(), nil +} + +// GenerateMake returns the generated Make include containing only metadata +// values. Tool installation recipes remain in build/tools.mk. +func GenerateMake(m Manifest) ([]byte, error) { + if err := m.Validate(); err != nil { + return nil, err + } + + var output bytes.Buffer + output.WriteString("# Code generated by cmd/tool-updater; DO NOT EDIT.\n\n") + for _, tool := range m.Tools { + fmt.Fprintf(&output, "%s_VERSION ?= %s\n", tool.MakePrefix, tool.Version) + for _, platform := range m.Platforms { + entry, ok := tool.Platform(platform) + if !ok || entry.Checksum == "" { + continue + } + name := strings.ToUpper(strings.ReplaceAll(platform, "-", "_")) + fmt.Fprintf(&output, "%s_CHECKSUM_%s ?= %s\n", tool.MakePrefix, name, entry.Checksum) + } + output.WriteByte('\n') + } + return output.Bytes(), nil +} + +// WriteMakeFile writes generated Make metadata only when its contents change. +func WriteMakeFile(path string, manifest Manifest) (bool, error) { + contents, err := GenerateMake(manifest) + if err != nil { + return false, fmt.Errorf("generate Make metadata: %w", err) + } + return writeIfChanged(path, contents) +} + +// WriteManifest patches tool versions and checksums into the manifest, +// preserving its existing formatting, and writes only when contents change. +func WriteManifest(path string, manifest Manifest) (bool, error) { + if err := manifest.Validate(); err != nil { + return false, err + } + contents, err := updateManifestYAML(path, manifest) + if err != nil { + return false, fmt.Errorf("update tool manifest: %w", err) + } + return writeIfChanged(path, contents) +} + +// scalarEdit is a manifest value to rewrite at its original position. +type scalarEdit struct { + node *yaml.Node + value string +} + +func updateManifestYAML(path string, manifest Manifest) ([]byte, error) { + contents, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } + + var document yaml.Node + if err := yaml.Unmarshal(contents, &document); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + if len(document.Content) != 1 { + return nil, fmt.Errorf("manifest must contain one YAML document") + } + root := document.Content[0] + toolsNode, err := mappingValue(root, "tools") + if err != nil { + return nil, err + } + if toolsNode.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("tools must be a YAML sequence") + } + + var edits []scalarEdit + for _, tool := range manifest.Tools { + toolNode, err := findToolNode(toolsNode, tool.Name) + if err != nil { + return nil, err + } + versionNode, err := mappingValue(toolNode, "version") + if err != nil { + return nil, err + } + edits = append(edits, scalarEdit{node: versionNode, value: tool.Version}) + + if tool.ChecksumSource.Type == "none" { + continue + } + platformsNode, err := mappingValue(toolNode, "platforms") + if err != nil { + return nil, err + } + for platform, entry := range tool.Platforms { + platformNode, err := mappingValue(platformsNode, platform) + if err != nil { + return nil, err + } + checksumNode, err := mappingValue(platformNode, "checksum") + if err != nil { + return nil, err + } + edits = append(edits, scalarEdit{node: checksumNode, value: entry.Checksum}) + } + } + return applyScalarEdits(contents, edits) +} + +// applyScalarEdits patches values in the original bytes. Re-serializing the +// parsed document instead would reindent every line and drop the document +// marker and blank lines, burying real updates in formatting noise. +func applyScalarEdits(contents []byte, edits []scalarEdit) ([]byte, error) { + lines := strings.Split(string(contents), "\n") + // Highest position first so edits sharing a line keep their offsets valid. + slices.SortFunc(edits, func(a, b scalarEdit) int { + if a.node.Line != b.node.Line { + return b.node.Line - a.node.Line + } + return b.node.Column - a.node.Column + }) + + for _, edit := range edits { + index := edit.node.Line - 1 + if index < 0 || index >= len(lines) { + return nil, fmt.Errorf("manifest line %d is out of range", edit.node.Line) + } + line := lines[index] + start := edit.node.Column - 1 + if edit.node.Style != 0 || start < 0 || start > len(line) || !strings.HasPrefix(line[start:], edit.node.Value) { + return nil, fmt.Errorf("manifest line %d must hold %q as a plain scalar", edit.node.Line, edit.node.Value) + } + lines[index] = line[:start] + edit.value + line[start+len(edit.node.Value):] + } + return []byte(strings.Join(lines, "\n")), nil +} + +func findToolNode(toolsNode *yaml.Node, name string) (*yaml.Node, error) { + for _, toolNode := range toolsNode.Content { + if toolNode.Kind != yaml.MappingNode { + continue + } + nameNode, err := mappingValue(toolNode, "name") + if err != nil { + return nil, err + } + if nameNode.Value == name { + return toolNode, nil + } + } + return nil, fmt.Errorf("tool %s is missing from the manifest YAML", name) +} + +func mappingValue(node *yaml.Node, key string) (*yaml.Node, error) { + if node.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected YAML mapping while looking for %s", key) + } + for index := 0; index+1 < len(node.Content); index += 2 { + if node.Content[index].Value == key { + return node.Content[index+1], nil + } + } + return nil, fmt.Errorf("YAML key %s is missing", key) +} + +// WriteTextFile writes a text file only when its contents change. +func WriteTextFile(path, contents string) (bool, error) { + return writeIfChanged(path, []byte(contents)) +} + +// SyncVersionFiles updates the declared consumers of a tool version. Plain +// files are replaced completely; replace files update the value between their +// declared prefix and suffix exactly once. +func SyncVersionFiles(root string, tool Tool) error { + for _, versionFile := range tool.VersionFiles { + filePath := filepath.Join(root, filepath.FromSlash(versionFile.Path)) + contents, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read version consumer %s: %w", versionFile.Path, err) + } + + var updated string + text := string(contents) + switch versionFile.Format { + case "plain": + updated = tool.Version + "\n" + case "replace": + if strings.Count(text, versionFile.Prefix) != 1 { + return fmt.Errorf("version prefix for %s must occur exactly once in %s", tool.Name, versionFile.Path) + } + prefixStart := strings.Index(text, versionFile.Prefix) + valueStart := prefixStart + len(versionFile.Prefix) + suffixOffset := strings.Index(text[valueStart:], versionFile.Suffix) + if suffixOffset < 0 { + return fmt.Errorf("version suffix for %s was not found in %s", tool.Name, versionFile.Path) + } + valueEnd := valueStart + suffixOffset + updated = text[:valueStart] + tool.Version + text[valueEnd:] + default: + return fmt.Errorf("unsupported version file format %q", versionFile.Format) + } + if _, err := WriteTextFile(filePath, updated); err != nil { + return fmt.Errorf("write version consumer %s: %w", versionFile.Path, err) + } + } + return nil +} + +func writeIfChanged(path string, contents []byte) (bool, error) { + existing, err := os.ReadFile(path) + if err == nil && bytes.Equal(existing, contents) { + return false, nil + } + if err != nil && !os.IsNotExist(err) { + return false, fmt.Errorf("read %s: %w", path, err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return false, fmt.Errorf("create directory for %s: %w", path, err) + } + + temporary, err := os.CreateTemp(filepath.Dir(path), ".tooling-*") + if err != nil { + return false, fmt.Errorf("create temporary file for %s: %w", path, err) + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if _, err := temporary.Write(contents); err != nil { + temporary.Close() + return false, fmt.Errorf("write temporary file for %s: %w", path, err) + } + if err := temporary.Close(); err != nil { + return false, fmt.Errorf("close temporary file for %s: %w", path, err) + } + if err := replaceFile(temporaryPath, path); err != nil { + return false, fmt.Errorf("replace %s: %w", path, err) + } + return true, nil +} + +func replaceFile(source, destination string) error { + err := os.Rename(source, destination) + if err == nil { + return nil + } + if !errors.Is(err, fs.ErrExist) { + return err + } + if removeErr := os.Remove(destination); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) { + return fmt.Errorf("remove existing destination: %w", removeErr) + } + return os.Rename(source, destination) +} diff --git a/internal/tooling/manifest_test.go b/internal/tooling/manifest_test.go new file mode 100644 index 00000000000..f8978ee5d4c --- /dev/null +++ b/internal/tooling/manifest_test.go @@ -0,0 +1,401 @@ +package tooling + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestLoadRepositoryManifest(t *testing.T) { + buildPath := filepath.Join("..", "..", "build") + manifestPath := filepath.Join(buildPath, "tools.yaml") + manifest, err := LoadManifest(manifestPath) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + + contents, err := GenerateMake(manifest) + if err != nil { + t.Fatalf("GenerateMake() error = %v", err) + } + committed, err := os.ReadFile(filepath.Join(buildPath, "tools.generated.mk")) + if err != nil { + t.Fatalf("read committed Make metadata: %v", err) + } + if string(contents) != string(committed) { + t.Fatal("generated Make metadata differs from build/tools.generated.mk; run make update-tools") + } +} + +func TestLoadManifestRejectsUnknownFields(t *testing.T) { + contents, err := os.ReadFile(filepath.Join("..", "..", "build", "tools.yaml")) + if err != nil { + t.Fatal(err) + } + invalid := strings.Replace(string(contents), "update: false", "updat: false", 1) + if invalid == string(contents) { + t.Fatal("test fixture does not contain update: false") + } + path := filepath.Join(t.TempDir(), "tools.yaml") + if err := os.WriteFile(path, []byte(invalid), 0o644); err != nil { + t.Fatal(err) + } + + _, err = LoadManifest(path) + if err == nil || !strings.Contains(err.Error(), "field updat not found") { + t.Fatalf("LoadManifest() error = %v, want unknown-field error", err) + } +} + +func TestLoadManifestRejectsMultipleDocuments(t *testing.T) { + contents, err := os.ReadFile(filepath.Join("..", "..", "build", "tools.yaml")) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "tools.yaml") + if err := os.WriteFile(path, append(contents, []byte("\n---\nextra: document\n")...), 0o644); err != nil { + t.Fatal(err) + } + + _, err = LoadManifest(path) + if err == nil || !strings.Contains(err.Error(), "multiple YAML documents") { + t.Fatalf("LoadManifest() error = %v, want multiple-document error", err) + } +} + +func TestManifestValidationRejectsInvalidSourcesAndFormats(t *testing.T) { + tests := []struct { + name string + setup func(*Manifest) + want string + }{ + { + name: "unsupported source type", + setup: func(manifest *Manifest) { + manifest.Tools[0].Source.Type = "github" + }, + want: "unsupported version source", + }, + { + name: "insecure latest URL", + setup: func(manifest *Manifest) { + manifest.Tools[0].Source.LatestURL = "http://example.test/latest" + }, + want: "valid HTTPS URL", + }, + { + name: "insecure download URL", + setup: func(manifest *Manifest) { + manifest.Tools[0].DownloadTemplate = "http://example.test/{version}/{asset}" + }, + want: "downloadTemplate must expand to a valid HTTPS URL", + }, + { + name: "insecure checksum URL", + setup: func(manifest *Manifest) { + manifest.Tools[0].ChecksumSource.URLTemplate = "http://example.test/checksums" + }, + want: "checksumSource.urlTemplate must expand to a valid HTTPS URL", + }, + { + name: "unsupported checksum format", + setup: func(manifest *Manifest) { + manifest.Tools[0].ChecksumSource.Format = "md5" + }, + want: "unsupported checksum format", + }, + { + name: "GitHub checksum source without repository", + setup: func(manifest *Manifest) { + manifest.Tools[0].ChecksumSource = ChecksumSource{ + Type: "github-release-file", + FileTemplate: "checksums.txt", + Format: "standard", + } + }, + want: "repository is required for GitHub checksum source", + }, + { + name: "yq format for URL checksum source", + setup: func(manifest *Manifest) { + manifest.Tools[0].ChecksumSource.Format = "yq" + }, + want: "unsupported checksum format", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := validManifestForValidation() + test.setup(&manifest) + err := manifest.Validate() + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Validate() error = %v, want substring %q", err, test.want) + } + }) + } +} + +func validManifestForValidation() Manifest { + return Manifest{ + SchemaVersion: 1, + Platforms: []string{"linux_amd64"}, + Tools: []Tool{{ + Name: "tool", + MakePrefix: "TOOL", + Version: "v1.0.0", + Source: Source{ + Type: "stable-text", + LatestURL: "https://example.test/latest", + }, + DownloadTemplate: "https://example.test/{version}/{asset}", + Platforms: map[string]Platform{ + "linux_amd64": { + Asset: "tool", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + ChecksumSource: ChecksumSource{ + Type: "url-file", + URLTemplate: "https://example.test/checksums", + Format: "standard", + }, + }}, + } +} + +func TestExpandTemplate(t *testing.T) { + values := map[string]string{ + "version": "v4.2.3", + "asset": "helm-v4.2.3-linux-amd64.tar.gz", + } + got, err := ExpandTemplate("https://example.test/{version}/{asset}", values) + if err != nil { + t.Fatalf("ExpandTemplate() error = %v", err) + } + want := "https://example.test/v4.2.3/helm-v4.2.3-linux-amd64.tar.gz" + if got != want { + t.Fatalf("ExpandTemplate() = %q, want %q", got, want) + } + + if _, err := ExpandTemplate("{missing}", values); err == nil { + t.Fatal("ExpandTemplate() accepted an unknown variable") + } +} + +func TestParseChecksum(t *testing.T) { + tests := []struct { + name string + format string + content string + asset string + want string + }{ + { + name: "standard", + format: "standard", + content: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef tool.tar.gz\n", + asset: "tool.tar.gz", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }, + { + name: "basename", + format: "basename", + content: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef _dist/tool\n", + asset: "tool", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }, + { + name: "first", + format: "first", + content: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef tool\n", + asset: "tool", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := parseChecksum([]byte(test.content), test.format, test.asset) + if err != nil { + t.Fatalf("parseChecksum() error = %v", err) + } + if got != test.want { + t.Fatalf("parseChecksum() = %q, want %q", got, test.want) + } + }) + } +} + +func TestParseYQChecksum(t *testing.T) { + checksum := "asset 1111111111111111111111111111111111111111111111111111111111111111 2222222222222222222222222222222222222222222222222222222222222222\n" + got, err := parseYQChecksum([]byte("MD5\nSHA-256\n"), []byte(checksum), "asset") + if err != nil { + t.Fatalf("parseYQChecksum() error = %v", err) + } + want := "2222222222222222222222222222222222222222222222222222222222222222" + if got != want { + t.Fatalf("parseYQChecksum() = %q, want %q", got, want) + } +} + +func TestSyncVersionFiles(t *testing.T) { + root := t.TempDir() + goFile := filepath.Join(root, "version.go") + chartFile := filepath.Join(root, "values.yaml") + if err := os.WriteFile(goFile, []byte("var terraformVersion = \"1.0.0\"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(chartFile, []byte(" version: \"1.0.0\"\n"), 0o644); err != nil { + t.Fatal(err) + } + + tool := Tool{ + Name: "terraform", + Version: "1.1.0", + VersionFiles: []VersionFile{ + { + Path: "version.go", + Format: "replace", + Prefix: `var terraformVersion = "`, + Suffix: `"`, + }, + { + Path: "values.yaml", + Format: "replace", + Prefix: ` version: "`, + Suffix: `"`, + }, + }, + } + if err := SyncVersionFiles(root, tool); err != nil { + t.Fatalf("SyncVersionFiles() error = %v", err) + } + + for _, path := range []string{goFile, chartFile} { + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(contents), "1.1.0") { + t.Errorf("%s does not contain the updated version: %s", path, contents) + } + } +} + +func TestWriteManifestPreservesComments(t *testing.T) { + path := filepath.Join(t.TempDir(), "tools.yaml") + contents := `schemaVersion: 1 +platforms: + - linux_amd64 +tools: + - name: tool + makePrefix: TOOL + version: v1.0.0 # keep this inline note + source: + type: stable-text + latestURL: https://example.test/stable + downloadTemplate: https://example.test/{version} + platforms: + linux_amd64: + asset: tool + checksum: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + checksumSource: + type: download + # Keep the reason for this tool close to its metadata. + notes: important tool +` + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatal(err) + } + manifest, err := LoadManifest(path) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + manifest.Tools[0].Version = "v1.1.0" + manifest.Tools[0].Platforms["linux_amd64"] = Platform{ + Asset: "tool", + Checksum: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + } + if _, err := WriteManifest(path, manifest); err != nil { + t.Fatalf("WriteManifest() error = %v", err) + } + updated, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + text := string(updated) + for _, expected := range []string{ + "version: v1.1.0 # keep this inline note", + "checksum: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "# Keep the reason for this tool close to its metadata.", + } { + if !strings.Contains(text, expected) { + t.Errorf("updated manifest does not contain %q:\n%s", expected, text) + } + } +} + +func TestUpdateManifestYAMLOnlyRewritesChangedValues(t *testing.T) { + path := filepath.Join("..", "..", "build", "tools.yaml") + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + manifest, err := LoadManifest(path) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + + const newVersion = "v99.99.99" + const newChecksum = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + tool := &manifest.Tools[0] + platform := manifest.Platforms[0] + entry := tool.Platforms[platform] + entry.Checksum = newChecksum + tool.Platforms[platform] = entry + tool.Version = newVersion + + updated, err := updateManifestYAML(path, manifest) + if err != nil { + t.Fatalf("updateManifestYAML() error = %v", err) + } + + originalLines := strings.Split(string(original), "\n") + updatedLines := strings.Split(string(updated), "\n") + if len(originalLines) != len(updatedLines) { + t.Fatalf("line count changed from %d to %d; the manifest was reformatted", len(originalLines), len(updatedLines)) + } + var changed []string + for index := range originalLines { + if originalLines[index] != updatedLines[index] { + changed = append(changed, strings.TrimSpace(updatedLines[index])) + } + } + want := []string{"version: " + newVersion, "checksum: " + newChecksum} + if !slices.Equal(changed, want) { + t.Fatalf("changed lines = %q, want only %q", changed, want) + } +} + +func TestReplaceFilePreservesDestinationForUnrelatedRenameError(t *testing.T) { + directory := t.TempDir() + destination := filepath.Join(directory, "destination") + if err := os.WriteFile(destination, []byte("keep"), 0o644); err != nil { + t.Fatal(err) + } + + err := replaceFile(filepath.Join(directory, "missing"), destination) + if err == nil { + t.Fatal("replaceFile() error = nil, want rename error") + } + contents, readErr := os.ReadFile(destination) + if readErr != nil { + t.Fatalf("read destination after failed rename: %v", readErr) + } + if string(contents) != "keep" { + t.Fatalf("destination contents = %q, want %q", contents, "keep") + } +} diff --git a/internal/tooling/updater.go b/internal/tooling/updater.go new file mode 100644 index 00000000000..8e44364c12a --- /dev/null +++ b/internal/tooling/updater.go @@ -0,0 +1,386 @@ +package tooling + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/Masterminds/semver/v3" +) + +const ( + requestAttempts = 3 + maximumResponse = 128 << 20 +) + +// HTTPClient is the subset of http.Client used by the updater. +type HTTPClient interface { + Do(request *http.Request) (*http.Response, error) +} + +// Client retrieves release metadata and checksum sources. +type Client struct { + HTTP HTTPClient + Token string + UserAgent string + fileCache map[string][]byte +} + +// NewClient constructs a source client. GITHUB_TOKEN and GH_TOKEN are honored +// to keep local and GitHub Actions runs equivalent. +func NewClient(token string) *Client { + if token == "" { + token = os.Getenv("GITHUB_TOKEN") + } + if token == "" { + token = os.Getenv("GH_TOKEN") + } + httpClient := &http.Client{ + Timeout: 90 * time.Second, + CheckRedirect: func(request *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + if request.URL.Scheme != "https" || request.URL.Hostname() != "api.github.com" { + request.Header.Del("Authorization") + } + return nil + }, + } + return &Client{ + HTTP: httpClient, + Token: token, + UserAgent: "radius-tool-updater", + fileCache: map[string][]byte{}, + } +} + +// UpdateManifest refreshes versions and checksums in memory. It writes no +// files, so a failed source lookup cannot leave a partially updated manifest. +func UpdateManifest(ctx context.Context, manifest *Manifest, client *Client) ([]string, error) { + if err := manifest.Validate(); err != nil { + return nil, err + } + if client == nil || client.HTTP == nil { + return nil, fmt.Errorf("an HTTP client is required") + } + + var changes []string + for index := range manifest.Tools { + tool := &manifest.Tools[index] + targetVersion := tool.Version + latest, err := client.LatestVersion(ctx, *tool) + if err != nil { + return nil, fmt.Errorf("check %s version: %w", tool.Name, err) + } + newer, err := newerVersion(latest, tool.Version) + if err != nil { + return nil, fmt.Errorf("compare %s versions: %w", tool.Name, err) + } + if newer && tool.UpdatesEnabled() { + changes = append(changes, fmt.Sprintf("%s version %s -> %s", tool.Name, tool.Version, latest)) + targetVersion = latest + } + + updatedChecksums := make(map[string]string, len(tool.Platforms)) + for _, platform := range manifest.Platforms { + if len(tool.Platforms) == 0 { + break + } + checksum, err := client.Checksum(ctx, *tool, platform, targetVersion) + if err != nil { + return nil, fmt.Errorf("check %s %s checksum: %w", tool.Name, platform, err) + } + updatedChecksums[platform] = checksum + if checksum != tool.Platforms[platform].Checksum { + changes = append(changes, fmt.Sprintf("%s %s checksum refreshed", tool.Name, platform)) + } + } + + if targetVersion != tool.Version { + tool.Version = targetVersion + } + for platform, checksum := range updatedChecksums { + entry := tool.Platforms[platform] + entry.Checksum = checksum + tool.Platforms[platform] = entry + } + } + return changes, nil +} + +// LatestVersion resolves the latest stable version for a tool source. +func (client *Client) LatestVersion(ctx context.Context, tool Tool) (string, error) { + contents, err := client.get(ctx, tool.Source.LatestURL) + if err != nil { + return "", err + } + + switch tool.Source.Type { + case "github-release": + var release struct { + TagName string `json:"tag_name"` + } + if err := json.Unmarshal(contents, &release); err != nil { + return "", fmt.Errorf("parse GitHub release: %w", err) + } + if release.TagName == "" { + return "", fmt.Errorf("GitHub release has no tag") + } + return tool.VersionFromTag(release.TagName), nil + case "stable-text": + version := strings.TrimSpace(string(contents)) + if version == "" { + return "", fmt.Errorf("stable version response is empty") + } + return version, nil + case "hashicorp-checkpoint": + var checkpoint struct { + CurrentVersion string `json:"current_version"` + } + if err := json.Unmarshal(contents, &checkpoint); err != nil { + return "", fmt.Errorf("parse HashiCorp checkpoint: %w", err) + } + if checkpoint.CurrentVersion == "" { + return "", fmt.Errorf("HashiCorp checkpoint has no current_version") + } + return checkpoint.CurrentVersion, nil + default: + return "", fmt.Errorf("unsupported version source %q", tool.Source.Type) + } +} + +// Checksum reads or computes the checksum for one target platform. +func (client *Client) Checksum(ctx context.Context, tool Tool, platform, version string) (string, error) { + values, err := tool.TemplateValues(platform, version) + if err != nil { + return "", err + } + asset, err := ExpandTemplate(values["asset"], values) + if err != nil { + return "", fmt.Errorf("expand asset: %w", err) + } + values["asset"] = asset + + switch tool.ChecksumSource.Type { + case "github-release-file": + file, err := ExpandTemplate(tool.ChecksumSource.FileTemplate, values) + if err != nil { + return "", fmt.Errorf("expand checksum file: %w", err) + } + fileURL := fmt.Sprintf("https://github.com/%s/releases/download/%s/%s", tool.Source.Repository, values["tag"], file) + contents, err := client.getFile(ctx, fileURL) + if err != nil { + return "", err + } + if tool.ChecksumSource.Format == "yq" { + orderFile, err := ExpandTemplate(tool.ChecksumSource.OrderFileTemplate, values) + if err != nil { + return "", fmt.Errorf("expand checksum order file: %w", err) + } + orderURL := fmt.Sprintf("https://github.com/%s/releases/download/%s/%s", tool.Source.Repository, values["tag"], orderFile) + orderContents, err := client.getFile(ctx, orderURL) + if err != nil { + return "", err + } + return parseYQChecksum(orderContents, contents, asset) + } + return parseChecksum(contents, tool.ChecksumSource.Format, asset) + case "url-file": + url, err := ExpandTemplate(tool.ChecksumSource.URLTemplate, values) + if err != nil { + return "", fmt.Errorf("expand checksum URL: %w", err) + } + contents, err := client.getFile(ctx, url) + if err != nil { + return "", err + } + return parseChecksum(contents, tool.ChecksumSource.Format, asset) + case "download": + url, err := ExpandTemplate(tool.DownloadTemplate, values) + if err != nil { + return "", fmt.Errorf("expand download URL: %w", err) + } + return client.hash(ctx, url) + case "none": + return "", nil + default: + return "", fmt.Errorf("unsupported checksum source %q", tool.ChecksumSource.Type) + } +} + +// getFile fetches a small checksum metadata file, caching it for the lifetime +// of the client so a shared checksums file is fetched once per run instead of +// once per platform. Binary downloads (the "download" checksum source) bypass +// this cache to avoid holding large blobs in memory. +func (client *Client) getFile(ctx context.Context, url string) ([]byte, error) { + if contents, ok := client.fileCache[url]; ok { + return contents, nil + } + contents, err := client.get(ctx, url) + if err != nil { + return nil, err + } + if client.fileCache != nil { + client.fileCache[url] = contents + } + return contents, nil +} + +func (client *Client) get(ctx context.Context, url string) ([]byte, error) { + var contents bytes.Buffer + if err := client.writeResponse(ctx, url, &contents); err != nil { + return nil, err + } + return contents.Bytes(), nil +} + +func (client *Client) hash(ctx context.Context, url string) (string, error) { + hasher := sha256.New() + if err := client.writeResponse(ctx, url, hasher); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (client *Client) writeResponse(ctx context.Context, url string, destination io.Writer) error { + var lastError error + for attempt := 0; attempt < requestAttempts; attempt++ { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create request for %s: %w", url, err) + } + request.Header.Set("User-Agent", client.UserAgent) + isGitHubAPI := request.URL.Scheme == "https" && request.URL.Hostname() == "api.github.com" + if isGitHubAPI { + request.Header.Set("Accept", "application/vnd.github+json") + } + if client.Token != "" && isGitHubAPI { + request.Header.Set("Authorization", "Bearer "+client.Token) + } + + response, err := client.HTTP.Do(request) + if err != nil { + lastError = err + continue + } + if response.StatusCode >= http.StatusOK && response.StatusCode < http.StatusMultipleChoices { + // io.Copy can fail on either side, so the message stays neutral. + written, copyErr := io.Copy(destination, io.LimitReader(response.Body, maximumResponse+1)) + closeErr := response.Body.Close() + if copyErr != nil { + return fmt.Errorf("copy response from %s: %w", url, copyErr) + } + if closeErr != nil { + return fmt.Errorf("close response from %s: %w", url, closeErr) + } + if written > maximumResponse { + return fmt.Errorf("response from %s exceeds %d bytes", url, maximumResponse) + } + return nil + } + + contents, readErr := io.ReadAll(io.LimitReader(response.Body, maximumResponse+1)) + closeErr := response.Body.Close() + if readErr != nil { + return fmt.Errorf("read %s: %w", url, readErr) + } + if closeErr != nil { + return fmt.Errorf("close response from %s: %w", url, closeErr) + } + if len(contents) > maximumResponse { + return fmt.Errorf("response from %s exceeds %d bytes", url, maximumResponse) + } + lastError = fmt.Errorf("HTTP %s from %s: %s", response.Status, url, strings.TrimSpace(string(contents))) + if response.StatusCode < 500 && response.StatusCode != http.StatusTooManyRequests { + break + } + if attempt < requestAttempts-1 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt+1) * 500 * time.Millisecond): + } + } + } + return lastError +} + +func newerVersion(candidate, current string) (bool, error) { + candidateVersion, err := semver.NewVersion(strings.TrimPrefix(candidate, "v")) + if err != nil { + return false, fmt.Errorf("parse candidate %q: %w", candidate, err) + } + currentVersion, err := semver.NewVersion(strings.TrimPrefix(current, "v")) + if err != nil { + return false, fmt.Errorf("parse current %q: %w", current, err) + } + return candidateVersion.GreaterThan(currentVersion), nil +} + +func parseChecksum(contents []byte, format, asset string) (string, error) { + lines := strings.Split(string(contents), "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + if format == "first" { + return validateHash(fields[0]) + } + if len(fields) < 2 { + continue + } + filename := strings.TrimPrefix(fields[1], "*") + if format == "basename" { + filename = path.Base(filename) + } + if filename == asset { + return validateHash(fields[0]) + } + } + return "", fmt.Errorf("checksum for %s not found", asset) +} + +func parseYQChecksum(orderContents, checksumContents []byte, asset string) (string, error) { + orderLines := strings.Split(string(orderContents), "\n") + column := 0 + for index, line := range orderLines { + if strings.TrimSpace(line) == "SHA-256" { + column = index + 1 + break + } + } + if column == 0 { + return "", fmt.Errorf("SHA-256 column not found in checksums_hashes_order") + } + + for _, line := range strings.Split(string(checksumContents), "\n") { + fields := strings.Fields(line) + if len(fields) > column && fields[0] == asset { + return validateHash(fields[column]) + } + } + return "", fmt.Errorf("checksum for %s not found", asset) +} + +func validateHash(value string) (string, error) { + value = strings.ToLower(strings.TrimSpace(value)) + if len(value) != sha256.Size*2 { + return "", fmt.Errorf("invalid SHA-256 value %q", value) + } + if _, err := hex.DecodeString(value); err != nil { + return "", fmt.Errorf("invalid SHA-256 value %q: %w", value, err) + } + return value, nil +} diff --git a/internal/tooling/updater_test.go b/internal/tooling/updater_test.go new file mode 100644 index 00000000000..4c27d099493 --- /dev/null +++ b/internal/tooling/updater_test.go @@ -0,0 +1,259 @@ +package tooling + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type httpClientFunc func(*http.Request) (*http.Response, error) + +func (function httpClientFunc) Do(request *http.Request) (*http.Response, error) { + return function(request) +} + +func TestUpdateManifestRefreshesVersionAndChecksum(t *testing.T) { + const checksum = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/latest": + fmt.Fprint(response, `{"tag_name":"v2.0.0"}`) + case "/checksums/v2.0.0/tool-v2.0.0.tar.gz": + fmt.Fprintf(response, "%s tool-v2.0.0.tar.gz\n", checksum) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + + manifest := Manifest{ + SchemaVersion: 1, + Platforms: []string{"linux_amd64"}, + Tools: []Tool{{ + Name: "tool", + MakePrefix: "TOOL", + Version: "v1.0.0", + Source: Source{ + Type: "github-release", + Repository: "example/tool", + LatestURL: server.URL + "/latest", + }, + DownloadTemplate: server.URL + "/download/{tag}/{asset}", + Platforms: map[string]Platform{ + "linux_amd64": { + Asset: "tool-{version}.tar.gz", + Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + ChecksumSource: ChecksumSource{ + Type: "url-file", + URLTemplate: server.URL + "/checksums/{version}/{asset}", + Format: "standard", + }, + }}, + } + + client := NewClient("") + client.HTTP = server.Client() + changes, err := UpdateManifest(context.Background(), &manifest, client) + if err != nil { + t.Fatalf("UpdateManifest() error = %v", err) + } + if len(changes) != 2 { + t.Fatalf("got %d changes, want version and checksum changes", len(changes)) + } + if got := manifest.Tools[0].Version; got != "v2.0.0" { + t.Fatalf("version = %q, want v2.0.0", got) + } + if got := manifest.Tools[0].Platforms["linux_amd64"].Checksum; got != checksum { + t.Fatalf("checksum = %q, want %q", got, checksum) + } +} + +func TestNewerVersionDoesNotDowngrade(t *testing.T) { + newer, err := newerVersion("v1.9.0", "v2.0.0") + if err != nil { + t.Fatalf("newerVersion() error = %v", err) + } + if newer { + t.Fatal("newerVersion() reported a downgrade as an upgrade") + } +} + +func TestClientOnlyAuthenticatesExactGitHubAPIHost(t *testing.T) { + tests := []struct { + name string + url string + wantAuthorization string + wantAccept string + }{ + { + name: "GitHub API", + url: "https://api.github.com/repos/example/tool/releases/latest", + wantAuthorization: "Bearer secret", + wantAccept: "application/vnd.github+json", + }, + { + name: "lookalike host", + url: "https://api.github.com.attacker.example/releases/latest", + }, + { + name: "GitHub release download", + url: "https://github.com/example/tool/releases/latest", + }, + { + name: "insecure GitHub API URL", + url: "http://api.github.com/repos/example/tool/releases/latest", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := NewClient("secret") + client.HTTP = httpClientFunc(func(request *http.Request) (*http.Response, error) { + if got := request.Header.Get("Authorization"); got != test.wantAuthorization { + t.Errorf("Authorization = %q, want %q", got, test.wantAuthorization) + } + if got := request.Header.Get("Accept"); got != test.wantAccept { + t.Errorf("Accept = %q, want %q", got, test.wantAccept) + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("{}")), + }, nil + }) + if _, err := client.get(context.Background(), test.url); err != nil { + t.Fatalf("get() error = %v", err) + } + }) + } +} + +func TestClientStripsAuthorizationOnRedirectAwayFromGitHubAPI(t *testing.T) { + client := NewClient("secret") + httpClient, ok := client.HTTP.(*http.Client) + if !ok { + t.Fatalf("HTTP client has type %T, want *http.Client", client.HTTP) + } + request, err := http.NewRequest(http.MethodGet, "https://uploads.github.com/object", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer secret") + if err := httpClient.CheckRedirect(request, nil); err != nil { + t.Fatalf("CheckRedirect() error = %v", err) + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization after redirect = %q, want empty", got) + } +} + +func TestClientLimitsRedirects(t *testing.T) { + client := NewClient("secret") + httpClient, ok := client.HTTP.(*http.Client) + if !ok { + t.Fatalf("HTTP client has type %T, want *http.Client", client.HTTP) + } + request, err := http.NewRequest(http.MethodGet, "https://api.github.com/redirect", nil) + if err != nil { + t.Fatal(err) + } + via := make([]*http.Request, 10) + if err := httpClient.CheckRedirect(request, via); err == nil { + t.Fatal("CheckRedirect() accepted more than 10 redirects") + } +} + +func TestChecksumCachesSharedFile(t *testing.T) { + const ( + amd64Sum = "1111111111111111111111111111111111111111111111111111111111111111" + arm64Sum = "2222222222222222222222222222222222222222222222222222222222222222" + ) + + var fetches int + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/checksums" { + fetches++ + fmt.Fprintf(response, "%s tool_linux_amd64\n%s tool_linux_arm64\n", amd64Sum, arm64Sum) + return + } + http.NotFound(response, request) + })) + defer server.Close() + + tool := Tool{ + Name: "tool", + MakePrefix: "TOOL", + Version: "v1.0.0", + DownloadTemplate: server.URL + "/download/{asset}", + Source: Source{ + Type: "github-release", + Repository: "example/tool", + LatestURL: server.URL + "/latest", + }, + Platforms: map[string]Platform{ + "linux_amd64": {Asset: "tool_linux_amd64", Checksum: amd64Sum}, + "linux_arm64": {Asset: "tool_linux_arm64", Checksum: arm64Sum}, + }, + ChecksumSource: ChecksumSource{ + Type: "url-file", + URLTemplate: server.URL + "/checksums", + Format: "standard", + }, + } + + client := NewClient("") + client.HTTP = server.Client() + + for _, platform := range []string{"linux_amd64", "linux_arm64"} { + if _, err := client.Checksum(context.Background(), tool, platform, tool.Version); err != nil { + t.Fatalf("Checksum(%s) error = %v", platform, err) + } + } + if fetches != 1 { + t.Fatalf("shared checksum file fetched %d times, want 1", fetches) + } +} + +func TestChecksumStreamsDownloadedAsset(t *testing.T) { + const contents = "downloaded tool contents" + want := fmt.Sprintf("%x", sha256.Sum256([]byte(contents))) + + var attempts int + client := NewClient("") + client.HTTP = httpClientFunc(func(request *http.Request) (*http.Response, error) { + attempts++ + if attempts == 1 { + return nil, fmt.Errorf("temporary download error") + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(contents)), + }, nil + }) + tool := Tool{ + Version: "v1.0.0", + DownloadTemplate: "https://downloads.example.test/{asset}", + Platforms: map[string]Platform{ + "linux_amd64": {Asset: "tool_linux_amd64"}, + }, + ChecksumSource: ChecksumSource{Type: "download"}, + } + + got, err := client.Checksum(context.Background(), tool, "linux_amd64", tool.Version) + if err != nil { + t.Fatalf("Checksum() error = %v", err) + } + if got != want { + t.Fatalf("Checksum() = %q, want %q", got, want) + } + if attempts != 2 { + t.Fatalf("download attempts = %d, want 2", attempts) + } +} diff --git a/pkg/recipes/terraform/version.go b/pkg/recipes/terraform/version.go index 759c2c25c5b..de142b37d98 100644 --- a/pkg/recipes/terraform/version.go +++ b/pkg/recipes/terraform/version.go @@ -19,14 +19,13 @@ package terraform // terraformVersion is the version of Terraform that Radius downloads and // uses to execute Terraform recipes. // -// The canonical source of truth is the .terraform-version file at the -// repository root (matching the .node-version / .python-version -// convention used by tfenv, tfswitch, asdf, and mise). The Makefile reads -// that file and overrides this value at build time via the -// -X linker flag. The hard-coded default below is used by `go test`, -// `go run`, and other invocations that do not go through the Makefile; -// the TestTerraformVersionMatchesFile test guarantees it stays in sync -// with the file. +// The canonical source of truth is the build/tools.yaml manifest. The updater +// synchronizes the .terraform-version compatibility file, chart default, and +// this fallback with that manifest. The Makefile also overrides this value at +// build time via the -X linker flag. The hard-coded default below is used by +// `go test`, `go run`, and other invocations that do not go through the +// Makefile; the TestTerraformVersionMatchesFile test guarantees it stays in +// sync with the compatibility file. var terraformVersion = "1.14.9" // TerraformVersion returns the Terraform version Radius will install. diff --git a/pkg/recipes/terraform/version_test.go b/pkg/recipes/terraform/version_test.go index b7812760698..4150f716002 100644 --- a/pkg/recipes/terraform/version_test.go +++ b/pkg/recipes/terraform/version_test.go @@ -27,9 +27,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestTerraformVersionMatchesFile guarantees that the hard-coded -// terraformVersion default in version.go stays in sync with the -// .terraform-version file at the repository root. +// TestTerraformVersionMatchesFile guarantees that the hard-coded fallback in +// version.go stays in sync with the .terraform-version compatibility file that +// the tool manifest updater maintains. func TestTerraformVersionMatchesFile(t *testing.T) { _, thisFile, _, ok := runtime.Caller(0) require.True(t, ok, "unable to determine test file location")