diff --git a/README.md b/README.md index 9d0e675..54181ef 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,82 @@ Set of reusable workflows used by Onfido Github actions. +## Lock + +A distributed lock mechanism to prevent parallel execution of integration tests across repositories. Uses a git branch in a dedicated lock repository (`onfido/ci-lock`) as an atomic mutex. + +### How it works + +- Lock branch **does not exist** = lock is **free** +- Lock branch **exists** = lock is **taken** + +The `lock/acquire` action attempts to create a branch (e.g., `locks/client-libraries-integration-tests`) in the lock repository. Git ref creation is atomic — only one process can succeed. If it fails, the action polls until the branch is deleted (lock released) or times out. + +The `lock/release` action deletes the lock branch, making the lock available again. + +### Setup + +1. Create the lock repository (e.g., `onfido/ci-lock`) with at least a `main` branch +2. Create a GitHub App with write access to the lock repository +3. Install the app on the org and grant it access to the `ci-lock` repo +4. Store `ONFIDO_CI_LOCK_APP_ID` and `ONFIDO_CI_LOCK_PRIVATE_KEY` as org-level secrets + +### Usage + +```yaml +jobs: + integration-tests: + runs-on: ubuntu-latest + steps: + - uses: onfido/onfido-actions/lock/acquire@main + with: + app-id: ${{ secrets.ONFIDO_CI_LOCK_APP_ID }} + app-private-key: ${{ secrets.ONFIDO_CI_LOCK_PRIVATE_KEY }} + lock-name: "client-libraries-integration-tests" # optional, this is the default + timeout: "1800" # optional, default 30 min + poll-interval: "30" # optional, default 30s + + # ... run your integration tests ... + + - uses: onfido/onfido-actions/lock/release@main + if: always() + with: + app-id: ${{ secrets.ONFIDO_CI_LOCK_APP_ID }} + app-private-key: ${{ secrets.ONFIDO_CI_LOCK_PRIVATE_KEY }} + lock-name: "client-libraries-integration-tests" +``` + +### Inputs + +#### `lock/acquire` + +| Input | Required | Default | Description | +|-------|----------|---------|-------------| +| `app-id` | yes | | GitHub App ID | +| `app-private-key` | yes | | GitHub App private key | +| `lock-name` | yes | `client-libraries-integration-tests` | Lock name (becomes branch `locks/`) | +| `lock-repo` | no | `onfido/ci-lock` | Repository used for locking | +| `timeout` | no | `1800` | Max wait time in seconds | +| `poll-interval` | no | `30` | Seconds between retries | + +#### `lock/release` + +| Input | Required | Default | Description | +|-------|----------|---------|-------------| +| `app-id` | yes | | GitHub App ID | +| `app-private-key` | yes | | GitHub App private key | +| `lock-name` | yes | `client-libraries-integration-tests` | Lock name (must match acquire) | +| `lock-repo` | no | `onfido/ci-lock` | Repository used for locking | + +### Stale lock recovery + +If a workflow crashes without releasing the lock, manually delete the branch: +```bash +gh api repos/onfido/ci-lock/git/refs/heads/locks/client-libraries-integration-tests -X DELETE +``` + +> **Important:** Always use `if: always()` on the release step to ensure the lock is freed even if tests fail. + ## Changelog update To manually update changelog, run: @@ -11,4 +87,4 @@ RELEASE_BODY=" - Nested changelog line from spec change - Extra changelog line from library change " ../onfido-action/update-changelog/update-changelog.sh -``` \ No newline at end of file +``` diff --git a/lock/acquire/action.yml b/lock/acquire/action.yml new file mode 100644 index 0000000..a81eb61 --- /dev/null +++ b/lock/acquire/action.yml @@ -0,0 +1,125 @@ +name: Acquire Lock +description: | + Acquire a distributed lock using a git branch in a shared repository. + Attempts to create a lock branch — if it succeeds, the lock is acquired. + If the branch already exists (lock held), polls until it's released or times out. + +inputs: + app-id: + description: "GitHub App ID for generating a token" + required: true + app-private-key: + description: "GitHub App private key for generating a token" + required: true + lock-name: + description: "Name of the lock (used as branch name under locks/ prefix)" + required: true + default: "client-libraries-integration-tests" + lock-repo: + description: "Repository used for locking (owner/repo format)" + required: false + default: "onfido/ci-lock" + timeout: + description: "Maximum time in seconds to wait for the lock (default: 1800 = 30 min)" + required: false + default: "1800" + poll-interval: + description: "Seconds between lock acquisition attempts (default: 30)" + required: false + default: "30" + +outputs: + acquired: + description: "Whether the lock was successfully acquired" + value: ${{ steps.acquire.outputs.acquired }} + lock-branch: + description: "The branch name used for this lock" + value: ${{ steps.acquire.outputs.lock_branch }} + +runs: + using: "composite" + steps: + - name: Generate token from GitHub App + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ inputs.app-private-key }} + owner: ${{ github.repository_owner }} + repositories: ci-lock + + - name: Acquire lock + id: acquire + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + LOCK_NAME: ${{ inputs.lock-name }} + LOCK_REPO: ${{ inputs.lock-repo }} + TIMEOUT: ${{ inputs.timeout }} + POLL_INTERVAL: ${{ inputs.poll-interval }} + run: | + LOCK_BRANCH="locks/${LOCK_NAME}" + echo "lock_branch=${LOCK_BRANCH}" >> "$GITHUB_OUTPUT" + + elapsed=0 + + # Get the default branch SHA to use as base for the lock branch + get_base_sha() { + gh api "repos/${LOCK_REPO}/git/ref/heads/main" --jq '.object.sha' 2>/dev/null || \ + gh api "repos/${LOCK_REPO}/git/ref/heads/master" --jq '.object.sha' 2>/dev/null + } + + # Try to create the lock branch (atomic operation) + try_acquire() { + local base_sha + base_sha=$(get_base_sha) + + if [ -z "$base_sha" ]; then + echo "::error::Could not find base branch in ${LOCK_REPO}" + exit 1 + fi + + # Creating a ref is atomic — only one process can succeed + local result + result=$(gh api "repos/${LOCK_REPO}/git/refs" \ + -X POST \ + -f "ref=refs/heads/${LOCK_BRANCH}" \ + -f "sha=${base_sha}" 2>&1) && return 0 || return 1 + } + + echo "Attempting to acquire lock '${LOCK_NAME}' (branch: ${LOCK_BRANCH} in ${LOCK_REPO})..." + + while [ $elapsed -lt $TIMEOUT ]; do + if try_acquire; then + echo "✅ Lock '${LOCK_NAME}' acquired successfully" + echo "acquired=true" >> "$GITHUB_OUTPUT" + + # Update the lock branch with a commit containing metadata + LOCK_INFO="Lock acquired by ${GITHUB_REPOSITORY} - ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + TMPDIR=$(mktemp -d) + git clone --depth 1 --branch "${LOCK_BRANCH}" "https://x-access-token:${GH_TOKEN}@github.com/${LOCK_REPO}.git" "$TMPDIR" 2>/dev/null || true + if [ -d "$TMPDIR/.git" ]; then + cd "$TMPDIR" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git commit --allow-empty -m "$LOCK_INFO" + git push origin "${LOCK_BRANCH}" 2>/dev/null || true + cd - > /dev/null + fi + rm -rf "$TMPDIR" + + exit 0 + fi + + # Show who currently holds the lock + HOLDER=$(gh api "repos/${LOCK_REPO}/commits/${LOCK_BRANCH}" --jq '.commit.message' 2>/dev/null || echo "unknown") + echo "⏳ Lock '${LOCK_NAME}' is held (${elapsed}/${TIMEOUT}s elapsed, retrying in ${POLL_INTERVAL}s)" + echo " Holder: ${HOLDER}" + sleep "$POLL_INTERVAL" + elapsed=$((elapsed + POLL_INTERVAL)) + done + + echo "❌ Timed out waiting for lock '${LOCK_NAME}' after ${TIMEOUT}s" + echo "acquired=false" >> "$GITHUB_OUTPUT" + echo "::error::Timed out waiting for lock '${LOCK_NAME}' after ${TIMEOUT}s" + exit 1 diff --git a/lock/release/action.yml b/lock/release/action.yml new file mode 100644 index 0000000..c1127f9 --- /dev/null +++ b/lock/release/action.yml @@ -0,0 +1,57 @@ +name: Release Lock +description: | + Release a distributed lock by deleting the lock branch from the shared repository. + Should always be called with `if: always()` to ensure the lock is released even on failure. + +inputs: + app-id: + description: "GitHub App ID for generating a token" + required: true + app-private-key: + description: "GitHub App private key for generating a token" + required: true + lock-name: + description: "Name of the lock (used as branch name under locks/ prefix)" + required: true + default: "client-libraries-integration-tests" + lock-repo: + description: "Repository used for locking (owner/repo format)" + required: false + default: "onfido/ci-lock" + +runs: + using: "composite" + steps: + - name: Generate token from GitHub App + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ inputs.app-private-key }} + owner: ${{ github.repository_owner }} + repositories: ci-lock + + - name: Release lock + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + LOCK_NAME: ${{ inputs.lock-name }} + LOCK_REPO: ${{ inputs.lock-repo }} + run: | + LOCK_BRANCH="locks/${LOCK_NAME}" + + echo "Releasing lock '${LOCK_NAME}' (deleting branch: ${LOCK_BRANCH} in ${LOCK_REPO})..." + + # Verify we own the lock before releasing it + LOCK_MSG=$(gh api "repos/${LOCK_REPO}/commits/${LOCK_BRANCH}" --jq '.commit.message' 2>/dev/null || echo "") + if [ -n "$LOCK_MSG" ] && ! echo "$LOCK_MSG" | grep -q "${GITHUB_RUN_ID}"; then + echo "⚠️ Lock is held by another run: ${LOCK_MSG}" + echo "⚠️ Skipping release to avoid breaking another workflow's lock" + exit 0 + fi + + if gh api "repos/${LOCK_REPO}/git/refs/heads/${LOCK_BRANCH}" -X DELETE 2>/dev/null; then + echo "✅ Lock '${LOCK_NAME}' released successfully" + else + echo "⚠️ Lock branch not found — may have already been released" + fi diff --git a/should-run-integration-tests/action.yml b/should-run-integration-tests/action.yml new file mode 100644 index 0000000..cb69950 --- /dev/null +++ b/should-run-integration-tests/action.yml @@ -0,0 +1,59 @@ +name: Should Run Integration Tests +description: | + Determines whether integration tests should run based on the workflow event. + Returns result=true for: + - pull_request: only if NOT from a fork (forks don't have access to secrets) + - push: only if it's a merge from a fork PR (to catch what couldn't run pre-merge) + - release, workflow_dispatch, schedule: always + +inputs: + github-token: + description: "GitHub token for API access (typically github.token)" + required: false + default: ${{ github.token }} + +outputs: + result: + description: "'true' if integration tests should run, empty otherwise" + value: ${{ steps.check.outputs.result }} + +runs: + using: "composite" + steps: + - name: Check if integration tests should run + id: check + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + EVENT_NAME: ${{ github.event_name }} + PR_HEAD_REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }} + REPO: ${{ github.repository }} + REPO_OWNER: ${{ github.repository_owner }} + SHA: ${{ github.sha }} + run: | + case "$EVENT_NAME" in + release|workflow_dispatch|schedule) + echo "result=true" >> "$GITHUB_OUTPUT" + ;; + pull_request) + # Skip fork PRs — secrets are not available + if [ "$PR_HEAD_REPO_FULL_NAME" = "$REPO" ]; then + echo "result=true" >> "$GITHUB_OUTPUT" + else + echo "Skipping: PR from fork ($PR_HEAD_REPO_FULL_NAME)" + fi + ;; + push) + # On push, only run tests if it's a merge from a fork PR + # (fork PRs can't run integration tests pre-merge due to missing secrets) + PR_OWNER=$(gh pr list --repo "$REPO" --state merged \ + --search "$SHA" --json headRepositoryOwner \ + --jq '.[0].headRepositoryOwner.login // empty' 2>/dev/null) + if [ -n "$PR_OWNER" ] && [ "$PR_OWNER" != "$REPO_OWNER" ]; then + echo "Push from fork PR (owner: $PR_OWNER) — running integration tests" + echo "result=true" >> "$GITHUB_OUTPUT" + else + echo "Skipping: push is not from a fork PR" + fi + ;; + esac