diff --git a/.devops/templates/release-forced-warning.yml b/.devops/templates/release-forced-warning.yml new file mode 100644 index 00000000000000..e0566e96bd3720 --- /dev/null +++ b/.devops/templates/release-forced-warning.yml @@ -0,0 +1,33 @@ +# Flags a release that published to npm but deliberately skipped the git push. +# +# Only fires when the git push preflight failed and the release was forced. In that case packages +# are live on npm while the repo still has the old versions, so someone must run the recovery. +# +# Include immediately AFTER the beachball publish step. + +parameters: + - name: dryRun + type: boolean + default: false + +steps: + - ${{ if eq(parameters.dryRun, false) }}: + - script: | + echo "##vso[task.logissue type=warning]Packages were published to npm but NOT pushed to git." + echo "##vso[task.logissue type=warning]Run the 'release-recovery' skill against this pipeline run to resync the repo." + echo "" + echo "The repository is now out of sync with npm." + echo "" + echo "To recover, from a clean fluentui checkout run:" + echo " /release-recovery $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + echo "" + echo "Then fix the underlying cause - usually rotating the GitHub PAT in the" + echo "'Github and NPM secrets' variable group - or the next release will fail the same way." + echo "" + + # Green-with-warning: the npm release genuinely succeeded, so a red run would be + # misleading and would train people to ignore failed release runs. But plain green would + # let the outstanding manual recovery slip by unnoticed. + echo "##vso[task.complete result=SucceededWithIssues;]Published to npm; git push skipped - recovery required" + displayName: 'Flag forced release for recovery' + condition: and(succeeded(), ne(variables['gitPushAvailable'], 'true')) diff --git a/.devops/templates/release-git-preflight.yml b/.devops/templates/release-git-preflight.yml new file mode 100644 index 00000000000000..4939f301b06c33 --- /dev/null +++ b/.devops/templates/release-git-preflight.yml @@ -0,0 +1,90 @@ +# Guards a release against publishing to npm with a git token that cannot push. +# +# Background: `beachball publish` publishes to npm BEFORE it commits/tags/pushes. npm publishes are +# irreversible, so an invalid git token leaves npm and the repo permanently out of sync (this +# happened on 2026-06-30 when an enterprise policy started rejecting classic PATs with a lifetime +# > 8 days: all 18 v8 packages published, then every push retry failed with 403). +# +# Usage: include this template BEFORE the beachball publish step, and add $(beachballPushArgs) to +# the publish command. +# +# - template: .devops/templates/release-git-preflight.yml@self +# parameters: +# branch: master +# dryRun: ${{ parameters.dryRun }} +# forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} +# +# - script: | +# yarn beachball publish $(beachballPushArgs) --config ... --message '...' +# env: +# GITHUB_PAT: $(githubPAT) +# NPM_TOKEN: $(npmToken) + +parameters: + # Branch the release pushes bumps/changelogs/tags to. + - name: branch + type: string + default: master + + # Git remote the release pushes to. + - name: remote + type: string + default: origin + + # Emergency escape hatch. When true, a failed preflight no longer blocks the release: packages are + # still published to npm, but with `--no-push`, and a recovery bundle is produced instead. + # This is a permission to proceed, NOT a mandate to skip - a healthy token still does a full release. + - name: forceReleaseWithoutGitPush + type: boolean + default: false + + # Dry runs never publish or push, so the preflight is skipped. + - name: dryRun + type: boolean + default: false + +steps: + # Always define the variable so `$(beachballPushArgs)` is never left as an unexpanded macro, + # even on code paths where the preflight itself is skipped. + - script: | + echo "##vso[task.setvariable variable=beachballPushArgs;]" + displayName: 'Preflight: initialize push mode' + + - ${{ if eq(parameters.dryRun, false) }}: + - script: | + set -euo pipefail + + # ADO renders boolean parameters as "True"/"False"; normalize before comparing. + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + echo "Force mode enabled: a failed preflight will not block the release." + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote "${{ parameters.remote }}" \ + --branch "${{ parameters.branch }}" \ + $allowFailure + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight: verify git push access' + + # Translate the preflight result into the beachball flags used by the publish step. + # + # `--no-push` makes beachball publish to npm but skip commit/tag/push. Note it also skips the + # `precommit` hook and tagging, which is why force mode reproduces those separately when + # building the recovery bundle. + - script: | + set -euo pipefail + + if [ "$(gitPushAvailable)" = "true" ]; then + echo "Git push available - performing a normal release." + echo "##vso[task.setvariable variable=beachballPushArgs;]" + else + echo "Git push NOT available - publishing to npm only (--no-push)." + echo "##vso[task.setvariable variable=beachballPushArgs;]--no-push" + fi + displayName: 'Preflight: select beachball push mode' diff --git a/AGENTS.md b/AGENTS.md index ba571e028485b6..731fa18eea3bb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,11 +76,12 @@ state.root.className = mergeClasses( ## Workflows -| Topic | Location | -| ------------------------------------ | ---------------------------------------------------------------- | -| PR checklist, change files, commands | [docs/workflows/contributing.md](docs/workflows/contributing.md) | -| Testing guide (unit, VRT, SSR, E2E) | [docs/workflows/testing.md](docs/workflows/testing.md) | -| Team routing and label taxonomy | [docs/team-routing.md](docs/team-routing.md) | +| Topic | Location | +| ---------------------------------------------------- | ---------------------------------------------------------------------------------- | +| PR checklist, change files, commands | [docs/workflows/contributing.md](docs/workflows/contributing.md) | +| Testing guide (unit, VRT, SSR, E2E) | [docs/workflows/testing.md](docs/workflows/testing.md) | +| Release git push preflight / forced release recovery | [docs/workflows/release-git-preflight.md](docs/workflows/release-git-preflight.md) | +| Team routing and label taxonomy | [docs/team-routing.md](docs/team-routing.md) | ## Quality Tracking diff --git a/azure-pipelines.release-vnext.yml b/azure-pipelines.release-vnext.yml index 831d2d45b3748b..8a22082106ee4e 100644 --- a/azure-pipelines.release-vnext.yml +++ b/azure-pipelines.release-vnext.yml @@ -11,6 +11,14 @@ parameters: type: boolean default: false + # Escape hatch for when the GitHub PAT is broken and a release cannot wait for it to be rotated. + # Publishes to npm but skips the git push. The run finishes as partiallySucceeded, and the repo is + # resynced afterwards by running the "release-recovery" skill against the run. + - name: forceReleaseWithoutGitPush + displayName: 'Force release even if git PAT is invalid (npm only, manual repo update required)' + type: boolean + default: false + variables: - group: 'Github and NPM secrets' - template: .devops/templates/variables.yml @@ -77,6 +85,27 @@ extends: filePath: yarn-ci.sh displayName: yarn + # Fail fast: catch a broken GitHub PAT now rather than after a long build/test cycle. + # This is purely an optimization - the authoritative check runs again immediately + # before publish, so a token that expires mid-build is still caught. + - script: | + set -euo pipefail + + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote origin --branch master $allowFailure + condition: and(succeeded(), not(${{ parameters.dryRun }})) + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight (early): verify git push access' + - script: | FLUENT_PROD_BUILD=true yarn nx run-many -t build -p tag:vNext --exclude 'tag:tools,tag:type:stories,apps/**' --nxBail displayName: build @@ -94,15 +123,29 @@ extends: displayName: 'Deprecate preview packages' condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/release-git-preflight.yml@self + parameters: + branch: master + dryRun: ${{ parameters.dryRun }} + forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} + - script: | - yarn beachball publish --config scripts/beachball/src/release-vNext.config.js --message 'release: applying package updates - react-components' - git reset --hard origin/master + yarn beachball publish $(beachballPushArgs) --config scripts/beachball/src/release-vNext.config.js --message 'release: applying package updates - react-components' env: GITHUB_PAT: $(githubPAT) NPM_TOKEN: $(npmToken) displayName: Publish changes and bump versions condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/release-forced-warning.yml@self + parameters: + dryRun: ${{ parameters.dryRun }} + + - script: | + git reset --hard origin/master + displayName: Reset workspace after publish + condition: not(${{ parameters.dryRun }}) + - script: | node -r ./scripts/ts-node/src/register scripts/executors/src/tag-react-components.ts --token $(npmToken) displayName: Tag prelease packages with prerelease tag diff --git a/azure-pipelines.release.headless.yml b/azure-pipelines.release.headless.yml index 9fad65c70cd948..6450fa20a877ef 100644 --- a/azure-pipelines.release.headless.yml +++ b/azure-pipelines.release.headless.yml @@ -11,6 +11,14 @@ parameters: type: boolean default: false + # Escape hatch for when the GitHub PAT is broken and a release cannot wait for it to be rotated. + # Publishes to npm but skips the git push. The run finishes as partiallySucceeded, and the repo is + # resynced afterwards by running the "release-recovery" skill against the run. + - name: forceReleaseWithoutGitPush + displayName: 'Force release even if git PAT is invalid (npm only, manual repo update required)' + type: boolean + default: false + variables: - group: 'Github and NPM secrets' - template: .devops/templates/variables.yml @@ -66,6 +74,27 @@ extends: filePath: yarn-ci.sh displayName: yarn + # Fail fast: catch a broken GitHub PAT now rather than after a long build/test cycle. + # This is purely an optimization - the authoritative check runs again immediately + # before publish, so a token that expires mid-build is still caught. + - script: | + set -euo pipefail + + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote origin --branch master $allowFailure + condition: and(succeeded(), not(${{ parameters.dryRun }})) + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight (early): verify git push access' + - script: | echo "Following packages will be published (if they contain changes):" yarn nx show projects -p 'tag:react-headless,!tag:npm:private' --exclude 'apps/**' @@ -83,15 +112,29 @@ extends: FLUENT_PROD_BUILD=true yarn nx run-many -t lint -p 'tag:react-headless,!tag:npm:private,!tag:type:stories' --exclude 'apps/**' --nxBail displayName: lint + - template: .devops/templates/release-git-preflight.yml@self + parameters: + branch: master + dryRun: ${{ parameters.dryRun }} + forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} + - script: | - yarn beachball publish --config scripts/beachball/src/release-headless.config.js --message 'release: applying package updates - react-headless' - git reset --hard origin/master + yarn beachball publish $(beachballPushArgs) --config scripts/beachball/src/release-headless.config.js --message 'release: applying package updates - react-headless' env: GITHUB_PAT: $(githubPAT) NPM_TOKEN: $(npmToken) displayName: Publish changes and bump versions condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/release-forced-warning.yml@self + parameters: + dryRun: ${{ parameters.dryRun }} + + - script: | + git reset --hard origin/master + displayName: Reset workspace after publish + condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/cleanup.yml@self parameters: checkForModifiedFiles: false diff --git a/azure-pipelines.release.tools.yml b/azure-pipelines.release.tools.yml index cb1701fd804418..823d0f0f2644f8 100644 --- a/azure-pipelines.release.tools.yml +++ b/azure-pipelines.release.tools.yml @@ -11,6 +11,14 @@ parameters: type: boolean default: false + # Escape hatch for when the GitHub PAT is broken and a release cannot wait for it to be rotated. + # Publishes to npm but skips the git push. The run finishes as partiallySucceeded, and the repo is + # resynced afterwards by running the "release-recovery" skill against the run. + - name: forceReleaseWithoutGitPush + displayName: 'Force release even if git PAT is invalid (npm only, manual repo update required)' + type: boolean + default: false + variables: - group: 'Github and NPM secrets' - template: .devops/templates/variables.yml @@ -66,6 +74,27 @@ extends: filePath: yarn-ci.sh displayName: yarn + # Fail fast: catch a broken GitHub PAT now rather than after a long build/test cycle. + # This is purely an optimization - the authoritative check runs again immediately + # before publish, so a token that expires mid-build is still caught. + - script: | + set -euo pipefail + + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote origin --branch master $allowFailure + condition: and(succeeded(), not(${{ parameters.dryRun }})) + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight (early): verify git push access' + - script: | echo "Following packages will be published(if they contain changes):" yarn nx show projects -p 'tag:tools,!tag:npm:private,!tag:v8' --exclude 'apps/**' @@ -82,15 +111,29 @@ extends: FLUENT_PROD_BUILD=true yarn nx run-many -t lint -p 'tag:tools,!tag:npm:private,!tag:v8' --exclude 'apps/**' --nxBail displayName: lint + - template: .devops/templates/release-git-preflight.yml@self + parameters: + branch: master + dryRun: ${{ parameters.dryRun }} + forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} + - script: | - yarn beachball publish --config scripts/beachball/src/release-tools.config.js --message 'release: applying package updates - tools' - git reset --hard origin/master + yarn beachball publish $(beachballPushArgs) --config scripts/beachball/src/release-tools.config.js --message 'release: applying package updates - tools' env: GITHUB_PAT: $(githubPAT) NPM_TOKEN: $(npmToken) displayName: Publish changes and bump versions condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/release-forced-warning.yml@self + parameters: + dryRun: ${{ parameters.dryRun }} + + - script: | + git reset --hard origin/master + displayName: Reset workspace after publish + condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/cleanup.yml@self parameters: checkForModifiedFiles: false diff --git a/azure-pipelines.release.web-components.yml b/azure-pipelines.release.web-components.yml index 7e7fd8d0a0fae0..08af0850840e05 100644 --- a/azure-pipelines.release.web-components.yml +++ b/azure-pipelines.release.web-components.yml @@ -11,6 +11,14 @@ parameters: type: boolean default: false + # Escape hatch for when the GitHub PAT is broken and a release cannot wait for it to be rotated. + # Publishes to npm but skips the git push. The run finishes as partiallySucceeded, and the repo is + # resynced afterwards by running the "release-recovery" skill against the run. + - name: forceReleaseWithoutGitPush + displayName: 'Force release even if git PAT is invalid (npm only, manual repo update required)' + type: boolean + default: false + variables: - group: 'Github and NPM secrets' - template: .devops/templates/variables.yml @@ -76,19 +84,54 @@ extends: filePath: yarn-ci.sh displayName: yarn + # Fail fast: catch a broken GitHub PAT now rather than after a long build/test cycle. + # This is purely an optimization - the authoritative check runs again immediately + # before publish, so a token that expires mid-build is still caught. + - script: | + set -euo pipefail + + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote origin --branch master $allowFailure + condition: and(succeeded(), not(${{ parameters.dryRun }})) + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight (early): verify git push access' + - script: | yarn nx run-many -t format:check lint test build -p tag:web-components --exclude vr-tests-web-components --nxBail displayName: Build, Test, Lint + - template: .devops/templates/release-git-preflight.yml@self + parameters: + branch: master + dryRun: ${{ parameters.dryRun }} + forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} + - script: | - yarn beachball publish --config scripts/beachball/src/release-web-components.config.js --message 'release: applying package updates - web-components' - git reset --hard origin/master + yarn beachball publish $(beachballPushArgs) --config scripts/beachball/src/release-web-components.config.js --message 'release: applying package updates - web-components' env: GITHUB_PAT: $(githubPAT) NPM_TOKEN: $(npmToken) displayName: Publish changes and bump versions condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/release-forced-warning.yml@self + parameters: + dryRun: ${{ parameters.dryRun }} + + - script: | + git reset --hard origin/master + displayName: Reset workspace after publish + condition: not(${{ parameters.dryRun }}) + - template: .devops/templates/cleanup.yml@self parameters: checkForModifiedFiles: false diff --git a/azure-pipelines.release.yml b/azure-pipelines.release.yml index 6bfc06bedbe290..f6a31d92e245ad 100644 --- a/azure-pipelines.release.yml +++ b/azure-pipelines.release.yml @@ -11,6 +11,14 @@ parameters: type: boolean default: false + # Escape hatch for when the GitHub PAT is broken and a release cannot wait for it to be rotated. + # Publishes to npm but skips the git push. The run finishes as partiallySucceeded, and the repo is + # resynced afterwards by running the "release-recovery" skill against the run. + - name: forceReleaseWithoutGitPush + displayName: 'Force release even if git PAT is invalid (npm only, manual repo update required)' + type: boolean + default: false + variables: - group: 'Github and NPM secrets' - template: .devops/templates/variables.yml @@ -102,6 +110,27 @@ extends: filePath: yarn-ci.sh displayName: yarn + # Fail fast: catch a broken GitHub PAT now rather than after ~an hour of build/test. + # This is purely an optimization - the authoritative check runs again immediately + # before publish, so a token that expires mid-build is still caught. + - script: | + set -euo pipefail + + force=$(echo "$FORCE_RELEASE_WITHOUT_GIT_PUSH" | tr '[:upper:]' '[:lower:]') + + allowFailure="" + if [ "$force" = "true" ]; then + allowFailure="--allow-failure" + fi + + node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-git-push-access.ts \ + --remote origin --branch master $allowFailure + condition: and(succeeded(), not(${{ parameters.dryRun }})) + env: + GITHUB_PAT: $(githubPAT) + FORCE_RELEASE_WITHOUT_GIT_PUSH: ${{ parameters.forceReleaseWithoutGitPush }} + displayName: 'Preflight (early): verify git push access' + - script: | yarn generate-version-files displayName: Generate version files @@ -126,15 +155,29 @@ extends: FLUENT_PROD_BUILD=true yarn nx run-many -t verify-packaging -p tag:v8 --exclude tag:vNext,tag:tools,ssr-tests,vr-tests,perf-test --nxBail displayName: verify packaged assets + - template: .devops/templates/release-git-preflight.yml@self + parameters: + branch: master + dryRun: ${{ parameters.dryRun }} + forceReleaseWithoutGitPush: ${{ parameters.forceReleaseWithoutGitPush }} + - script: | - yarn beachball publish --config scripts/beachball/src/release-v8.config.js --message 'release: applying package updates - react v8' - git reset --hard origin/master + yarn beachball publish $(beachballPushArgs) --config scripts/beachball/src/release-v8.config.js --message 'release: applying package updates - react v8' condition: and(succeeded(), not(${{ parameters.dryRun }})) env: GITHUB_PAT: $(githubPAT) NPM_TOKEN: $(npmToken) displayName: Publish changes and bump versions + - template: .devops/templates/release-forced-warning.yml@self + parameters: + dryRun: ${{ parameters.dryRun }} + + - script: | + git reset --hard origin/master + condition: and(succeeded(), not(${{ parameters.dryRun }})) + displayName: Reset workspace after publish + - script: | echo Making $(Build.ArtifactStagingDirectory)/api mkdir -p $(Build.ArtifactStagingDirectory)/api diff --git a/docs/workflows/release-git-preflight.md b/docs/workflows/release-git-preflight.md new file mode 100644 index 00000000000000..3ad4248db77b67 --- /dev/null +++ b/docs/workflows/release-git-preflight.md @@ -0,0 +1,153 @@ +# Release safety: git push preflight + +## Why this exists + +`beachball publish` runs in this order: + +1. bump versions on disk → **publish to npm** +2. commit, tag, **push to git** (5 retries) + +npm publishes are irreversible, so if the GitHub token turns out to be unable to push, the packages +are already public while the repo still has the old versions. + +This happened on **2026-06-30**: an enterprise policy started rejecting classic PATs with a lifetime +greater than 8 days. All 18 v8 packages published successfully, then every push attempt failed with +`403`. npm and `master` were out of sync until a manual recovery commit two days later, and the git +tags for that release were never created. + +To prevent a repeat, release pipelines now **verify git push access before publishing**. + +## What runs + +Two checks, because they catch different failures: + +| Check | Catches | +| ------------------------------ | ----------------------------------------------------------------------------- | +| `GET /user` via the GitHub API | expired / revoked tokens, org or enterprise policy rejections, missing scopes | +| `git push --dry-run` | valid credentials that nevertheless lack push permission on the branch | + +Implemented in [`scripts/executors/src/check-git-push-access.ts`](../../scripts/executors/src/check-git-push-access.ts), +wired up by [`.devops/templates/release-git-preflight.yml`](../../.devops/templates/release-git-preflight.yml). + +The preflight runs twice: + +- **early**, right after `yarn`, so a broken token fails the run in ~2 minutes instead of after a + full build/test cycle. This is only an optimization. +- **immediately before publish**, which is the check that actually provides the guarantee (it also + catches a token that expires part-way through a long build). + +A non-fast-forward rejection is deliberately **not** treated as a failure: it means authentication +succeeded and the branch simply moved on, which beachball already handles by fetching and merging +before it pushes. Failing there would block healthy releases. + +## Normal behavior + +If the token is fine, nothing changes — the release publishes and pushes as before. + +If the token is broken, the pipeline **fails before publishing anything**. npm and the repo stay in +sync. Fix it by rotating the PAT in the `Github and NPM secrets` variable group. + +## Forcing a release with a broken token + +If a release genuinely cannot wait for the PAT to be rotated, re-run the pipeline with: + +> **Force release even if git PAT is invalid (npm only, manual repo update required)** + +| `forceReleaseWithoutGitPush` | Token valid | Result | +| ---------------------------- | ----------- | ----------------------------------------------------- | +| `false` (default) | yes | normal release | +| `false` (default) | no | **fails before publishing** | +| `true` | yes | normal release | +| `true` | no | publishes to npm, skips push, emits a recovery bundle | + +The flag is a _permission to proceed_, not an instruction to skip pushing — with a healthy token it +still performs a complete, normal release. + +### What force mode produces + +beachball runs with `--no-push`, so packages reach npm but nothing is committed, tagged or pushed. +The run finishes as **partiallySucceeded** (green with a warning): the release really did succeed on +npm, so red would be misleading, but plain green would let the outstanding manual step slip by. + +The run also prints the exact command to fix it. + +### Recovering afterwards + +Run the **`release-recovery`** skill from a clean checkout, pointing it at the failed run: + +``` +/release-recovery https://dev.azure.com///_build/results?buildId= +``` + +It diagnoses the drift against npm (the source of truth), shows a read-only plan, and — only after +you approve — regenerates the bumps and changelogs with beachball and opens a recovery PR. + +The diagnosis alone can be run at any time: + +```bash +node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-release-sync.ts \ + --remote upstream --ref upstream/master +``` + +Rotate the PAT first — otherwise the next release fails exactly the same way. + +### What recovery does not do + +Recovery restores **version bumps and changelogs only**. It does not recreate release git tags: + +- a recovery commit is not the commit the release was built from, so tags against it would point at + the wrong history +- a single release spans dozens of packages (a v9 release touches ~90), making bulk tag creation + noisy and unreviewable +- tags are not load-bearing for consumers — npm is what they install from + +Pass `--check-tags` to the diagnostic to _report_ missing tags for awareness. If tags are genuinely +required for a particular release, create them as a separate deliberate task. + +## Implementation notes + +Verified against beachball `3.0.0-alpha.7`; re-check these on upgrade: + +- `--no-push` **still writes** bumps, changelogs and lockfile changes to disk (`performBump` runs + during the publish step) — but they are discarded by the pipeline's `git reset --hard`, which is + why recovery replays them locally rather than trying to salvage them from the agent. +- Neither `--no-push` **nor** the `bump` command runs beachball's `precommit` hook (it only runs on + the push path, inside `mergePublishBranch`). Recovery therefore has to run the + `dependency-mismatch` / `normalize-package-dependencies` generators and refresh the lockfile + itself. Keep that in sync with `hooks.precommit` in + [`scripts/beachball/src/shared.config.ts`](../../scripts/beachball/src/shared.config.ts). +- `beachball bump` **never commits, tags or pushes** — it only writes files. `tagPackages` and + `git push` exist solely in `bumpAndPush`, which only the `publish` command reaches. Verified + empirically: a full `bump` run changed 123 files and produced 0 tags, 0 commits, 0 branches. +- `--no-push` **skips tagging**. Tags are intentionally left uncreated (see above). + +The four experimental/nightly release pipelines already pass `--no-push` and never push, so they are +unaffected. + +### Recovering an already-broken release + +The `release-recovery` skill works on **any** desync, including releases that predate this tooling — +it derives everything from npm rather than from a pipeline artifact. + +Check at any time whether the repo and npm agree: + +```bash +node -r ./scripts/ts-node/src/register ./scripts/executors/src/check-release-sync.ts \ + --remote upstream --ref upstream/master +``` + +It reports the problem that matters: + +- **version desync** — published to npm, but the repo never recorded the bump + +Add `--check-tags` to also list published versions with no git tag. That is informational only — +recovery does not recreate tags (see above). + +## Known limitation + +The preflight shrinks the window in which a token can go bad from "the whole build" to "the seconds +between the check and the push" — it cannot eliminate it. Genuine atomicity is not achievable while +npm publish precedes the git push. + +The durable fix is to stop using long-lived classic PATs (a GitHub App installation token or a +fine-grained PAT with automated rotation), which addresses the cause rather than the blast radius. diff --git a/scripts/executors/package.json b/scripts/executors/package.json index f9aa3d95f4a76f..c48eea54e30f9f 100644 --- a/scripts/executors/package.json +++ b/scripts/executors/package.json @@ -4,6 +4,7 @@ "private": true, "main": "index.js", "dependencies": { + "@fluentui/scripts-github": "*", "@fluentui/scripts-monorepo": "*", "@fluentui/scripts-prettier": "*", "@fluentui/scripts-utils": "*" diff --git a/scripts/executors/src/check-git-push-access.spec.ts b/scripts/executors/src/check-git-push-access.spec.ts new file mode 100644 index 00000000000000..908b2e8756ca54 --- /dev/null +++ b/scripts/executors/src/check-git-push-access.spec.ts @@ -0,0 +1,174 @@ +import { spawnSync } from 'child_process'; + +import { checkCanPush, checkTokenIsUsable, redact } from './check-git-push-access'; + +jest.mock('child_process', () => ({ spawnSync: jest.fn() })); + +const getAuthenticated = jest.fn(); +jest.mock('@octokit/rest', () => ({ + Octokit: jest.fn().mockImplementation(() => ({ users: { getAuthenticated } })), +})); + +const spawnSyncMock = spawnSync as unknown as jest.Mock; + +const pushOptions = { remote: 'origin', branch: 'master', allowFailure: false }; + +/** Format an expiry the way GitHub sends it in the token expiration header. */ +function expiresIn(ms: number): string { + return new Date(Date.now() + ms).toISOString().replace('T', ' ').replace(/\..*/, ' UTC'); +} + +function gitResult(overrides: Partial<{ status: number; stderr: string; stdout: string; error: Error }> = {}) { + return { status: 0, stderr: '', stdout: '', ...overrides }; +} + +beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(console, 'log').mockImplementation(() => undefined); + jest.spyOn(console, 'warn').mockImplementation(() => undefined); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +describe('redact', () => { + it('removes every occurrence of the token', () => { + expect(redact('using ghp_secret and ghp_secret again', 'ghp_secret')).toBe('using *** and *** again'); + }); + + it('passes text through when there is no token to hide', () => { + expect(redact('nothing to hide', undefined)).toBe('nothing to hide'); + }); +}); + +describe('checkTokenIsUsable', () => { + const ok = (headers: Record = {}) => ({ data: { login: 'releaser' }, headers }); + + it('accepts a valid token', async () => { + getAuthenticated.mockResolvedValue(ok()); + + await expect(checkTokenIsUsable('ghp_valid')).resolves.toBeUndefined(); + }); + + it.each([ + [401, 'revoked or malformed credentials'], + [403, 'credentials rejected by an enterprise policy'], + ])('fails on HTTP %i (%s)', async (status, _description) => { + getAuthenticated.mockRejectedValue(Object.assign(new Error('nope'), { status })); + + const failure = await checkTokenIsUsable('ghp_bad'); + + expect(failure?.summary).toContain(`HTTP ${status}`); + }); + + it('never echoes the token in failure details', async () => { + getAuthenticated.mockRejectedValue(Object.assign(new Error('bad credentials for ghp_secret'), { status: 401 })); + + const failure = await checkTokenIsUsable('ghp_secret'); + + expect(JSON.stringify(failure)).not.toContain('ghp_secret'); + expect(failure?.details?.join('\n')).toContain('***'); + }); + + it('does not fail the release when the GitHub API is unreachable', async () => { + // The push check is authoritative, so an API outage must not block a release on its own. + getAuthenticated.mockRejectedValue(Object.assign(new Error('getaddrinfo ENOTFOUND'), { status: 500 })); + + await expect(checkTokenIsUsable('ghp_valid')).resolves.toBeUndefined(); + }); + + it('fails when the token is already expired', async () => { + getAuthenticated.mockResolvedValue(ok({ 'github-authentication-token-expiration': expiresIn(-60 * 60 * 1000) })); + + const failure = await checkTokenIsUsable('ghp_expired'); + + expect(failure?.summary).toContain('expired'); + }); + + it('accepts a token expiring in less than a day', async () => { + // Regression: flooring the delta to whole days reported 0 days left for a token with 23h of + // life remaining, which failed a release that would otherwise have succeeded. + getAuthenticated.mockResolvedValue( + ok({ 'github-authentication-token-expiration': expiresIn(23 * 60 * 60 * 1000) }), + ); + + await expect(checkTokenIsUsable('ghp_short')).resolves.toBeUndefined(); + }); + + it('warns without failing when the token expires soon', async () => { + getAuthenticated.mockResolvedValue( + ok({ 'github-authentication-token-expiration': expiresIn(23 * 60 * 60 * 1000) }), + ); + + await checkTokenIsUsable('ghp_short'); + + expect((console.warn as jest.Mock).mock.calls.join('\n')).toContain('rotate it soon'); + }); + + it('does not warn for a token with plenty of life left', async () => { + getAuthenticated.mockResolvedValue( + ok({ 'github-authentication-token-expiration': expiresIn(30 * 24 * 60 * 60 * 1000) }), + ); + + await checkTokenIsUsable('ghp_long'); + + expect((console.warn as jest.Mock).mock.calls.join('\n')).not.toContain('rotate it soon'); + }); +}); + +describe('checkCanPush', () => { + it('passes when the dry-run push succeeds', () => { + spawnSyncMock.mockReturnValue(gitResult()); + + expect(checkCanPush(pushOptions, 'ghp_valid')).toBeUndefined(); + }); + + it('runs a dry-run that cannot mutate the remote', () => { + spawnSyncMock.mockReturnValue(gitResult()); + + checkCanPush(pushOptions, 'ghp_valid'); + + expect(spawnSyncMock).toHaveBeenCalledWith( + 'git', + ['push', '--dry-run', '--no-verify', 'origin', 'HEAD:master'], + expect.anything(), + ); + }); + + it('fails when the remote rejects the push', () => { + spawnSyncMock.mockReturnValue( + gitResult({ status: 128, stderr: 'remote: Permission denied\nfatal: unable to push' }), + ); + + expect(checkCanPush(pushOptions, 'ghp_readonly')?.summary).toContain('failed'); + }); + + it.each([ + ['non-fast-forward', '! [rejected] master -> master (non-fast-forward)'], + ['fetch first', '! [rejected] master -> master (fetch first)'], + ['updates were rejected', 'Updates were rejected because the remote contains work you do not have'], + ])('passes on a stale-ref rejection (%s), which is not an auth problem', (_name, stderr) => { + // Authentication succeeded - the push only failed because the branch moved. beachball fetches + // and merges inside its own retry loop, so failing here would block healthy releases. + spawnSyncMock.mockReturnValue(gitResult({ status: 1, stderr })); + + expect(checkCanPush(pushOptions, 'ghp_valid')).toBeUndefined(); + }); + + it('fails when git cannot be executed', () => { + spawnSyncMock.mockReturnValue({ error: new Error('spawn git ENOENT'), status: null, stderr: '', stdout: '' }); + + expect(checkCanPush(pushOptions, 'ghp_valid')?.summary).toContain('ENOENT'); + }); + + it('never echoes the token from git output', () => { + spawnSyncMock.mockReturnValue( + gitResult({ status: 128, stderr: 'fatal: https://ghp_secret@github.com/microsoft/fluentui rejected' }), + ); + + const failure = checkCanPush(pushOptions, 'ghp_secret'); + + expect(JSON.stringify(failure)).not.toContain('ghp_secret'); + }); +}); diff --git a/scripts/executors/src/check-git-push-access.ts b/scripts/executors/src/check-git-push-access.ts new file mode 100644 index 00000000000000..770457bdb264ad --- /dev/null +++ b/scripts/executors/src/check-git-push-access.ts @@ -0,0 +1,276 @@ +import { spawnSync } from 'child_process'; + +import { fluentRepoDetails } from '@fluentui/scripts-github'; +import { Octokit } from '@octokit/rest'; +import yargs from 'yargs'; + +/** + * Preflight validation that the configured GitHub PAT can actually push to the release branch. + * + * Why this exists: + * `beachball publish` publishes to npm BEFORE it commits/tags/pushes to git. npm publishes are + * irreversible, so if the git token turns out to be invalid we end up with npm and the repo + * permanently out of sync (this happened on 2026-06-30 when an enterprise policy started rejecting + * classic PATs with a lifetime > 8 days). + * + * Running this before the publish step turns that silent, expensive failure into a cheap, loud one. + */ + +const adoVariableName = 'gitPushAvailable'; + +interface PreflightOptions { + branch: string; + remote: string; + allowFailure: boolean; +} + +interface PreflightFailure { + /** Short summary shown in the pipeline error */ + summary: string; + /** Extra context (API message, git stderr, remediation hints) */ + details?: string[]; +} + +/** + * Remove the token from arbitrary text before logging. + * + * git usually redacts credentials embedded in remote URLs, but it is not guaranteed for every error + * path, and we also echo API responses. Since a leaked PAT in build logs is a security incident, + * scrub defensively rather than trusting upstream behavior. + */ +function redact(text: string, token: string | undefined): string { + if (!text) { + return ''; + } + return token ? text.split(token).join('***') : text; +} + +/** + * Verify the token itself is valid and not expired/revoked/blocked by policy. + * + * This is the check that produces a genuinely useful error message: the enterprise policy failure + * returns a descriptive 403 body, and GitHub also returns the token expiration as a response header. + */ +export async function checkTokenIsUsable(token: string): Promise { + const github = new Octokit({ auth: 'token ' + token }); + + try { + const response = await github.users.getAuthenticated(); + + const expiration = response.headers['github-authentication-token-expiration']; + const scopes = response.headers['x-oauth-scopes']; + + console.log(` authenticated as: ${response.data.login}`); + console.log(` token scopes: ${scopes || '(none reported - likely a fine-grained token or GitHub App)'}`); + + if (expiration) { + const expiresAt = new Date(String(expiration).replace(' UTC', 'Z').replace(' ', 'T')); + const msLeft = expiresAt.getTime() - Date.now(); + // Expiry is decided on the raw delta, never on whole days: flooring would round a token with + // 23h of life left down to 0 days and fail a release that would have succeeded. The rounded + // value is only ever used for humans reading the log. + const daysLeft = Math.ceil(msLeft / (1000 * 60 * 60 * 24)); + + console.log(` token expires: ${expiration} (~${daysLeft} day(s) from now)`); + + if (msLeft <= 0) { + return { summary: `The GitHub token expired on ${expiration}.` }; + } + // Not a failure on its own - the push check below is authoritative - but worth surfacing early + // so the token gets rotated before it breaks a release. + if (msLeft <= 3 * 24 * 60 * 60 * 1000) { + console.warn( + ` ##vso[task.logissue type=warning]GitHub token expires in ~${daysLeft} day(s) - rotate it soon.`, + ); + } + } else { + console.log(' token expires: (no expiration reported)'); + } + } catch (err) { + const error = err as { status?: number; message?: string }; + const status = error.status; + + // 401 = bad/revoked credentials, 403 = valid credentials rejected by policy (the 2026-06-30 case) + if (status === 401 || status === 403) { + return { + summary: `GitHub rejected the token with HTTP ${status}.`, + details: [ + redact(error.message ?? 'Unknown error', token), + '', + 'Common causes:', + ' - the PAT expired or was revoked', + ' - the PAT violates an org/enterprise policy (e.g. max lifetime for classic PATs)', + ' - the PAT is missing the "repo" scope', + ], + }; + } + + // Network blips and API outages should not be treated as an invalid token, but we also cannot + // confirm the token is good - defer the verdict to the push check. + console.warn(` [WARN] Could not verify the token via the GitHub API: ${redact(String(error.message), token)}`); + console.warn(' [WARN] Continuing to the git push check, which is authoritative.'); + } + + return undefined; +} + +/** + * Verify we can actually push to the target branch. + * + * This is the authoritative check: a token can have perfectly valid credentials yet still lack push + * permission on the repo. `--dry-run` performs the full negotiation with the server (including the + * `git-receive-pack` request that returned 403 in the 2026-06-30 incident) without updating any refs. + */ +export function checkCanPush(options: PreflightOptions, token: string | undefined): PreflightFailure | undefined { + const { remote, branch } = options; + const args = ['push', '--dry-run', '--no-verify', remote, `HEAD:${branch}`]; + + console.log(` running: git ${args.join(' ')}`); + + const result = spawnSync('git', args, { encoding: 'utf8' }); + + if (result.error) { + return { + summary: `Failed to run git push --dry-run: ${result.error.message}`, + }; + } + + if (result.status !== 0) { + const stderr = redact(result.stderr || '', token).trim(); + const stdout = redact(result.stdout || '', token).trim(); + const combined = `${stderr}\n${stdout}`; + + // A non-fast-forward rejection means the push was REFUSED BY REF STATE, not by auth - we + // successfully authenticated and got far enough to compare refs. This happens whenever a commit + // lands on the target branch between checkout and preflight. Failing here would block perfectly + // good releases, and beachball already handles staleness itself (it fetches + merges inside its + // retry loop before pushing). + const isStaleRefRejection = /non-fast-forward|fetch first|Updates were rejected because/i.test(combined); + + if (isStaleRefRejection) { + console.warn( + [ + ` [WARN] ${remote}/${branch} has moved ahead of the current checkout.`, + ' [WARN] Authentication succeeded, so this is NOT a token problem and the release may proceed.', + ' [WARN] beachball fetches and merges the latest changes before pushing.', + ].join('\n'), + ); + return undefined; + } + + return { + summary: `git push --dry-run to ${remote}/${branch} failed (exit code ${result.status}).`, + details: [stderr, stdout].filter(Boolean), + }; + } + + console.log(` push access to ${remote}/${branch} confirmed`); + + return undefined; +} + +/** Tell ADO whether the later publish step may push, so it can pick the right beachball flags. */ +function setPipelineVariable(value: boolean): void { + console.log(`##vso[task.setvariable variable=${adoVariableName};]${value}`); +} + +async function runPreflight(options: PreflightOptions): Promise { + const token = process.env.GITHUB_PAT; + + console.log('Validating git push access before publishing...\n'); + console.log(` repo: ${fluentRepoDetails.owner}/${fluentRepoDetails.repo}`); + console.log(` target: ${options.remote}/${options.branch}\n`); + + let failure: PreflightFailure | undefined; + + if (!token) { + failure = { + summary: 'GITHUB_PAT environment variable is not set.', + details: ['The release pipeline must provide GITHUB_PAT for the publish step to push bumps and tags.'], + }; + } else { + failure = await checkTokenIsUsable(token); + } + + if (!failure) { + failure = checkCanPush(options, token); + } + + if (!failure) { + console.log('\nPreflight passed - the release may publish and push normally.\n'); + setPipelineVariable(true); + return; + } + + setPipelineVariable(false); + + const message = [ + '', + 'Git push preflight FAILED.', + '', + failure.summary, + ...(failure.details?.length ? ['', ...failure.details] : []), + '', + ].join('\n'); + + if (options.allowFailure) { + // Force mode: the operator has explicitly accepted an npm-only release. + console.warn(message); + console.warn( + [ + 'Continuing because the release was forced (forceReleaseWithoutGitPush=true).', + 'Packages will be published to npm, but version bumps, changelogs and tags will NOT be pushed.', + 'Run the "release-recovery" skill against this pipeline run afterwards to resync the repo.', + '', + ].join('\n'), + ); + return; + } + + console.error(message); + console.error( + [ + 'Nothing has been published - the repo and npm are still in sync.', + '', + 'To fix: rotate the GitHub PAT used by this pipeline (variable group "Github and NPM secrets").', + 'To release anyway (npm only, manual repo update afterwards): re-run with', + 'the "Force release without git push" parameter enabled.', + '', + ].join('\n'), + ); + + process.exit(1); +} + +export { redact }; + +// Only parse argv and run when invoked as a CLI - importing this module (from tests, or to reuse +// the classification logic) must not execute a preflight or call process.exit. +if (require.main === module) { + const argv = yargs + .option('branch', { + type: 'string', + describe: 'Branch the release will push to', + default: 'master', + }) + .option('remote', { + type: 'string', + describe: 'Git remote the release will push to', + default: 'origin', + }) + .option('allow-failure', { + type: 'boolean', + describe: 'Report the result but do not fail the step (used by force mode)', + default: false, + }) + .strict().argv; + + runPreflight({ + branch: argv.branch, + remote: argv.remote, + allowFailure: argv['allow-failure'], + }).catch(err => { + console.error('Unexpected error during git push preflight:', err); + process.exit(1); + }); +} diff --git a/yarn.lock b/yarn.lock index 07b8e32e859e65..7378f823b8f11c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6105,6 +6105,7 @@ __metadata: version: 0.0.0-use.local resolution: "@fluentui/scripts-executors@workspace:scripts/executors" dependencies: + "@fluentui/scripts-github": "npm:*" "@fluentui/scripts-monorepo": "npm:*" "@fluentui/scripts-prettier": "npm:*" "@fluentui/scripts-utils": "npm:*"