feat: add sources upload pipeline

.github/workflows/ado/sources-upload.yml

@@ -0,0 +1,72 @@
+# Microsoft Corporation
+#
+# Wrapper pipeline — passed to ADO as the entry point. This file owns all
+# OneBranch-specific wiring (governed templates repo, Official vs NonOfficial
+# variant, featureFlags) and delegates the actual stages/jobs/steps to the
+# raw stages template at:
+#   .github/workflows/ado/templates/sources-upload-stages.yml
+#
+# Authenticates via Workload Identity Federation (OIDC) and calls the Control
+# Tower prcheck API with PR context.
+#
+# Prerequisites (ADO / Azure Portal):
+#   1. Entra ID App Registration with audience URI
+#      "api://<ControlTower-ClientId>" (see variable group below).
+#   2. Federated identity credential on the app registration for the ADO
+#      service connection (issuer: https://vstoken.dev.azure.com/<org-id>,
+#      subject: sc://<org>/<project>/<service-connection-name>).
+#   3. ARM service connection in ADO project settings using Workload Identity
+#      Federation (manual).
+#   4. ADO branch policy or pipeline PR trigger configured to fire on PRs.
+#
+# Variable Group (ADO Pipelines > Library):
+#   Name: "ControlTower-PRCheck"
+#   Required variables:
+#     - ApiAudience          : Entra ID audience URI for the Control Tower app
+#     - ApiBaseUrl           : Base URL of the Control Tower service
+#     - AzldevCommit         : Commit hash for azldev (go install ...@<commit>)
+
+# Trigger controlled by ADO branch policy — not YAML triggers.
+trigger: none
+
+pr: none
+
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.Official.CrossPlat.yml@templates
+  parameters:
+    featureFlags:
+      golang:
+        internalModuleProxy:
+          enabled: true
+      LinuxHostVersion:
+        Network: R1
+      runOnHost: true
+      EnableCDPxPAT: false
+
+    # https://aka.ms/obpipelines/sdl
+    globalSdl:
+      disableLegacyManifest: true
+      sbom:
+        enabled: false
+      tsa:
+        enabled: false
+
+    stages:
+      - template: /.github/workflows/ado/templates/sources-upload-stages.yml@self
+        parameters:
+          outputDirectory: $(Build.ArtifactStagingDirectory)/output
+          artifactBaseName: prcheck
+          containerImage: mcr.microsoft.com/onebranch/azurelinux/build:3.0
+          poolType: linux
+          serviceConnection: CT-Endpoints-Access-ServiceConnection-DEV
+          variableGroup: ControlTower-PRCheck
+          # Must exceed the script's --poll-timeout-seconds (default 7200s = 120m)
+          # with enough headroom for setup steps and the final API call.
+          timeoutInMinutes: 150

.github/workflows/ado/templates/sources-upload-stages.yml

@@ -0,0 +1,194 @@
+# Microsoft Corporation
+#
+# Raw stages template for the sources-upload PR check pipeline.
+#
+# This template is OneBranch-agnostic: it declares the stages/jobs/steps that
+# do the actual work and exposes the OneBranch-coupled knobs as parameters.
+# The wrapper at .github/workflows/ado/sources-upload.yml is responsible for
+# choosing the OneBranch governed template variant (Official vs NonOfficial),
+# configuring its featureFlags, and supplying concrete values for the
+# parameters declared below.
+
+parameters:
+  - name: outputDirectory
+    type: string
+  - name: artifactBaseName
+    type: string
+  - name: containerImage
+    type: string
+  - name: poolType
+    type: string
+    default: linux
+  - name: serviceConnection
+    type: string
+  - name: variableGroup
+    type: string
+  - name: timeoutInMinutes
+    type: number
+
+stages:
+  - stage: PRCheck
+    jobs:
+      - job: CallControlTowerAPI
+        # Must exceed the script's --poll-timeout-seconds (default 7200s = 120m)
+        # with enough headroom for setup steps and the final API call.
+        timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+        pool:
+          type: ${{ parameters.poolType }}
+        variables:
+          - group: ${{ parameters.variableGroup }}
+          - name: ob_outputDirectory
+            value: ${{ parameters.outputDirectory }}
+          - name: ob_artifactBaseName
+            value: ${{ parameters.artifactBaseName }}
+          - name: LinuxContainerImage
+            value: ${{ parameters.containerImage }}
+        steps:
+          - task: PipAuthenticate@1
+            displayName: "Authenticate pip"
+            inputs:
+              artifactFeeds: "azl/ControlTowerFeed"
+
+          - script: |
+              set -euo pipefail
+
+              echo "##[group]Mock"
+              tdnf install -y mock mock-rpmautospec python3-chardet
+              sudo usermod -aG mock "$(whoami)"
+              echo "##[endgroup]"
+
+              echo "##[group]Azldev"
+              echo "Installing azldev@${AZLDEV_COMMIT}..."
+              go install "github.com/microsoft/azure-linux-dev-tools/cmd/azldev@${AZLDEV_COMMIT}"
+
+              go_bin_path="$(go env GOPATH)/bin"
+              echo "##vso[task.prependpath]$go_bin_path"
+
+              "$go_bin_path/azldev" --version
+              echo "##[endgroup]"
+
+              echo "##[group]Python dependencies"
+              pip install -r .github/workflows/scripts/control-tower-prcheck/requirements.txt
+              echo "##[endgroup]"
+            displayName: "Install dependencies"
+            env:
+              AZLDEV_COMMIT: $(AzldevCommit)
+
+          - script: |
+              set -euo pipefail
+
+              # Workaround for an ADO git config error during spec rendering.
+              # The config key may not be present on every agent image, so tolerate its absence.
+              git config --unset extensions.worktreeConfig || true
+
+              # Full history is needed for spec rendering to work.
+              if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then
+                git fetch --unshallow
+              fi
+
+              # TODO: Replace with using the TOML config to read the specs dir once available.
+              #       The '-o "$specs_dir"' should be removed as part of the update. Future command:
+              #         specs_dir="$(azldev config dump -q -f json | jq -r .project.renderedSpecsDir)"
+              # ADO task: 19149
+              specs_dir="specs"
+
+              echo "##[group]Specs rendering"
+              azldev component render -q -a -o "$specs_dir" --force --clean-stale
+              echo "##[endgroup]"
+
+              # Check for any new or modified files under the specs directory.
+              if ! python3 .github/workflows/scripts/check_rendered_specs.py --specs-dir "$specs_dir"; then
+                echo "##[group]Git diff"
+                git --no-pager diff -- "$specs_dir"
+                echo "##[endgroup]"
+                echo "##[warning]Specs are not up to date. Ignoring until main branch is updated and stable."
+                # TODO: Re-enabled failing once specs in the main branch are updated and stable.
+                # ADO task: 19150
+                # exit 1
+              fi
+            displayName: "Verify rendered specs are up to date"
+
+          - script: |
+              set -euo pipefail
+
+              # For both triggers, Build.SourceVersion is the relevant source commit:
+              # - PR trigger: ADO sets it to the GitHub-created merge commit (refs/pull/{n}/merge).
+              # - CI/merge-queue trigger: it is the commit that landed on the base branch.
+              source_hash="$SOURCE_COMMIT"
+
+              if [[ "$BUILD_REASON" == "PullRequest" ]]; then
+                # PR trigger: ADO provides the target branch directly.
+                target_hash=$(git ls-remote "$UPSTREAM_REPO_URL" "$PR_TARGET_BRANCH" | cut -f1)
+              else
+                # CI/merge-queue trigger: extract the base branch from the merge-queue branch name.
+                # Branch format: refs/heads/gh-readonly-queue/{base_branch}/pr-{pr_number}-{head_sha}
+                # Using 'test/' branches until ready with the merge queue, but the parsing logic is the same.
+                if ! base_branch=$(grep -oP '(?<=test/gh-readonly-queue/).+(?=/pr-[^/]+$)' <<< "$SOURCE_BRANCH"); then
+                  echo "##[error]Unsupported SOURCE_BRANCH '$SOURCE_BRANCH' for non-PullRequest build. Expected 'refs/heads/gh-readonly-queue/<base>/pr-<n>-<sha>'."
+                  exit 1
+                fi
+                target_hash=$(git ls-remote "$UPSTREAM_REPO_URL" "refs/heads/$base_branch" | cut -f1)
+              fi
+
+              echo "Source commit: $source_hash"
+              echo "Target commit: $target_hash"
+
+              echo "##vso[task.setvariable variable=sourceCommit]$source_hash"
+              echo "##vso[task.setvariable variable=targetCommit]$target_hash"
+            env:
+              BUILD_REASON: $(Build.Reason)
+              PR_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
+              SOURCE_BRANCH: $(Build.SourceBranch)
+              SOURCE_COMMIT: $(Build.SourceVersion)
+              UPSTREAM_REPO_URL: $(Build.Repository.Uri)
+            displayName: "Determine source and target commit hashes"
+
+          - script: |
+              set -euo pipefail
+
+              cat << EOF
+              NOTE: It's possible more components changed between the source and target commits than displayed below.
+                    This workflow only checks for updates in the 'sources' files and ignores all other changes.
+              EOF
+
+              components=()
+              while IFS= read -r line; do
+                components+=("$(basename "$(dirname "$line")")")
+              done < <(git diff "$SOURCE_COMMIT" "$TARGET_COMMIT" --name-only | grep '/sources$' | sort -u)
+
+              components="$(IFS=, ; echo "${components[*]}")"
+              echo "##[warning] Debugging with dummy components 'atlas,words'"
+              components="atlas,words"
+              echo "Affected components: $components"
+              echo "##vso[task.setvariable variable=components]$components"
+            env:
+              SOURCE_COMMIT: $(sourceCommit)
+              TARGET_COMMIT: $(targetCommit)
+            displayName: "Determine affected components"
+
+          - task: AzureCLI@2
+            displayName: "Call Control Tower 'prcheck' API"
+            inputs:
+              azureSubscription: ${{ parameters.serviceConnection }}
+              scriptType: bash
+              scriptLocation: inlineScript
+              inlineScript: |
+                set -euo pipefail
+
+                python3 .github/workflows/scripts/control-tower-prcheck/run_control_tower_prcheck.py \
+                  --api-audience "$API_AUDIENCE" \
+                  --api-base-url "$API_BASE_URL" \
+                  --build-reason "$BUILD_REASON" \
+                  --components "$COMPONENTS" \
+                  --source-commit "$SOURCE_COMMIT" \
+                  --repo-uri "$UPSTREAM_REPO_URL"
+            env:
+              API_AUDIENCE: $(ApiAudience)
+              API_BASE_URL: $(ApiBaseUrl)
+              BUILD_REASON: $(Build.Reason)
+              COMPONENTS: $(components)
+              SOURCE_COMMIT: $(sourceCommit)
+              # TODO: Target commit is not used. Will be needed once we move detection of affected components to CT.
+              # ADO task: 18816
+              TARGET_COMMIT: $(targetCommit)
+              UPSTREAM_REPO_URL: $(Build.Repository.Uri)

.github/workflows/scripts/README.md

@@ -38,7 +38,7 @@ The easiest way to get quick, interactive feedback is with [GitHub Copilot](http
 ```bash
 cd ~/repos/azurelinux
 python -m venv .venv && source .venv/bin/activate
-pip install -r .github/workflows/scripts/requirements.txt
+pip install -r .github/workflows/scripts/spec-review/requirements.txt
     gh copilot
     # OR
     npm install -g @github/copilot
@@ -51,19 +51,19 @@ copilot --version  # verify CLI
 # Defaults: reviews all *.spec in repo, writes to repo root
 #   spec_review_report.json, copilot_log.md, spec_review_kb.md
 
-./.github/workflows/scripts/spec_review.sh \
+./.github/workflows/scripts/spec-review/spec_review.sh \
   --spec base/comps/azurelinux-release/azurelinux-release.spec \
   --spec base/comps/azurelinux-repos/azurelinux-repos.spec
 
 # Validate / inspect
-python .github/workflows/scripts/spec_review_schema.py /tmp/spec_review_report.json --all
-python .github/workflows/scripts/spec_review_schema.py /tmp/spec_review_report.json --json | jq
+python .github/workflows/scripts/spec-review/spec_review_schema.py /tmp/spec_review_report.json --all
+python .github/workflows/scripts/spec-review/spec_review_schema.py /tmp/spec_review_report.json --json | jq
 ```
 
 The script supports additional options (like selecting a model, or using different URLs); run with `--help` for details.
 
 ```bash
-./.github/workflows/scripts/spec_review.sh --help
+./.github/workflows/scripts/spec-review/spec_review.sh --help
 ```
 
 ## Run multi-model review (spec_review_multi.sh)
@@ -79,11 +79,11 @@ uses a third model pass to synthesize their findings into a single high-quality
 
 ```bash
 # Defaults: claude-opus-4.6 + gpt-5.2-codex reviewers, gpt-5.2-codex synthesizer
-./.github/workflows/scripts/spec_review_multi.sh \
+./.github/workflows/scripts/spec-review/spec_review_multi.sh \
   --spec base/comps/azurelinux-release/azurelinux-release.spec
 
 # Custom models
-./.github/workflows/scripts/spec_review_multi.sh \
+./.github/workflows/scripts/spec-review/spec_review_multi.sh \
   --spec foo.spec \
   --model1 gpt-5.2-codex \
   --model2 claude-opus-4.6 \
@@ -101,7 +101,7 @@ ls /tmp/spec_review_workdir/
 Run with `--help` for all options:
 
 ```bash
-./.github/workflows/scripts/spec_review_multi.sh --help
+./.github/workflows/scripts/spec-review/spec_review_multi.sh --help
 ```
 
 ---
@@ -119,7 +119,7 @@ copilot --agent spec-review
 ```bash
 # For semi-automated or fully automated runs, use -i or -p flags:
 prompt="Review: base/comps/azurelinux-release/azurelinux-release.spec against packaging guidelines.\n"\
-"Write JSON to spec_review_report.json and validate with: python .github/workflows/scripts/spec_review_schema.py spec_review_report.json"
+"Write JSON to spec_review_report.json and validate with: python .github/workflows/scripts/spec-review/spec_review_schema.py spec_review_report.json"
 
 # Semi-interactive (runs prompt, then waits for the user)
 copilot --agent spec-review \
@@ -144,7 +144,7 @@ copilot --agent spec-review \
   -p "$prompt"
 
 # Review output
-python .github/workflows/scripts/spec_review_schema.py spec_review_report.json --all
+python .github/workflows/scripts/spec-review/spec_review_schema.py spec_review_report.json --all
 ```
 
 ## Future work