diff --git a/NuGet.config b/NuGet.config index 83f178c4680..bc6addd0970 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,10 +5,8 @@ - - @@ -24,10 +22,8 @@ - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4dd56e3d60f..1c3d0fe4373 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -165,25 +165,25 @@ - + https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 + 43f11677d894d27b41fce39c496836c8207f3b8d - + https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 + 43f11677d894d27b41fce39c496836c8207f3b8d - + https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 + 43f11677d894d27b41fce39c496836c8207f3b8d https://github.com/dotnet/arcade f4e11a15c7b8a949d4a366e792a9843ff6e88cd5 - + https://github.com/dotnet/arcade - efce34e9f9f25af27e2b471fbbf8c21f1ac2e318 + 43f11677d894d27b41fce39c496836c8207f3b8d https://github.com/dotnet/sourcelink diff --git a/eng/Versions.props b/eng/Versions.props index 8457d3a64c6..eca177d58f8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -52,7 +52,7 @@ 9.0.0-beta.24053.1 - 9.0.0-beta.26301.4 + 9.0.0-beta.26411.5 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 new file mode 100644 index 00000000000..9c7e3dcd6ac --- /dev/null +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -0,0 +1,164 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements: +# - A GitHub App whose private key has been uploaded into Key Vault as an RSA +# key (the PEM converted to a Key Vault *key*, NOT stored as a secret). +# - The caller (the federated Azure service connection used to run this script) +# must have the `Key Vault Crypto User` role (or at minimum the `Sign` +# action) on that key. +# - The App must be installed on the target organization/account +# (`InstallationOwner`) with the permissions/repositories it needs. +# +# Installation tokens (ghs_*) are exempt from the enterprise classic-PAT +# lifetime policy, which is why this replaces the long-lived PAT. + +[CmdletBinding()] +param( + # Name of the Key Vault that holds the GitHub App's RSA signing key. + [Parameter(Mandatory = $true)] + [string] $KeyVaultName, + + # Name of the RSA key inside the Key Vault (the App's private key). + [Parameter(Mandatory = $true)] + [string] $KeyName, + + # The GitHub App's Client ID (the value to put in the `iss` JWT claim). + [Parameter(Mandatory = $true)] + [string] $AppClientId, + + # Login of the organization or user account whose installation we should + # mint the token for (e.g. `dotnet`, `microsoft`). + [Parameter(Mandatory = $true)] + [string] $InstallationOwner, + + # Optional Azure DevOps pipeline variable name to set with the installation + # token (marked as a secret). When not specified, the token is written to + # stdout instead. + [Parameter(Mandatory = $false)] + [string] $OutputVariableName +) + +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +function ConvertTo-Base64Url([byte[]] $bytes) { + return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +# Build JWT header and payload. Use [ordered] hashtables so JSON +# serialization is deterministic. +$jwtHeader = [ordered]@{ + alg = 'RS256' + typ = 'JWT' +} +$now = [System.DateTimeOffset]::UtcNow +$jwtPayload = [ordered]@{ + iat = $now.AddMinutes(-1).ToUnixTimeSeconds() + exp = $now.AddMinutes(5).ToUnixTimeSeconds() + iss = $AppClientId +} + +$headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress))) +$payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress))) +$signingInput = "$headerEncoded.$payloadEncoded" + +# Key Vault `sign` expects the *digest* (base64), not the raw bytes. +$sha256 = [System.Security.Cryptography.SHA256]::Create() +$digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) +$digestBase64 = [Convert]::ToBase64String($digestBytes) + +Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..." +$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference +try { + # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds. + # Use the exit code to determine success for this invocation. + $PSNativeCommandUseErrorActionPreference = $false + $signatureBase64 = az keyvault key sign ` + --vault-name $KeyVaultName ` + --name $KeyName ` + --algorithm RS256 ` + --digest $digestBase64 ` + --query signature ` + --output tsv ` + --only-show-errors + $signExitCode = $LASTEXITCODE +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +finally { + $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference +} +if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + exit 1 +} +$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_') +$jwt = "$signingInput.$signatureUrl" + +$headers = @{ + Authorization = "Bearer $jwt" + 'X-GitHub-Api-Version' = '2022-11-28' + Accept = 'application/vnd.github+json' + 'User-Agent' = 'dotnet-arcade-onelocbuild' +} + +Write-Host "Looking up installation for '$InstallationOwner'..." +try { + $installations = @() + $page = 1 + do { + # Assign the response before wrapping it in @(). PowerShell otherwise + # preserves a top-level JSON array as one nested pipeline object. + $pageResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` + -Headers $headers ` + -Method Get + $pageInstallations = @($pageResponse) + $installations += $pageInstallations + $page++ + } while ($pageInstallations.Count -eq 100) +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." + exit 1 +} +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { + $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" + exit 1 +} +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." + +try { + $tokenResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations/$($installation.id)/access_tokens" ` + -Headers $headers ` + -Method Post ` + -ContentType 'application/json' +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to mint an installation access token for '$InstallationOwner' (installation $($installation.id)): $_" + exit 1 +} + +Write-Host "Got installation token for '$InstallationOwner' (expires $($tokenResponse.expires_at))." +if ($OutputVariableName) { + Write-Host "Setting pipeline variable '$OutputVariableName'." + Write-Host "##vso[task.setvariable variable=$OutputVariableName;issecret=true]$($tokenResponse.token)" +} +else { + Write-Host $tokenResponse.token -ForegroundColor Green +} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index edefa789d36..d76ed063ffa 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -8,6 +8,21 @@ parameters: CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). + # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. + # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not + # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + + # GitHub App authentication for the OneLoc check-in PR (dnceng/internal only). + # The infrastructure identifiers are centralized here and the App path is enabled by default. + # DevDiv requires its own project-scoped service connection before this path can be enabled there. + UseGitHubAppAuthentication: true + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppKeyName: 'oneloc-localization-app-key' + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false @@ -73,6 +88,30 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} + # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). + # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + - template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} + + # Mint a short-lived GitHub App installation token for the loc check-in PR (dnceng/internal only). + # All other projects fall back to PAT-based auth, since the app service connection is scoped to dnceng/internal. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: + - template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + keyName: ${{ parameters.GitHubAppKeyName }} + appClientId: ${{ parameters.GitHubAppClientId }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -89,10 +128,16 @@ jobs: ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} + ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + patVariable: $(CeapexEntraToken) + ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: + patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: + gitHubPatVariable: "$(GitHubAppInstallationToken)" + ${{ if or(eq(parameters.UseGitHubAppAuthentication, false), ne(variables['System.TeamProject'], 'internal')) }}: + gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 3cb20fb5041..5a0343f9738 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -52,8 +52,6 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false # unconditional - needed for logs publishing (redactor tool version) diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml index 4571a7864df..87c2c4c2096 100644 --- a/eng/common/core-templates/jobs/codeql-build.yml +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -19,7 +19,6 @@ jobs: enableTelemetry: true variables: - - group: Publish-Build-Assets # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994ae..3413a9a573e 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml new file mode 100644 index 00000000000..6d42a48d3c3 --- /dev/null +++ b/eng/common/core-templates/steps/get-github-app-token.yml @@ -0,0 +1,79 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with a private key stored in Azure Key Vault (RSA, RS256). The JWT is +# exchanged with the GitHub API for a token scoped to a single installation. +# +# Requirements (per GitHub App you want to authenticate as): +# - A GitHub App with its private key uploaded into Key Vault as an RSA key +# (PEM converted to a key, NOT stored as a secret). +# - The Azure service connection passed via `azureSubscription` must be +# granted the `Key Vault Crypto User` role (or at minimum `Sign` action) +# on that key. +# - The App must be installed on the target organization/account +# (`installationOwner`) with the permissions/repositories you need. +# +# Output: a secret pipeline variable named ${{ parameters.outputVariableName }} +# containing the installation access token. Token lifetime is ~1 hour and is +# automatically scrubbed from logs. Installation tokens are exempt from the +# enterprise classic-PAT lifetime policy. + +parameters: +# Azure DevOps service connection (federated) that can call +# `az keyvault key sign` on the App's signing key. +- name: azureSubscription + type: string + +# Name of the Key Vault that holds the GitHub App's RSA signing key. +- name: keyVaultName + type: string + +# Name of the RSA key inside the Key Vault (the App's private key). +- name: keyName + type: string + +# The GitHub App's Client ID (the value to put in the `iss` JWT claim). +# Prefer this over the numeric App ID; GitHub accepts either, but Client ID +# is the documented form going forward. +- name: appClientId + type: string + +# Login of the organization or user account whose installation we should +# mint the token for (e.g. `dotnet`, `microsoft`). +- name: installationOwner + type: string + +# Name of the pipeline variable that will receive the installation token. +- name: outputVariableName + type: string + +- name: is1ESPipeline + type: boolean + +- name: stepName + type: string + default: getGitHubAppInstallationToken + +- name: condition + type: string + default: '' + +- name: displayName + type: string + default: Get GitHub App installation token + +steps: +- task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.azureSubscription }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" ` + -KeyVaultName '${{ parameters.keyVaultName }}' ` + -KeyName '${{ parameters.keyName }}' ` + -AppClientId '${{ parameters.appClientId }}' ` + -InstallationOwner '${{ parameters.installationOwner }}' ` + -OutputVariableName '${{ parameters.outputVariableName }}' diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 0623ac6e112..f460b24eb95 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -28,10 +28,7 @@ steps: arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' -BinlogToolVersion ${{parameters.BinlogToolVersion}} -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' '$(microsoft-symbol-server-pat)' '$(symweb-symbol-server-pat)' '$(dn-bot-all-orgs-build-rw-code-rw)' diff --git a/eng/common/templates-official/steps/get-github-app-token.yml b/eng/common/templates-official/steps/get-github-app-token.yml new file mode 100644 index 00000000000..c89f3641a4d --- /dev/null +++ b/eng/common/templates-official/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/get-github-app-token.yml b/eng/common/templates/steps/get-github-app-token.yml new file mode 100644 index 00000000000..79e182c6416 --- /dev/null +++ b/eng/common/templates/steps/get-github-app-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a06513a5940..d89aa48781b 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -727,7 +727,7 @@ function InitializeToolset() { '' | Set-Content $proj - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile /p:RestoreIgnoreFailedSources=true $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 if (!(Test-Path $path)) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 01b09b65796..c412703d1ba 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -410,7 +410,7 @@ function InitializeToolset { fi echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" /p:RestoreIgnoreFailedSources=true local toolset_build_proj=`cat "$toolset_location_file"` diff --git a/global.json b/global.json index c6dbc287b14..d70e8c0cf07 100644 --- a/global.json +++ b/global.json @@ -12,8 +12,8 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26301.4", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26301.4" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26411.5", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26411.5" }, "sdk": { "version": "9.0.116"