From 98d0f580928d2c6e8d16288f6ce5d2ea338c077b Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 16:43:14 -0700 Subject: [PATCH 1/6] feat(workflows): add Repo Radius deploy workflow (rad startup/shutdown) Port the Repo Radius deploy workflow from the github-extension prototype into .github/extension/ and adapt it to the merged building blocks: multi-cluster v1 (global.targetCluster seam) and externalized state (rad startup / rad shutdown). - radius-deploy.yml: ephemeral k3d control plane, restore state with rad startup, run the dispatched radius_commands, persist state with rad shutdown, tear down. Honors the workflow_dispatch contract (environment + radius_commands as a single string or JSON array; rad prefix omitted; stop-on-first-failure) and uploads per-command output as the radius-output artifact. - Credentials are provider-native, not registered: AWS OIDC session creds are injected as pod env vars; Azure uses Workload Identity with the GitHub Actions OIDC JWT projected as the federated token file. rad env update sets scope only. - radius-verify-credentials.yml: port the verify workflow so the contract is testable without a deploy. - README.md: document both workflows and the RADIUS_TARGET_KUBECONFIG / credential / state-persistence contract. - eng/design-notes: add the deploy-workflow technical design (Investment 3). Related: #12118 Signed-off-by: Sylvain Niles --- .github/extension/README.md | 96 ++++ .github/extension/radius-deploy.yml | 477 ++++++++++++++++++ .../extension/radius-verify-credentials.yml | 115 +++++ .../2026-06-repo-radius-deploy-workflow.md | 234 +++++++++ 4 files changed, 922 insertions(+) create mode 100644 .github/extension/README.md create mode 100644 .github/extension/radius-deploy.yml create mode 100644 .github/extension/radius-verify-credentials.yml create mode 100644 eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md diff --git a/.github/extension/README.md b/.github/extension/README.md new file mode 100644 index 00000000000..a814983c550 --- /dev/null +++ b/.github/extension/README.md @@ -0,0 +1,96 @@ +# Repo Radius workflow assets + +This folder holds the **Repo Radius** workflow templates that are written into a user's repository so that Radius can run on a GitHub Actions runner. + +These files are templates: a copy is committed into the target repository under `.github/workflows/` and dispatched there. They are not run from this repository directly. + +They live here so the workflow contract has a canonical, reviewed home that any frontend (the Copilot app, the CLI, etc.) can drive. See [radius-project/radius#12118](https://github.com/radius-project/radius/issues/12118) for background, and the [Repo Radius feature specification](https://github.com/radius-project/radius/pull/12078) for the end-to-end design. + +## `radius-verify-credentials.yml` + +The credential-verification workflow Radius uses to confirm that a GitHub Environment is wired up correctly before any application is deployed. The environment-setup flow generates this file, commits it to the target repo as `.github/workflows/radius-verify-credentials.yml`, and triggers it once per provider. + +### What it does + +The workflow runs on `ubuntu-latest` and branches by provider based on which variables are present on the selected GitHub Environment. Steps that target a provider are skipped when that provider's variables are empty, so the same file serves both Azure-only and AWS-only environments. + +1. **Authenticate via OIDC.** For Azure it runs `azure/login`; for AWS it runs `aws-actions/configure-aws-credentials`. No long-lived cloud secrets are stored — the runner exchanges its GitHub OIDC token for short-lived cloud credentials. +2. **Verify access.** Azure runs `az account show`; AWS runs `aws sts get-caller-identity`. The AWS account ID in the caller ARN is masked in the logs. +3. **Discover resources.** Azure lists resource groups, AKS clusters, and physical locations; AWS lists EKS clusters, VPCs, and subnets. The results are written to `/tmp/radius-discovery.json` and uploaded as the `radius-discovery` artifact (one-day retention) for the caller to read back into the environment's variables. + +### Trigger and permissions + +- **Trigger:** `workflow_dispatch` with a single `environment` input (the GitHub Environment name, default `dev`). The job binds to that environment via `environment: ${{ inputs.environment }}`. +- **Permissions:** `id-token: write` (required for OIDC) and `contents: read`. + +## `radius-deploy.yml` + +The deploy workflow that runs Radius on demand: it stands up an ephemeral [k3d](https://k3d.io) control plane on the runner, restores persisted state, runs the caller's `rad` commands against the user's target cluster, then persists state again and tears the control plane down. The frontend generates this file, commits it to the target repo as `.github/workflows/radius-deploy.yml`, and dispatches it. + +### What it does + +1. **Authenticate via OIDC** (Azure `azure/login` or AWS `aws-actions/configure-aws-credentials`). +2. **Build a target-cluster kubeconfig** on the runner (`az aks get-credentials` / `aws eks get-token`) for the workload cluster named by `RADIUS_K8S_CLUSTER`. +3. **Create an ephemeral k3d control-plane cluster** and install the `rad` CLI. +4. **Create the `target-kubeconfig` secret** and **install Radius** with `database.enabled=true` (for state persistence) and, when a target cluster is configured, `global.targetCluster.enabled=true` — the chart seam that mounts the kubeconfig and sets `RADIUS_TARGET_KUBECONFIG` on `applications-rp`, `dynamic-rp`, and `bicep-de`. +5. **`rad startup`** restores the control-plane PostgreSQL databases and Terraform recipe state saved by the previous run. +6. **Create the Radius environment**, set its cloud **scope** with `rad env update`, and configure the provider's credentials (see [the credential contract](#cloud-credentials-provider-native) below). +7. **Run the caller's `rad` commands** (the `radius_commands` input), uploading each command's output as the `radius-output` artifact. +8. **`rad shutdown`** backs the control-plane and Terraform state up to the `radius-state` git orphan branch (runs even on failure). +9. **Tear down** the k3d cluster; collect logs on failure. + +### Trigger, inputs, and permissions + +- **Trigger:** `workflow_dispatch`. +- **Inputs:** + + | Input | Required | Description | + |-------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | `environment` | Yes | The GitHub Environment name, used as the Radius environment. | + | `radius_commands` | Yes | A single `rad` CLI command string, or a JSON-encoded array of command strings run in order. The `rad` prefix is omitted from each command (e.g. `deploy app.bicep` or `["deploy app.bicep", "app graph"]`). | + +- **Permissions:** `id-token: write` (OIDC), `contents: write` (so `rad shutdown` can push the `radius-state` branch), and `packages: write` (so container-image recipes can push to GHCR). + +## The integration contract (owned by Radius) + +These workflows depend on a stable seam that Radius owns and the multi-cluster v1 work ([#12106](https://github.com/radius-project/radius/pull/12106)) honors: + +### Target cluster (`RADIUS_TARGET_KUBECONFIG`) + +The workflow builds a kubeconfig for the external workload cluster on the runner and stores it as the `target-kubeconfig` secret in `radius-system`. Installing the chart with `--set global.targetCluster.enabled=true` mounts that secret into `applications-rp`, `dynamic-rp`, and `bicep-de` and sets `RADIUS_TARGET_KUBECONFIG` to the mounted path. Radius then directs recipe execution and directly-rendered output resources at that cluster; the Terraform kubernetes provider follows the same kubeconfig through the cluster access resolver. The secret's lifecycle (creation, EKS-token refresh, RBAC) is the workflow's responsibility, not the chart's. + +### Cloud credentials (provider-native) + +Repo Radius does not store long-lived cloud secrets; each provider uses its native OIDC model: + +- **AWS** follows Terraform's model: the workflow does **not** run `rad credential register`. The OIDC session credentials obtained on the runner are injected into the `applications-rp`, `dynamic-rp`, and `bicep-de` pods as environment variables, and the AWS SDKs/Terraform AWS provider in those pods consume them. +- **Azure** uses **Workload Identity**. The chart is installed with `global.azureWorkloadIdentity.enabled=true` (which labels the RP/DE pods), the client/tenant IDs are registered with `rad credential register azure wi` (the Terraform azurerm provider fetches them from UCP), and the workflow projects the GitHub Actions OIDC JWT into the pods as the federated token file (`/var/run/secrets/azure/tokens/azure-identity-token`) that the Azure SDKs exchange with Entra for a short-lived AAD token. + +In both cases `rad env update` records only the deployment **scope** (AWS region/account, Azure subscription/resource group), never credentials. + +> **Azure token lifetime.** The GitHub Actions OIDC JWT is short-lived. The Azure SDKs exchange it once for an ~1-hour AAD token, so a normal run is unaffected, but a run whose Azure work outlives that window may fail. Refreshing the federated token mid-run is a planned fast follow and is intentionally out of scope. + +### State persistence (`rad startup` / `rad shutdown`) + +`rad startup` and `rad shutdown` are kind-agnostic CLI commands that back up and restore all durable Radius state (control-plane PostgreSQL + Terraform recipe-state Secrets) to a `radius-state` git orphan branch. They do not manage cluster lifecycle — the workflow owns creating and destroying the ephemeral control plane around them. + +### Required GitHub Environment variables + +The workflows read only Actions **variables** (`vars`), never secrets, for cloud configuration: + +| Provider | Variables | +|----------|----------------------------------------------------------------------------------------------------------------| +| Azure | `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`, `AZURE_RESOURCE_GROUP`, `RADIUS_K8S_CLUSTER` | +| AWS | `AWS_IAM_ROLE_ARN`, `AWS_REGION`, `AWS_ACCOUNT_ID`, `RADIUS_K8S_CLUSTER`, `RADIUS_VPC_ID`, `RADIUS_SUBNET_IDS` | +| Common | `RADIUS_K8S_NAMESPACE` (target namespace; defaults to `default`) | + +A provider's branch runs only when its identifying variable (`AZURE_CLIENT_ID` or `AWS_IAM_ROLE_ARN`) is non-empty. Configuring both on one environment is rejected. + +### Cloud-side OIDC prerequisites + +OIDC trust must already exist before either workflow can authenticate (this provisioning is **out of scope** for these templates): + +- **Azure:** a federated credential on the Entra ID app whose subject is exactly `repo:/:environment:`, audience `api://AzureADTokenExchange`. +- **AWS:** an IAM role trust policy that allows `sts:AssumeRoleWithWebIdentity` from `token.actions.githubusercontent.com` with audience `sts.amazonaws.com` and subject `repo:/:environment:`. + +For the full setup flow, see [docs/contributing/contributing-deploy-environments.md](../../docs/contributing/contributing-deploy-environments.md). diff --git a/.github/extension/radius-deploy.yml b/.github/extension/radius-deploy.yml new file mode 100644 index 00000000000..99198d4182e --- /dev/null +++ b/.github/extension/radius-deploy.yml @@ -0,0 +1,477 @@ +# This workflow is a Repo Radius template. Radius writes a copy of it into a +# user's repository at .github/workflows/radius-deploy.yml and dispatches it +# there; it is not run from radius-project/radius directly. See +# .github/extension/README.md for the contract this file honors. +# +# It stands up an ephemeral k3d cluster for the Radius control plane, restores +# durable state with `rad startup`, connects recipe execution to the user's +# target cluster (EKS/AKS) via the chart's `global.targetCluster` seam, runs the +# caller-supplied Radius commands, then backs state up again with `rad shutdown` +# before tearing the control plane down. +name: Radius - Deploy Application + +on: + workflow_dispatch: + inputs: + environment: + description: 'GitHub Environment name' + required: true + default: 'dev' + radius_commands: + # A single rad CLI command string, or a JSON-encoded array of command + # strings run in order. The `rad` prefix is omitted from each command + # (e.g. `deploy app.bicep` or `["deploy app.bicep", "app graph"]`). + description: 'rad CLI command string, or JSON array of command strings, with the `rad` prefix omitted.' + required: true + +permissions: + id-token: write # required for cloud OIDC + contents: write # `rad shutdown` pushes the radius-state orphan branch + packages: write # container-image recipes push to GHCR + +env: + ENVIRONMENT: ${{ inputs.environment || 'dev' }} + RADIUS_NAMESPACE: ${{ vars.RADIUS_K8S_NAMESPACE || 'default' }} + RADIUS_IMAGE_REGISTRY: ${{ vars.RADIUS_IMAGE_REGISTRY || 'ghcr.io/radius-project' }} + RADIUS_IMAGE_TAG: ${{ vars.RADIUS_IMAGE_TAG || 'latest' }} + RESOURCE_TYPES_CONTRIB_REPO: https://github.com/radius-project/resource-types-contrib.git + RESOURCE_TYPES_CONTRIB_REF: ${{ vars.RESOURCE_TYPES_CONTRIB_REF || 'main' }} + # Where the chart's targetCluster seam expects the kubeconfig secret/key. + TARGET_KUBECONFIG_SECRET: target-kubeconfig + TARGET_KUBECONFIG_KEY: kubeconfig + TARGET_KUBECONFIG_PATH: /home/runner/.kube/target-cluster + +jobs: + deploy: + name: Deploy with Radius + runs-on: ubuntu-latest + environment: ${{ inputs.environment || 'dev' }} + steps: + - name: Checkout + # persist-credentials is intentionally left at its default (true): the + # `rad shutdown` step pushes the radius-state orphan branch back to this + # repository using the checkout's token. + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Validate provider configuration + run: | + set -euo pipefail + if [ -n "${{ vars.AZURE_CLIENT_ID }}" ] && [ -n "${{ vars.AWS_IAM_ROLE_ARN }}" ]; then + echo "Both AZURE_CLIENT_ID and AWS_IAM_ROLE_ARN are set. Configure only one provider per GitHub Environment." >&2 + exit 1 + fi + + - name: Azure Login (OIDC) + if: ${{ vars.AZURE_CLIENT_ID != '' }} + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Configure AWS Credentials (OIDC) + if: ${{ vars.AWS_IAM_ROLE_ARN != '' }} + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ vars.AWS_IAM_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Build target cluster kubeconfig + run: | + set -euo pipefail + mkdir -p "$(dirname "$TARGET_KUBECONFIG_PATH")" + + if [ -n "${{ vars.AZURE_CLIENT_ID }}" ] && [ -n "${{ vars.RADIUS_K8S_CLUSTER }}" ]; then + az aks get-credentials \ + --resource-group "${{ vars.AZURE_RESOURCE_GROUP }}" \ + --name "${{ vars.RADIUS_K8S_CLUSTER }}" \ + --subscription "${{ vars.AZURE_SUBSCRIPTION_ID }}" \ + --file "$TARGET_KUBECONFIG_PATH" + elif [ -n "${{ vars.AWS_IAM_ROLE_ARN }}" ] && [ -n "${{ vars.RADIUS_K8S_CLUSTER }}" ]; then + CLUSTER="${{ vars.RADIUS_K8S_CLUSTER }}" + REGION="${{ vars.AWS_REGION }}" + ROLE_ARN="${{ vars.AWS_IAM_ROLE_ARN }}" + + # Ensure the OIDC role can reach the EKS cluster. Idempotent. + echo "Ensuring EKS access entry for $ROLE_ARN..." + aws eks create-access-entry --cluster-name "$CLUSTER" --principal-arn "$ROLE_ARN" \ + --type STANDARD --region "$REGION" 2>/dev/null || echo "Access entry already exists" + aws eks associate-access-policy --cluster-name "$CLUSTER" --principal-arn "$ROLE_ARN" \ + --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy \ + --access-scope type=cluster --region "$REGION" 2>/dev/null || echo "Access policy already associated" + + # The RP/DE containers have no aws CLI, so write a static bearer-token + # kubeconfig rather than an exec-based one. + ENDPOINT=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.endpoint' --output text) + CA_DATA=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.certificateAuthority.data' --output text) + TOKEN=$(aws eks get-token --cluster-name "$CLUSTER" --region "$REGION" --output json | jq -r '.status.token') + printf 'apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: %s\n server: %s\n name: eks\ncontexts:\n- context:\n cluster: eks\n user: eks-user\n name: eks\ncurrent-context: eks\nkind: Config\nusers:\n- name: eks-user\n user:\n token: %s\n' "$CA_DATA" "$ENDPOINT" "$TOKEN" > "$TARGET_KUBECONFIG_PATH" + kubectl --kubeconfig "$TARGET_KUBECONFIG_PATH" cluster-info || echo "WARNING: could not reach EKS cluster" + else + echo "No target cluster configured; resources will deploy to the control-plane cluster." + fi + + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Create ephemeral Radius control plane cluster + run: | + set -euo pipefail + k3d cluster create radius-cp \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "${{ github.workspace }}/:/app/demo" \ + --wait + kubectl wait --for=condition=Ready node --all --timeout=120s + + - name: Install Radius CLI + run: | + set -euo pipefail + wget -q "https://raw.githubusercontent.com/radius-project/radius/main/deploy/install.sh" -O - | /bin/bash + rad version + + - name: Create target kubeconfig secret + # The chart's targetCluster seam mounts this secret into applications-rp, + # dynamic-rp, and bicep-de, so it must exist before `rad install`. + if: ${{ hashFiles('/home/runner/.kube/target-cluster') != '' }} + run: | + set -euo pipefail + kubectl create namespace radius-system --dry-run=client -o yaml | kubectl apply -f - + kubectl create secret generic "$TARGET_KUBECONFIG_SECRET" \ + --namespace radius-system \ + --from-file="$TARGET_KUBECONFIG_KEY=$TARGET_KUBECONFIG_PATH" \ + --dry-run=client -o yaml | kubectl apply -f - + + - name: Install Radius on control plane + run: | + set -euo pipefail + TARGET_FLAGS="" + if [ -f "$TARGET_KUBECONFIG_PATH" ]; then + TARGET_FLAGS="--set global.targetCluster.enabled=true" + fi + # Azure auth uses Workload Identity: the chart adds the + # `azure.workload.identity/use` label to the RP/DE pods. On this k3d + # cluster there is no WI admission webhook, so the workflow projects the + # federated token and sets the AZURE_* env vars itself (see below). + if [ -n "${{ vars.AZURE_CLIENT_ID }}" ]; then + TARGET_FLAGS="$TARGET_FLAGS --set global.azureWorkloadIdentity.enabled=true" + fi + # database.enabled=true gives rad startup/shutdown a PostgreSQL store to + # back up and restore. + rad install kubernetes \ + --set rp.publicEndpointOverride=localhost \ + --set global.imageRegistry=${RADIUS_IMAGE_REGISTRY} \ + --set global.imageTag=${RADIUS_IMAGE_TAG} \ + --set database.enabled=true \ + $TARGET_FLAGS + kubectl wait --for=condition=Available deployment --all -n radius-system --timeout=300s + + - name: Enable container image builds in dynamic-rp + # Mounts the Docker socket, source tree, and a GHCR auth config into + # dynamic-rp so Terraform's docker provider can build and push images. + run: | + set -euo pipefail + kubectl patch deployment dynamic-rp -n radius-system --type=json -p='[ + {"op": "add", "path": "/spec/template/spec/volumes/-", + "value": {"name": "docker-sock", "hostPath": {"path": "/var/run/docker.sock", "type": "Socket"}}}, + {"op": "add", "path": "/spec/template/spec/volumes/-", + "value": {"name": "docker-bin", "emptyDir": {}}}, + {"op": "add", "path": "/spec/template/spec/volumes/-", + "value": {"name": "app-source", "hostPath": {"path": "/app/demo", "type": "Directory"}}}, + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": {"name": "docker-sock", "mountPath": "/var/run/docker.sock"}}, + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": {"name": "docker-bin", "mountPath": "/usr/local/bin/docker", "subPath": "docker"}}, + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": {"name": "app-source", "mountPath": "/app/demo", "readOnly": true}}, + {"op": "replace", "path": "/spec/template/spec/containers/0/securityContext", + "value": {"allowPrivilegeEscalation": false, "runAsUser": 0}}, + {"op": "add", "path": "/spec/template/spec/initContainers/-", + "value": { + "name": "install-docker-cli", + "image": "docker:cli", + "command": ["cp", "/usr/local/bin/docker", "/out/docker"], + "volumeMounts": [{"name": "docker-bin", "mountPath": "/out"}] + }} + ]' + kubectl rollout status deployment/dynamic-rp -n radius-system --timeout=120s + + # Provide GHCR auth so the docker provider can push images. + DOCKER_AUTH=$(echo -n "${{ github.actor }}:${{ secrets.GHCR_PAT || secrets.GITHUB_TOKEN }}" | base64 -w0) + DOCKER_CONFIG_JSON=$(printf '{"auths":{"ghcr.io":{"auth":"%s"}}}' "$DOCKER_AUTH") + kubectl create secret generic docker-config -n radius-system \ + --from-literal=config.json="$DOCKER_CONFIG_JSON" --dry-run=client -o yaml | kubectl apply -f - + kubectl patch deployment dynamic-rp -n radius-system --type=json -p='[ + {"op": "add", "path": "/spec/template/spec/volumes/-", + "value": {"name": "docker-config-secret", "secret": {"secretName": "docker-config"}}}, + {"op": "add", "path": "/spec/template/spec/volumes/-", + "value": {"name": "docker-config", "emptyDir": {}}}, + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": {"name": "docker-config", "mountPath": "/root/.docker"}}, + {"op": "add", "path": "/spec/template/spec/containers/0/env/-", + "value": {"name": "DOCKER_CONFIG", "value": "/root/.docker"}}, + {"op": "add", "path": "/spec/template/spec/initContainers/-", + "value": { + "name": "copy-docker-config", + "image": "busybox:latest", + "command": ["sh", "-c", "cp /secret/config.json /docker-config/config.json"], + "volumeMounts": [ + {"name": "docker-config-secret", "mountPath": "/secret", "readOnly": true}, + {"name": "docker-config", "mountPath": "/docker-config"} + ] + }} + ]' + kubectl rollout status deployment/dynamic-rp -n radius-system --timeout=120s + kubectl wait --for=condition=Ready pod -n radius-system -l app.kubernetes.io/name=dynamic-rp --timeout=120s + + - name: Configure Radius workspace + run: rad workspace create kubernetes default --force + + - name: Restore Radius state + # Restores control-plane PostgreSQL + Terraform recipe state saved by a + # previous run's `rad shutdown`. Tolerant of an empty/first run. + run: rad startup --workspace default + + - name: Configure Radius environment + run: | + set -euo pipefail + # Ensure the target namespace exists on the workload cluster. + if [ -f "$TARGET_KUBECONFIG_PATH" ]; then + kubectl --kubeconfig "$TARGET_KUBECONFIG_PATH" get namespace "$RADIUS_NAMESPACE" 2>/dev/null || \ + kubectl --kubeconfig "$TARGET_KUBECONFIG_PATH" create namespace "$RADIUS_NAMESPACE" + fi + rad group create default + rad group switch default + rad env create "$ENVIRONMENT" --namespace "$RADIUS_NAMESPACE" + rad env switch "$ENVIRONMENT" + + - name: Set environment cloud scope + # Credentials themselves are NOT registered with `rad credential register`. + # Repo Radius follows Terraform's model: the OIDC session credentials are + # injected into the RP/DE pods as environment variables (next step), and + # the cloud SDKs in those pods pick them up. `rad env update` only records + # the deployment *scope* (where resources land), not credentials. + run: | + set -euo pipefail + if [ -n "${{ vars.AZURE_CLIENT_ID }}" ]; then + rad env update "$ENVIRONMENT" \ + --azure-subscription-id "${{ vars.AZURE_SUBSCRIPTION_ID }}" \ + --azure-resource-group "${{ vars.AZURE_RESOURCE_GROUP }}" + elif [ -n "${{ vars.AWS_IAM_ROLE_ARN }}" ]; then + rad env update "$ENVIRONMENT" \ + --aws-region "${{ vars.AWS_REGION }}" \ + --aws-account-id "${{ vars.AWS_ACCOUNT_ID }}" + fi + + - name: Configure cloud credentials for Radius pods + # AWS and Azure use different, provider-native credential models: + # + # AWS - the runner's OIDC session credentials are injected into the + # applications-rp, dynamic-rp, and bicep-de pods as environment + # variables. The AWS SDKs and Terraform AWS provider read them + # directly. No credential is registered with Radius. + # + # Azure - uses Workload Identity. The ClientID/TenantID are registered + # as a Radius credential (the Terraform azurerm provider fetches + # them from UCP), and the GitHub Actions OIDC JWT is projected + # into the pods as the federated token file the Azure SDKs + # exchange for an ~1h AAD token. NOTE: that GitHub OIDC JWT is + # short-lived; a run whose Azure work outlives the first token + # exchange window may fail. Refreshing it is a planned fast + # follow and intentionally out of scope here. + run: | + set -euo pipefail + + if [ -n "${{ vars.AWS_IAM_ROLE_ARN }}" ]; then + ENV_PATCH=$(jq -nc --arg ak "$AWS_ACCESS_KEY_ID" --arg sk "$AWS_SECRET_ACCESS_KEY" \ + --arg st "${AWS_SESSION_TOKEN:-}" --arg rg "${{ vars.AWS_REGION }}" ' + [ + {name:"AWS_ACCESS_KEY_ID", value:$ak}, + {name:"AWS_SECRET_ACCESS_KEY", value:$sk}, + {name:"AWS_SESSION_TOKEN", value:$st}, + {name:"AWS_REGION", value:$rg} + ]') + PATCH=$(jq -nc --argjson env "$ENV_PATCH" ' + [ $env[] | {op:"add", path:"/spec/template/spec/containers/0/env/-", value:.} ]') + for d in applications-rp dynamic-rp bicep-de; do + kubectl patch deployment "$d" -n radius-system --type=json -p="$PATCH" + done + + elif [ -n "${{ vars.AZURE_CLIENT_ID }}" ]; then + # Register the Workload Identity credential so the Terraform azurerm + # provider can fetch the client/tenant IDs from UCP. + rad credential register azure wi \ + --client-id "${{ vars.AZURE_CLIENT_ID }}" \ + --tenant-id "${{ vars.AZURE_TENANT_ID }}" + + # Mint the GitHub Actions OIDC JWT for the Entra token-exchange + # audience. This is the federated token the Azure SDKs present to + # Entra; the app registration's federated credential trusts GitHub's + # issuer + this environment's subject. + FEDERATED_TOKEN=$(curl -sSf \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=api://AzureADTokenExchange" | jq -r '.value') + + # Project the token as a Secret mounted at the well-known Azure + # Workload Identity path. A Secret *volume* is kept in sync by the + # kubelet, so all three pods (including bicep-de) see it without a + # restart. + kubectl create secret generic azure-identity-token -n radius-system \ + --from-literal=azure-identity-token="$FEDERATED_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + + ENV_PATCH=$(jq -nc \ + --arg ci "${{ vars.AZURE_CLIENT_ID }}" --arg ti "${{ vars.AZURE_TENANT_ID }}" ' + [ + {name:"AZURE_CLIENT_ID", value:$ci}, + {name:"AZURE_TENANT_ID", value:$ti}, + {name:"AZURE_FEDERATED_TOKEN_FILE", value:"/var/run/secrets/azure/tokens/azure-identity-token"}, + {name:"AZURE_AUTHORITY_HOST", value:"https://login.microsoftonline.com/"} + ]') + PATCH=$(jq -nc --argjson env "$ENV_PATCH" ' + [ {op:"add", path:"/spec/template/spec/volumes/-", + value:{name:"azure-identity-token", secret:{secretName:"azure-identity-token"}}}, + {op:"add", path:"/spec/template/spec/containers/0/volumeMounts/-", + value:{name:"azure-identity-token", mountPath:"/var/run/secrets/azure/tokens", readOnly:true}} ] + + [ $env[] | {op:"add", path:"/spec/template/spec/containers/0/env/-", value:.} ]') + for d in applications-rp dynamic-rp bicep-de; do + kubectl patch deployment "$d" -n radius-system --type=json -p="$PATCH" + done + + else + echo "No cloud provider configured; skipping credential configuration." + exit 0 + fi + + for d in applications-rp dynamic-rp bicep-de; do + kubectl rollout status "deployment/$d" -n radius-system --timeout=300s + done + + + - name: Register resource types and recipes + run: | + set -euo pipefail + REPO="$RESOURCE_TYPES_CONTRIB_REPO" + REF="$RESOURCE_TYPES_CONTRIB_REF" + git clone --depth 1 --branch "$REF" "$REPO" /tmp/resource-types-contrib + + for TYPE_YAML in \ + Compute/containerImages/containerImages.yaml \ + Data/mySqlDatabases/mySqlDatabases.yaml; do + if [ -f "/tmp/resource-types-contrib/$TYPE_YAML" ]; then + echo "Registering $TYPE_YAML..." + rad resource-type create -f "/tmp/resource-types-contrib/$TYPE_YAML" || \ + (echo "Retrying after 5s..." && sleep 5 && rad resource-type create -f "/tmp/resource-types-contrib/$TYPE_YAML") + fi + done + + register_recipe() { + local rtype="$1" path="$2"; shift 2 + rad recipe register default \ + --environment "$ENVIRONMENT" \ + --resource-type "$rtype" \ + --template-kind terraform \ + --template-path "git::$REPO//$path?ref=$REF" "$@" + } + register_recipe Radius.Compute/containerImages Compute/containerImages/recipes/kubernetes/terraform + register_recipe Radius.Compute/containers Compute/containers/recipes/kubernetes/terraform + register_recipe Radius.Compute/persistentVolumes Compute/persistentVolumes/recipes/kubernetes/terraform + register_recipe Radius.Compute/routes Compute/routes/recipes/kubernetes/terraform + register_recipe Radius.Data/postgreSqlDatabases Data/postgreSqlDatabases/recipes/kubernetes/terraform + register_recipe Radius.Security/secrets Security/secrets/recipes/kubernetes/terraform + if [ -n "${{ vars.RADIUS_VPC_ID }}" ]; then + register_recipe Radius.Data/mySqlDatabases Data/mySqlDatabases/recipes/aws/terraform \ + --parameters vpcId="${{ vars.RADIUS_VPC_ID }}" \ + --parameters "subnetIds=${{ vars.RADIUS_SUBNET_IDS }}" + fi + + - name: Refresh EKS token before running commands + # EKS tokens are short-lived; refresh the mounted kubeconfig right before + # running commands so the RPs authenticate with a fresh token. + if: ${{ vars.AWS_IAM_ROLE_ARN != '' && vars.RADIUS_K8S_CLUSTER != '' }} + run: | + set -euo pipefail + CLUSTER="${{ vars.RADIUS_K8S_CLUSTER }}" + REGION="${{ vars.AWS_REGION }}" + ENDPOINT=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.endpoint' --output text) + CA_DATA=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.certificateAuthority.data' --output text) + TOKEN=$(aws eks get-token --cluster-name "$CLUSTER" --region "$REGION" --output json | jq -r '.status.token') + printf 'apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: %s\n server: %s\n name: eks\ncontexts:\n- context:\n cluster: eks\n user: eks-user\n name: eks\ncurrent-context: eks\nkind: Config\nusers:\n- name: eks-user\n user:\n token: %s\n' "$CA_DATA" "$ENDPOINT" "$TOKEN" > "$TARGET_KUBECONFIG_PATH" + kubectl create secret generic "$TARGET_KUBECONFIG_SECRET" \ + --namespace radius-system \ + --from-file="$TARGET_KUBECONFIG_KEY=$TARGET_KUBECONFIG_PATH" \ + --dry-run=client -o yaml | kubectl apply -f - + for d in applications-rp dynamic-rp bicep-de; do + kubectl rollout restart "deployment/$d" -n radius-system + kubectl rollout status "deployment/$d" -n radius-system --timeout=300s + done + + - name: Run Radius commands + env: + # Pass caller input through the environment to avoid command injection. + RADIUS_COMMANDS: ${{ inputs.radius_commands }} + run: | + set -euo pipefail + mkdir -p /tmp/radius-output + + # radius_commands is either a single command string or a JSON array of + # command strings; the `rad` prefix is omitted from each. Normalize to a + # bash array. + if printf '%s' "$RADIUS_COMMANDS" | jq -e 'type == "array"' >/dev/null 2>&1; then + mapfile -t COMMANDS < <(printf '%s' "$RADIUS_COMMANDS" | jq -r '.[]') + else + COMMANDS=("$RADIUS_COMMANDS") + fi + + i=0 + for cmd in "${COMMANDS[@]}"; do + i=$((i + 1)) + [ -z "${cmd//[[:space:]]/}" ] && continue + OUT="/tmp/radius-output/$(printf '%02d' "$i")-output.txt" + echo "::group::rad $cmd" + # Each command's combined output is captured as an artifact so the + # frontend can read results incrementally. Stop on first failure. + if ! rad $cmd 2>&1 | tee "$OUT"; then + echo "::endgroup::" + echo "Command failed: rad $cmd" >&2 + exit 1 + fi + echo "::endgroup::" + done + echo "✅ Radius commands complete." + + - name: Upload command output + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: radius-output + path: /tmp/radius-output/ + retention-days: 1 + + - name: Back up Radius state + # Persist control-plane + Terraform state to the radius-state branch so the + # next run's `rad startup` can restore it. Runs even on deploy failure to + # avoid losing partially-applied state. + if: always() + run: rad shutdown --workspace default + + - name: Collect Radius logs + if: failure() + run: | + mkdir -p /tmp/radius-logs + for deploy in applications-rp dynamic-rp bicep-de controller ucpd; do + kubectl logs -n radius-system -l app.kubernetes.io/name=$deploy --tail=200 >> /tmp/radius-logs/$deploy.log 2>&1 || true + done + kubectl get pods -n radius-system -o wide >> /tmp/radius-logs/pods.txt 2>&1 || true + kubectl get events -n radius-system --sort-by=.lastTimestamp >> /tmp/radius-logs/events.txt 2>&1 || true + + - name: Upload Radius logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: radius-logs + path: /tmp/radius-logs/ + retention-days: 3 + + - name: Cleanup control plane cluster + if: always() + run: k3d cluster delete radius-cp || true diff --git a/.github/extension/radius-verify-credentials.yml b/.github/extension/radius-verify-credentials.yml new file mode 100644 index 00000000000..a732182ae75 --- /dev/null +++ b/.github/extension/radius-verify-credentials.yml @@ -0,0 +1,115 @@ +# This workflow is auto-generated by Radius to verify cloud credentials. +name: Radius - Verify Cloud Credentials + +on: + workflow_dispatch: + inputs: + environment: + description: 'GitHub Environment name' + required: true + default: 'dev' + +permissions: + id-token: write + contents: read + +jobs: + verify: + name: Verify cloud credentials + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Validate provider configuration + run: | + set -euo pipefail + AZURE_CLIENT_ID="${{ vars.AZURE_CLIENT_ID }}" + AWS_IAM_ROLE_ARN="${{ vars.AWS_IAM_ROLE_ARN }}" + + if [ -n "${AZURE_CLIENT_ID}" ] && [ -n "${AWS_IAM_ROLE_ARN}" ]; then + echo "Both AZURE_CLIENT_ID and AWS_IAM_ROLE_ARN are set. Configure only one provider per GitHub Environment." >&2 + exit 1 + fi + + if [ -n "${AZURE_CLIENT_ID}" ] && { [ -z "${{ vars.AZURE_TENANT_ID }}" ] || [ -z "${{ vars.AZURE_SUBSCRIPTION_ID }}" ]; }; then + echo "Azure variables are incomplete; require AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID." >&2 + exit 1 + fi + + if [ -n "${AWS_IAM_ROLE_ARN}" ] && [ -z "${{ vars.AWS_REGION }}" ]; then + echo "AWS variables are incomplete; require AWS_IAM_ROLE_ARN and AWS_REGION." >&2 + exit 1 + fi + + - name: Azure Login (OIDC) + if: ${{ vars.AZURE_CLIENT_ID != '' }} + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Verify Azure access + if: ${{ vars.AZURE_CLIENT_ID != '' }} + run: | + set -euo pipefail + echo "Verifying Azure access..." + az account show --query '{name:name, state:state}' --output table + echo "" + echo "✅ Azure credentials are working correctly." + + - name: Configure AWS Credentials (OIDC) + if: ${{ vars.AWS_IAM_ROLE_ARN != '' }} + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ vars.AWS_IAM_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + - name: Verify AWS access + if: ${{ vars.AWS_IAM_ROLE_ARN != '' }} + run: | + set -euo pipefail + echo "Verifying AWS access..." + CALLER_ARN=$(aws sts get-caller-identity --query 'Arn' --output text) + echo "${CALLER_ARN}" | sed 's/[0-9]\{12\}/****/g' + echo "" + echo "✅ AWS credentials are working correctly." + + - name: Discover AWS resources + if: ${{ vars.AWS_IAM_ROLE_ARN != '' }} + run: | + set -euo pipefail + REGION="${{ vars.AWS_REGION }}" + CLUSTERS=$(aws eks list-clusters --region "$REGION" --query 'clusters' --output json) + VPCS=$(aws ec2 describe-vpcs --region "$REGION" --query 'Vpcs[].{id:VpcId,name:Tags[?Key==`Name`].Value|[0],isDefault:IsDefault}' --output json) + SUBNETS=$(aws ec2 describe-subnets --region "$REGION" --query 'Subnets[].{id:SubnetId,vpcId:VpcId,az:AvailabilityZone,name:Tags[?Key==`Name`].Value|[0]}' --output json) + printf '{"eksClusters":%s,"vpcs":%s,"subnets":%s}' "$CLUSTERS" "$VPCS" "$SUBNETS" > /tmp/radius-discovery.json + + - name: Upload discovery data + if: ${{ vars.AWS_IAM_ROLE_ARN != '' && success() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: radius-discovery + path: /tmp/radius-discovery.json + retention-days: 1 + + - name: Discover Azure resources + if: ${{ vars.AZURE_CLIENT_ID != '' }} + run: | + set -euo pipefail + SUB="${{ vars.AZURE_SUBSCRIPTION_ID }}" + RESOURCE_GROUPS=$(az group list --subscription "$SUB" --query '[].{name:name, location:location}' --output json) + AKS_CLUSTERS=$(az aks list --subscription "$SUB" --query '[].{name:name, resourceGroup:resourceGroup, location:location}' --output json) + LOCATIONS=$(az account list-locations --query "[?metadata.regionType=='Physical'].{name:name, displayName:displayName}" --output json) + printf '{"resourceGroups":%s,"aksClusters":%s,"locations":%s}' "$RESOURCE_GROUPS" "$AKS_CLUSTERS" "$LOCATIONS" > /tmp/radius-discovery.json + + - name: Upload Azure discovery data + if: ${{ vars.AZURE_CLIENT_ID != '' && success() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: radius-discovery + path: /tmp/radius-discovery.json + retention-days: 1 diff --git a/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md b/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md new file mode 100644 index 00000000000..584c5268603 --- /dev/null +++ b/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md @@ -0,0 +1,234 @@ +# Repo Radius — Deploy Workflow (Technical Design) + +- **Author**: Sylvain Niles (@sylvainsf) +- **Status**: Draft +- **Feature spec**: Repo Radius (Zach Casper) — [PR #12078](https://github.com/radius-project/radius/pull/12078) +- **Issue**: [#12118 Add Repo Radius verify/deploy workflows to the repo](https://github.com/radius-project/radius/issues/12118) +- **Depends on**: [#12106 Multi-cluster deployment v1](https://github.com/radius-project/radius/pull/12106) (merged), [#12214 Repo Radius state storage (`rad startup` / `rad shutdown`)](https://github.com/radius-project/radius/pull/12214), [#12170 Repo Radius verify workflow](https://github.com/radius-project/radius/pull/12170) + +## Scope + +This document covers **Investment 3 of the Repo Radius feature spec: the Repo Radius workflow +with standardized inputs and outputs** — specifically the `deploy` workflow that runs Radius on +demand inside a GitHub Actions runner. It also documents the companion `verify` workflow (from +[#12170](https://github.com/radius-project/radius/pull/12170)) because the two share a contract +and a home. + +The workflows are **templates**: a frontend (the Copilot app, the CLI, etc.) writes a copy into a +user's repository under `.github/workflows/` and dispatches it there. They are not run from the +`radius-project/radius` repository directly. They live at +[`.github/extension/`](../../../.github/extension/) so the contract has a canonical, reviewed home. + +Explicitly **out of scope** for this document: + +- **Cloud-side OIDC / permission provisioning** — creating the AWS IAM role + trust policy or the + Entra app registration + federated credential. The workflows *consume* an environment that is + already federated; standing that up is tracked separately. +- **The state-storage mechanism** (`rad startup` / `rad shutdown`, the `radius-state` git orphan + branch) — owned by the [state-storage design](../2026-06-repo-radius-state-storage.md). +- **The multi-cluster seam internals** (`global.targetCluster`, the cluster access resolver) — + owned by the [multi-cluster design](2026-06-multi-cluster.md). This document only describes how + the workflow *drives* that seam. +- **Mid-run cloud-token refresh** — a known limitation (see [Credential lifetime](#credential-lifetime)), + deferred to a fast follow. + +## Background + +Repo Radius runs the Radius control plane on an **ephemeral k3d cluster** inside a GitHub Actions +runner. The cluster is created at the start of a run and destroyed at the end; application +workloads deploy to the developer's **external** AKS/EKS cluster, not the runner cluster. For this +to work, three independent pieces — each landing as its own PR — must be composed by a single +workflow: + +| Piece | Provides | Owner | +|------------------|-------------------------------------------------------------------------------------------|---------------------------------------------------------------| +| Multi-cluster v1 | `RADIUS_TARGET_KUBECONFIG` seam (chart `global.targetCluster.enabled`) | [#12106](https://github.com/radius-project/radius/pull/12106) | +| State storage | `rad startup` / `rad shutdown` + `database.enabled=true` chart wiring | [#12214](https://github.com/radius-project/radius/pull/12214) | +| Workflow (this) | The orchestration that installs Radius, restores state, runs commands, and persists state | this design | + +The prototype that proved the end-to-end flow lives as a generated string +(`DEPLOY_WORKFLOW`) in the [`github-extension`](https://github.com/radius-project/github-extension) +repo. `github-extension` never merges into Radius, so the contract it depends on must have a +reviewed home here. This design ports that prototype and adapts it to the merged building blocks +above. + +## The dispatch contract (stable; frontends depend on it) + +The frontend drives Repo Radius entirely through the GitHub API. The contract is the +`workflow_dispatch` input set plus the GitHub Environment the run binds to. + +### Inputs + +| Input | Required | Description | +|-------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `environment` | Yes | The GitHub Environment name. Used as the Radius environment name and to bind the job (`environment: ${{ inputs.environment }}`) so its variables and OIDC subject apply. | +| `radius_commands` | Yes | A single `rad` CLI command string, **or** a JSON-encoded array of command strings run in order. The `rad` prefix is omitted from each command (e.g. `deploy app.bicep` or `["deploy app.bicep", "app graph"]`). | + +This matches the feature spec's Step 3 exactly. The single workflow runs arbitrary `rad` +commands; it is **not** deploy-specific. Commands run in order and the run **stops on the first +failure**, then still persists state (below). + +### Outputs + +Each command's combined stdout/stderr is captured and uploaded as the `radius-output` artifact +(one numbered file per command). Per the feature spec's Step 5, artifacts are readable via the +GitHub API as soon as each upload completes, so the frontend can poll for results incrementally +while the run is still in progress. + +### GitHub Environment variables + +The workflows read only Actions **variables** (`vars`), never secrets, for cloud configuration. A +provider's branch runs only when its identifying variable (`AZURE_CLIENT_ID` or +`AWS_IAM_ROLE_ARN`) is non-empty; configuring both on one environment is rejected. + +| Provider | Variables | +|----------|----------------------------------------------------------------------------------------------------------------| +| Azure | `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`, `AZURE_RESOURCE_GROUP`, `RADIUS_K8S_CLUSTER` | +| AWS | `AWS_IAM_ROLE_ARN`, `AWS_REGION`, `AWS_ACCOUNT_ID`, `RADIUS_K8S_CLUSTER`, `RADIUS_VPC_ID`, `RADIUS_SUBNET_IDS` | +| Common | `RADIUS_K8S_NAMESPACE` (target namespace, defaults to `default`) | + +### Permissions + +`id-token: write` (OIDC), `contents: write` (so `rad shutdown` can push the `radius-state` +branch), and `packages: write` (so container-image recipes can push to GHCR). + +## Deploy workflow stages + +```mermaid +flowchart TD + A[OIDC login
azure/login or configure-aws-credentials] --> B[Build target-cluster kubeconfig
az aks get-credentials / aws eks get-token] + B --> C[Create k3d control plane
+ install rad CLI] + C --> D[Create target-kubeconfig secret] + D --> E[rad install kubernetes
database.enabled=true
global.targetCluster.enabled=true
global.azureWorkloadIdentity.enabled=true*] + E --> F[rad startup
restore PostgreSQL + Terraform state] + F --> G[rad env create/update
scope only] + G --> H[Configure cloud credentials
AWS: env vars · Azure: WI token] + H --> I[Register resource types + recipes] + I --> J[Refresh EKS token*
then run radius_commands
upload radius-output artifact] + J --> K[rad shutdown
back up + push radius-state] + K --> L[Delete k3d cluster] +``` + +\* Azure-only / AWS-only steps are skipped when the other provider is configured. + +### Why the order matters + +- **`rad startup` runs after `rad install` but before any command.** It waits for the + control-plane PostgreSQL to be ready, then restores both the control-plane databases and the + Terraform recipe-state Secrets from the previous run, so the first `rad deploy` plans against + prior state rather than an empty backend. +- **`rad shutdown` runs after the commands, with `if: always()`.** State is persisted even when a + command fails, so a partially-applied Terraform run is not lost. +- **Credentials are configured after `rad env create`** because the Azure path registers a Radius + credential against the environment and both paths patch the RP/DE deployments. + +## The integration contract (owned by Radius) + +### Target cluster — `RADIUS_TARGET_KUBECONFIG` + +The workflow builds a kubeconfig for the external workload cluster on the runner and stores it as +the `target-kubeconfig` Secret in `radius-system`. Installing the chart with +`--set global.targetCluster.enabled=true` mounts that Secret into `applications-rp`, +`dynamic-rp`, and `bicep-de` and sets `RADIUS_TARGET_KUBECONFIG` to the mounted path. Radius then +directs recipe execution **and** directly-rendered output resources at that cluster; the Terraform +kubernetes provider follows the same kubeconfig through the cluster access resolver. The Secret's +lifecycle (creation, EKS-token refresh, RBAC) is the workflow's responsibility, not the chart's. +This is the multi-cluster v1 seam; `KUBE_CONFIG_PATH` from the prototype is no longer needed — +the single env var now drives both Bicep and Terraform. + +### Cloud credentials — provider-native + +Repo Radius stores no long-lived cloud secrets. The two providers use different, provider-native +models: + +- **AWS** follows Terraform's model. The workflow does **not** run `rad credential register`. + The runner's OIDC session credentials are injected into the `applications-rp`, `dynamic-rp`, and + `bicep-de` pods as environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, + `AWS_SESSION_TOKEN`, `AWS_REGION`); the AWS SDKs and the Terraform AWS provider read them + directly. +- **Azure** uses **Workload Identity** — the only path the merged code supports. The chart is + installed with `global.azureWorkloadIdentity.enabled=true` (which labels the RP/DE pods), the + client/tenant IDs are registered with `rad credential register azure wi` (the Terraform + `azurerm` provider fetches them from UCP, and authenticates with `use_oidc=true` and the + hard-coded token path `/var/run/secrets/azure/tokens/azure-identity-token`), and the workflow + projects the federated token into that path (below). + +In both cases `rad env update` records only the deployment **scope** (AWS region/account, Azure +subscription/resource group), never credentials. + +#### Azure Workload Identity on an ephemeral cluster + +Normal AKS Workload Identity federates the *cluster's* OIDC issuer and relies on the WI admission +webhook to project an auto-refreshed ServiceAccount token. Neither exists on the runner's k3d +cluster, so the workflow substitutes both: + +1. The Entra app registration's federated credential trusts **GitHub's** OIDC issuer + (`token.actions.githubusercontent.com`), subject `repo:/:environment:`, + audience `api://AzureADTokenExchange`. This is what "set workload identity on the GitHub + Environment" means. +2. The workflow mints the GitHub Actions OIDC JWT for that audience and writes it to a Secret, + mounted as a **volume** at `/var/run/secrets/azure/tokens/azure-identity-token` on + `applications-rp`, `dynamic-rp`, and `bicep-de`, alongside the `AZURE_CLIENT_ID`, + `AZURE_TENANT_ID`, `AZURE_FEDERATED_TOKEN_FILE`, and `AZURE_AUTHORITY_HOST` env vars the Azure + SDKs read. + +Mounting the token as a Secret volume means the kubelet keeps the file in sync across all three +pods — including the Deployment Engine — with no pod restart, which is why the Azure path needs no +`rollout restart` (unlike the AWS/EKS refresh, below). + +#### Credential lifetime + +- **Azure** — the GitHub Actions OIDC JWT is short-lived, but the Azure SDKs exchange it **once** + with Entra for an AAD access token valid ~1 hour, cached in memory. A normal run is unaffected; + the federated token file is only re-read after the cached AAD token expires. The workflow mints + the token **once** and does not refresh it: a run whose Azure work outlives that window may fail. + Refreshing the federated token mid-run is an accepted, deferred fast follow. +- **AWS** — the EKS bearer token is valid ~15 minutes and is used **directly** on every Kubernetes + API call (no exchange, no caching). The workflow therefore re-mints it and rewrites the + `target-kubeconfig` Secret immediately before running the commands, then restarts the RP/DE + deployments to pick it up. + +### State persistence — `rad startup` / `rad shutdown` + +`rad startup` and `rad shutdown` are kind-agnostic CLI commands that back up and restore all +durable Radius state (control-plane PostgreSQL + Terraform recipe-state Secrets) to a +`radius-state` git orphan branch. They do not manage cluster lifecycle — the workflow owns +creating and destroying the ephemeral control plane around them. The mechanism is the plan of +record; see the [state-storage design](../2026-06-repo-radius-state-storage.md) for details. + +## The verify workflow + +[`radius-verify-credentials.yml`](../../../.github/extension/radius-verify-credentials.yml) +([#12170](https://github.com/radius-project/radius/pull/12170)) is a separate, lighter +`workflow_dispatch` workflow that confirms a GitHub Environment is wired correctly **before any +deploy**. It authenticates via OIDC, verifies access (`az account show` / `aws sts +get-caller-identity` with the AWS account ID masked), and discovers cloud resources (resource +groups / AKS clusters / locations for Azure; EKS clusters / VPCs / subnets for AWS), uploading them +as the `radius-discovery` artifact for the caller to read back into the environment's variables. It +shares the OIDC-trust prerequisites and the `vars`-only convention with the deploy workflow, which +is why both live under `.github/extension/`. Porting it here lets the contract be exercised without +a full deploy. + +## Testing + +The gated end-to-end lifecycle test `test/functional-portable/statestore` (behind +`RADIUS_STATE_E2E`) exercises install → deploy a Terraform-backed resource → `rad shutdown` → +teardown → reinstall → `rad startup` → deploy an update. Today it drives install/uninstall itself; +once this workflow lands it should be re-pointed at the workflow's cluster-create + deploy stages +rather than duplicating them. + +## Alternatives considered + +- **Keep the workflow only in `github-extension`.** Rejected: the contract Radius owns + (`RADIUS_TARGET_KUBECONFIG`, `rad startup`/`rad shutdown`, the dispatch inputs) would live only + in a prototype string in a separate repo, with no review or stability guarantee for the + frontends that depend on it. +- **Register AWS credentials with `rad credential register` (as the prototype did).** Replaced by + environment-variable injection per the feature spec (Shruthi's change aligning AWS with + Terraform's model); it removes a control-plane round-trip and keeps credentials out of Radius + state. +- **Inject Azure credentials as env vars like AWS.** Not possible: the merged Azure code paths + (control-plane `azidentity` and the Terraform `azurerm` provider) only support Workload Identity, + which requires a projected federated-token file plus a registered WI credential. +- **Refresh the Azure federated token mid-run.** Deferred. The one-time exchange covers ~1 hour, + which is sufficient for current demo deploys; long-running deploys are an accepted limitation. From f8d407e8bed93110402cb791dbad3b3e0938240b Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 16:48:10 -0700 Subject: [PATCH 2/6] docs(design): genericize Repo Radius deploy-workflow design note Remove references to the external prototype project and speak in generic terms (an earlier proof of concept, a separate project) so the design does not rely on naming an external repository. Related: #12118 Signed-off-by: Sylvain Niles --- .../2026-06-repo-radius-deploy-workflow.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md b/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md index 584c5268603..9745768a606 100644 --- a/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md +++ b/eng/design-notes/environments/2026-06-repo-radius-deploy-workflow.md @@ -46,11 +46,12 @@ workflow: | State storage | `rad startup` / `rad shutdown` + `database.enabled=true` chart wiring | [#12214](https://github.com/radius-project/radius/pull/12214) | | Workflow (this) | The orchestration that installs Radius, restores state, runs commands, and persists state | this design | -The prototype that proved the end-to-end flow lives as a generated string -(`DEPLOY_WORKFLOW`) in the [`github-extension`](https://github.com/radius-project/github-extension) -repo. `github-extension` never merges into Radius, so the contract it depends on must have a -reviewed home here. This design ports that prototype and adapts it to the merged building blocks -above. +An earlier proof of concept validated this end-to-end flow but kept the workflow +as a generated string outside Radius, where the contract it depends on had no +reviewed home and no stability guarantee. Bringing the workflow in-tree gives that +contract a canonical, reviewed home and removes any reliance on an external +project. This design lands the workflow here and adapts it to the merged building +blocks above. ## The dispatch contract (stable; frontends depend on it) @@ -133,7 +134,8 @@ the `target-kubeconfig` Secret in `radius-system`. Installing the chart with directs recipe execution **and** directly-rendered output resources at that cluster; the Terraform kubernetes provider follows the same kubeconfig through the cluster access resolver. The Secret's lifecycle (creation, EKS-token refresh, RBAC) is the workflow's responsibility, not the chart's. -This is the multi-cluster v1 seam; `KUBE_CONFIG_PATH` from the prototype is no longer needed — +This is the multi-cluster v1 seam; the separate `KUBE_CONFIG_PATH` variable used by +the earlier proof of concept is no longer needed — the single env var now drives both Bicep and Terraform. ### Cloud credentials — provider-native @@ -219,12 +221,12 @@ rather than duplicating them. ## Alternatives considered -- **Keep the workflow only in `github-extension`.** Rejected: the contract Radius owns +- **Keep the workflow outside Radius.** Rejected: the contract Radius owns (`RADIUS_TARGET_KUBECONFIG`, `rad startup`/`rad shutdown`, the dispatch inputs) would live only - in a prototype string in a separate repo, with no review or stability guarantee for the + in a generated string in a separate project, with no review or stability guarantee for the frontends that depend on it. -- **Register AWS credentials with `rad credential register` (as the prototype did).** Replaced by - environment-variable injection per the feature spec (Shruthi's change aligning AWS with +- **Register AWS credentials with `rad credential register`.** Replaced by + environment-variable injection per the feature spec (aligning AWS with Terraform's model); it removes a control-plane round-trip and keeps credentials out of Radius state. - **Inject Azure credentials as env vars like AWS.** Not possible: the merged Azure code paths From 122d84e9156fe616fac7b52b40bb94f4645e40ca Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 18:33:32 -0700 Subject: [PATCH 3/6] test(statestore): run the startup/shutdown lifecycle test in CI, un-gated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-point the `rad startup` / `rad shutdown` state-storage lifecycle test at this PR's deploy workflow model and remove the `RADIUS_STATE_E2E` gate so it runs in CI on its own dedicated, isolated cluster. - Add a `statestore-noncloud` leg to the non-cloud functional matrix. Each matrix leg runs on its own runner with its own KinD cluster, so the test's destructive install/uninstall/reinstall cycle never affects other legs. The shared "Install Radius" step is skipped for this leg because the test drives its own install. - Drive install with the build-under-test images (chart + per-RP image flags from testutil.SetDefault, DE_IMAGE/DE_TAG, and the secure local registry CA), mirroring the shared Install Radius step, plus `database.enabled=true` for the state backend. - Harden the lifecycle against the flakes seen in the upgrade test (#12245): replace fixed sleeps with polling — wait for the control plane treating 503 from the UCP aggregated APIService as retryable, and poll discovery until `api.ucp.dev/v1alpha3` deregisters before reinstalling so the next install doesn't race the teardown. - Add the `test-functional-statestore-noncloud` make target and a 40m timeout for the leg. Related: #12118 Signed-off-by: Sylvain Niles --- .../workflows/functional-test-noncloud.yaml | 12 +- build/test.mk | 7 + .../noncloud/statestore_lifecycle_test.go | 151 +++++++++++++++--- 3 files changed, 143 insertions(+), 27 deletions(-) diff --git a/.github/workflows/functional-test-noncloud.yaml b/.github/workflows/functional-test-noncloud.yaml index 59b6358c248..7666b435776 100644 --- a/.github/workflows/functional-test-noncloud.yaml +++ b/.github/workflows/functional-test-noncloud.yaml @@ -181,6 +181,7 @@ jobs: kubernetes-noncloud, msgrp-noncloud, multicluster-noncloud, + statestore-noncloud, samples-noncloud, ucp-noncloud, datastoresrp-noncloud, @@ -421,6 +422,10 @@ jobs: } >> "${GITHUB_ENV}" - name: Install Radius + # The statestore leg drives its own install/uninstall/reinstall lifecycle + # inside the test (it must reinstall to simulate an ephemeral control + # plane), so the shared install is skipped for it. + if: matrix.name != 'statestore-noncloud' run: | export PATH=$GITHUB_WORKSPACE/bin:$PATH which rad || { echo "cannot find rad"; exit 1; } @@ -611,7 +616,9 @@ jobs: make "test-functional-${MATRIX_NAME}" env: DOCKER_REGISTRY: ${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }} - TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }} + # The statestore leg installs/uninstalls/reinstalls Radius, so it needs + # far longer than the standard per-leg timeout. + TEST_TIMEOUT: ${{ matrix.name == 'statestore-noncloud' && '40m' || env.FUNCTIONALTEST_TIMEOUT }} RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }} RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples BICEP_RECIPE_REGISTRY: ${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }} @@ -621,6 +628,9 @@ jobs: RADIUS_TEST_FAST_CLEANUP: true GIT_HTTP_PASSWORD: ${{ env.GIT_HTTP_PASSWORD }} MATRIX_NAME: ${{ matrix.name }} + # Used by the statestore leg's in-test `rad install` to trust the secure + # local registry; ignored by other legs. + RADIUS_REGISTRY_CERT_FILE: ${{ steps.create-local-registry.outputs.temp-cert-dir }}/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt - name: Process Functional Test Results uses: ./.github/actions/process-test-results diff --git a/build/test.mk b/build/test.mk index 0750cb586fc..68646e8dbf4 100644 --- a/build/test.mk +++ b/build/test.mk @@ -148,6 +148,13 @@ test-functional-multicluster-noncloud: ## Runs multi-cluster functional tests th # because of that extra setup. CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/multicluster/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 1 $(GOTEST_OPTS) +.PHONY: test-functional-statestore-noncloud +test-functional-statestore-noncloud: ## Runs the rad startup/shutdown state-storage lifecycle test + # Destructive: the test installs, uninstalls (--purge), and reinstalls Radius + # to simulate an ephemeral control plane, so it must run on a dedicated cluster + # and never alongside other functional legs. Not part of test-functional-all-noncloud. + CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/statestore/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 1 $(GOTEST_OPTS) + .PHONY: test-functional-upgrade test-functional-upgrade: test-functional-upgrade-noncloud ## Runs all Upgrade functional tests diff --git a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go index a738b445ee2..00ed28f5d57 100644 --- a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go +++ b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go @@ -23,21 +23,16 @@ limitations under the License. // proves that both the control-plane databases and the Terraform state Secrets survived the // teardown. // -// The test is destructive (it uninstalls and reinstalls Radius on the target cluster) and requires -// a real cluster plus the Terraform recipe module server, so it does not run as part of the normal -// functional suite. It is skipped unless RADIUS_STATE_E2E is set to a truthy value. -// -// Test dependency: rad startup/shutdown do not create clusters or install Radius. This test -// currently drives the cluster install/uninstall itself (installRadius/uninstallRadius below). It -// is expected to depend on the separate Repo Radius workflow code (in flight) that creates the -// ephemeral cluster, installs Radius, and runs the deploy. Once that lands, re-point the helpers -// at the shared workflow code instead of duplicating the install/uninstall steps here. +// The test is destructive: it uninstalls and reinstalls Radius (`--purge`) to simulate the +// ephemeral control plane that Repo Radius runs on. Because of that it runs on its own dedicated +// cluster in CI (the `statestore-noncloud` leg), never alongside other functional tests, and +// drives its own install/uninstall instead of relying on the shared "Install Radius" CI step. package statestore import ( "context" + "fmt" "os" - "strconv" "testing" "time" @@ -47,6 +42,7 @@ import ( "github.com/radius-project/radius/test" "github.com/radius-project/radius/test/functional-portable/corerp" "github.com/radius-project/radius/test/radcli" + "github.com/radius-project/radius/test/rp" "github.com/radius-project/radius/test/testutil" ) @@ -54,40 +50,134 @@ const ( stateNamespace = "radius-system" secretPrefix = "tfstate-default-" + // resourceGroup is the Radius resource group the test deploys into. It must match the group + // segment of resourceID below so the Terraform state secret name resolves correctly. + resourceGroup = "kind-radius" + + // relativeChartPath points at the in-repo Helm chart so the test installs the build under test. + relativeChartPath = "../../../deploy/Chart" + // redisRecipeTemplate is the Terraform recipe fixture shared with the corerp recipe tests. redisRecipeTemplate = "../../corerp/noncloud/resources/testdata/corerp-resources-terraform-redis.bicep" + + // controlPlaneTimeout is how long to wait for the control plane API to become available after an + // install. It is generous because the UCP aggregated APIService may briefly return 503 while the + // pods roll. (Lesson from the flaky upgrade test, PR #12245.) + controlPlaneTimeout = 5 * time.Minute + controlPlanePollInterval = 5 * time.Second + + // apiServiceDeregistrationTimeout bounds the wait for the Radius aggregated APIService to + // deregister after an uninstall. Reinstalling while `api.ucp.dev/v1alpha3` is still registered + // makes the API server return 503, which flakes the next install. (Lesson from PR #12245.) + apiServiceDeregistrationTimeout = 60 * time.Second + apiServiceDeregistrationInterval = 2 * time.Second + radiusAPIGroupVersion = "api.ucp.dev/v1alpha3" + + // podTerminationTimeout bounds the wait for Radius pods to disappear after an uninstall. + podTerminationTimeout = 2 * time.Minute + podTerminationPoll = 5 * time.Second + radiusPodSelector = "app.kubernetes.io/part-of=radius" ) -// shouldRun reports whether the destructive lifecycle test has been opted into. -func shouldRun(t *testing.T) { +// installRadius installs Radius with the PostgreSQL state backend enabled, using the images and +// chart of the build under test. In CI the registry/tag come from DOCKER_REGISTRY/REL_VERSION and +// the secure local registry's CA is supplied via RADIUS_REGISTRY_CERT_FILE; locally it falls back +// to the public images. +func installRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { t.Helper() - v, _ := strconv.ParseBool(os.Getenv("RADIUS_STATE_E2E")) - if !v { - t.Skip("set RADIUS_STATE_E2E=1 to run the destructive rad startup/shutdown lifecycle test") + registry, tag := testutil.SetDefault() + + args := []string{ + "install", "kubernetes", + "--chart", relativeChartPath, + "--set", fmt.Sprintf("rp.image=%s/applications-rp,rp.tag=%s", registry, tag), + "--set", fmt.Sprintf("dynamicrp.image=%s/dynamic-rp,dynamicrp.tag=%s", registry, tag), + "--set", fmt.Sprintf("controller.image=%s/controller,controller.tag=%s", registry, tag), + "--set", fmt.Sprintf("ucp.image=%s/ucpd,ucp.tag=%s", registry, tag), + "--set", fmt.Sprintf("bicep.image=%s/bicep,bicep.tag=%s", registry, tag), + "--set", fmt.Sprintf("preupgrade.image=%s/pre-upgrade,preupgrade.tag=%s", registry, tag), + "--set", "database.enabled=true", + } + if deImage := os.Getenv("DE_IMAGE"); deImage != "" { + args = append(args, "--set", fmt.Sprintf("de.image=%s,de.tag=%s", deImage, os.Getenv("DE_TAG"))) + } + if cert := os.Getenv("RADIUS_REGISTRY_CERT_FILE"); cert != "" { + args = append(args, "--set-file", "global.rootCA.cert="+cert) } -} -// installRadius installs Radius with the PostgreSQL state backend enabled. -func installRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { - t.Helper() - out, err := cli.RunCommand(ctx, []string{"install", "kubernetes", "--set", "database.enabled=true"}) + out, err := cli.RunCommand(ctx, args) require.NoErrorf(t, err, "rad install failed: %s", out) + waitForControlPlane(t, ctx) } // uninstallRadius removes Radius and its state so the next install starts from an empty control -// plane, simulating an ephemeral teardown. +// plane, simulating an ephemeral teardown. It then waits for the Radius pods to terminate and the +// aggregated APIService to deregister so a subsequent install does not race the teardown. func uninstallRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { t.Helper() out, err := cli.RunCommand(ctx, []string{"uninstall", "kubernetes", "--purge"}) require.NoErrorf(t, err, "rad uninstall failed: %s", out) + waitForCleanTeardown(t, ctx) +} + +// waitForControlPlane polls until the Radius control plane API is reachable, treating transient +// errors (including 503 from the aggregated APIService while pods roll) as retryable. +func waitForControlPlane(t *testing.T, _ context.Context) { + t.Helper() + require.Eventually(t, func() bool { + ready := false + func() { + // NewRPTestOptions calls require/panic internally on a not-yet-ready control plane; + // catch that and treat it as "retry". + defer func() { _ = recover() }() + opts := rp.NewRPTestOptions(t) + ready = opts.ManagementClient != nil + }() + return ready + }, controlPlaneTimeout, controlPlanePollInterval, "control plane did not become available within timeout") +} + +// waitForCleanTeardown waits for Radius pods to terminate and the aggregated APIService to +// deregister after an uninstall, so the next install does not race a half-torn-down control plane. +func waitForCleanTeardown(t *testing.T, ctx context.Context) { + t.Helper() + k8s := test.NewTestOptions(t).K8sClient + + require.Eventually(t, func() bool { + pods, err := k8s.CoreV1().Pods(stateNamespace).List(ctx, metav1.ListOptions{LabelSelector: radiusPodSelector}) + if err != nil { + t.Logf("waiting to list pods: %v", err) + return false + } + if len(pods.Items) == 0 { + return true + } + t.Logf("waiting for %d Radius pod(s) to terminate...", len(pods.Items)) + return false + }, podTerminationTimeout, podTerminationPoll, "Radius pods did not terminate within timeout") + + // A 503 from the aggregated APIService means it is still registered but its backend is gone; + // poll discovery until the Radius API group is no longer served. + require.Eventually(t, func() bool { + _, resources, err := k8s.Discovery().ServerGroupsAndResources() + if err != nil { + // Partial results are expected mid-deregistration; inspect what we got. + t.Logf("discovery returned partial results (expected during deregistration): %v", err) + } + for _, rl := range resources { + if rl != nil && rl.GroupVersion == radiusAPIGroupVersion { + t.Log("Radius aggregated APIService still registered, waiting...") + return false + } + } + return true + }, apiServiceDeregistrationTimeout, apiServiceDeregistrationInterval, "aggregated APIService did not deregister within timeout") } // Test_StateStore_ShutdownStartup_TerraformCrossDeploy exercises every state path: // install, deploy a Terraform resource, shut down (backup), tear down, start up (restore), then // deploy an update to the same resource. func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { - shouldRun(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() @@ -98,7 +188,7 @@ func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { resourceName := "statestore-tf-redis" redisCacheName := "statestore-redis" - resourceID := "/planes/radius/local/resourcegroups/kind-radius/providers/Applications.Core/extenders/" + resourceName + resourceID := "/planes/radius/local/resourcegroups/" + resourceGroup + "/providers/Applications.Core/extenders/" + resourceName secretSuffix, err := corerp.GetSecretSuffix(resourceID, envName, appName) require.NoError(t, err) secretName := secretPrefix + secretSuffix @@ -122,17 +212,26 @@ func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { return getErr == nil } - // 1. Fresh install with PostgreSQL state backend. + // 1. Fresh install with the PostgreSQL state backend. installRadius(ctx, t, cli) t.Cleanup(func() { uninstallRadius(context.Background(), t, cli) }) + // Create the workspace and resource group the test deploys into (the shared CI "Install Radius" + // step is skipped for this leg, so the test owns this setup). + out, err := cli.RunCommand(ctx, []string{"workspace", "create", "kubernetes", "--force"}) + require.NoErrorf(t, err, "rad workspace create failed: %s", out) + out, err = cli.RunCommand(ctx, []string{"group", "create", resourceGroup}) + require.NoErrorf(t, err, "rad group create failed: %s", out) + out, err = cli.RunCommand(ctx, []string{"group", "switch", resourceGroup}) + require.NoErrorf(t, err, "rad group switch failed: %s", out) + // 2. Deploy the Terraform-backed resource. This creates control-plane state and a Terraform // state Secret. deploy() require.True(t, secretExists(), "Terraform state secret should exist after the first deploy") // 3. Back up all durable state. - out, err := cli.RunCommand(ctx, []string{"shutdown"}) + out, err = cli.RunCommand(ctx, []string{"shutdown"}) require.NoErrorf(t, err, "rad shutdown failed: %s", out) // 4. Tear the control plane down completely (ephemeral teardown). From b0cd1de7a8bae5e4c2c71d5fb1f3e476387c4e1e Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 22:27:18 -0700 Subject: [PATCH 4/6] fix(statestore): correct relative Helm chart path for deeper test package The statestore test lives one directory deeper than the upgrade test it was modeled on (statestore/noncloud vs upgrade), so the chart path needs four ../ segments to reach the repo root, not three. CI failed with 'stat ../../../deploy/Chart: no such file or directory'. Signed-off-by: Sylvain Niles --- .../statestore/noncloud/statestore_lifecycle_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go index 00ed28f5d57..bd1d2a47e98 100644 --- a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go +++ b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go @@ -55,7 +55,9 @@ const ( resourceGroup = "kind-radius" // relativeChartPath points at the in-repo Helm chart so the test installs the build under test. - relativeChartPath = "../../../deploy/Chart" + // The path is relative to this test package directory + // (test/functional-portable/statestore/noncloud), which is four levels below the repo root. + relativeChartPath = "../../../../deploy/Chart" // redisRecipeTemplate is the Terraform recipe fixture shared with the corerp recipe tests. redisRecipeTemplate = "../../corerp/noncloud/resources/testdata/corerp-resources-terraform-redis.bicep" From e2ae9cc5f0956430f21d351b58409c8b2079892c Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 22:44:50 -0700 Subject: [PATCH 5/6] fix(statestore): make control-plane readiness check workspace-independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waitForControlPlane called rp.NewRPTestOptions, which requires a configured rad workspace. It ran inside installRadius (before the test creates the workspace), so it panicked with 'default workspace is not set' every iteration and looped until the timeout. Poll the radius-system Deployments for the Available condition via the Kubernetes client instead — the real, workspace-independent readiness signal, matching the workflow's 'kubectl wait --for=condition=Available'. Signed-off-by: Sylvain Niles --- .../noncloud/statestore_lifecycle_test.go | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go index bd1d2a47e98..65b9d9e4e28 100644 --- a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go +++ b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go @@ -37,12 +37,13 @@ import ( "time" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/radius-project/radius/test" "github.com/radius-project/radius/test/functional-portable/corerp" "github.com/radius-project/radius/test/radcli" - "github.com/radius-project/radius/test/rp" "github.com/radius-project/radius/test/testutil" ) @@ -122,20 +123,36 @@ func uninstallRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { waitForCleanTeardown(t, ctx) } -// waitForControlPlane polls until the Radius control plane API is reachable, treating transient -// errors (including 503 from the aggregated APIService while pods roll) as retryable. -func waitForControlPlane(t *testing.T, _ context.Context) { +// waitForControlPlane polls until every Radius control-plane deployment in radius-system reports +// Available, treating transient API errors (including 503 from the aggregated APIService while +// pods roll) as retryable. It is deliberately workspace-independent: it talks to Kubernetes +// directly, so it can run immediately after install and before any rad workspace exists. +func waitForControlPlane(t *testing.T, ctx context.Context) { t.Helper() + k8s := test.NewTestOptions(t).K8sClient require.Eventually(t, func() bool { - ready := false - func() { - // NewRPTestOptions calls require/panic internally on a not-yet-ready control plane; - // catch that and treat it as "retry". - defer func() { _ = recover() }() - opts := rp.NewRPTestOptions(t) - ready = opts.ManagementClient != nil - }() - return ready + deployments, err := k8s.AppsV1().Deployments(stateNamespace).List(ctx, metav1.ListOptions{LabelSelector: radiusPodSelector}) + if err != nil { + t.Logf("waiting to list control-plane deployments: %v", err) + return false + } + if len(deployments.Items) == 0 { + return false + } + for _, d := range deployments.Items { + available := false + for _, c := range d.Status.Conditions { + if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue { + available = true + break + } + } + if !available { + t.Logf("waiting for deployment %s to become Available...", d.Name) + return false + } + } + return true }, controlPlaneTimeout, controlPlanePollInterval, "control plane did not become available within timeout") } From 15d7e4d35bb4e886d3a55593198de5b9be80ad3b Mon Sep 17 00:00:00 2001 From: Sylvain Niles Date: Wed, 24 Jun 2026 23:00:32 -0700 Subject: [PATCH 6/6] fix(statestore): non-interactive uninstall and remote-less state repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures after install/deploy succeeded: - 'rad uninstall' prompted for confirmation and failed opening /dev/tty in CI. Pass --yes for non-interactive teardown. - 'rad shutdown' tried to push the radius-state branch to the checkout's GitHub origin, which has no push credentials in CI. Run shutdown/startup from a dedicated throwaway git repo with no remote (via a second CLI with WorkingDirectory set) so gitstate commits state locally only — the design's supported local/test case. Both commands share the same repo so the state committed by shutdown survives into startup. Signed-off-by: Sylvain Niles --- .../noncloud/statestore_lifecycle_test.go | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go index 65b9d9e4e28..fa2b72adbfc 100644 --- a/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go +++ b/test/functional-portable/statestore/noncloud/statestore_lifecycle_test.go @@ -33,6 +33,7 @@ import ( "context" "fmt" "os" + "os/exec" "testing" "time" @@ -118,7 +119,7 @@ func installRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { // aggregated APIService to deregister so a subsequent install does not race the teardown. func uninstallRadius(ctx context.Context, t *testing.T, cli *radcli.CLI) { t.Helper() - out, err := cli.RunCommand(ctx, []string{"uninstall", "kubernetes", "--purge"}) + out, err := cli.RunCommand(ctx, []string{"uninstall", "kubernetes", "--purge", "--yes"}) require.NoErrorf(t, err, "rad uninstall failed: %s", out) waitForCleanTeardown(t, ctx) } @@ -193,6 +194,28 @@ func waitForCleanTeardown(t *testing.T, ctx context.Context) { }, apiServiceDeregistrationTimeout, apiServiceDeregistrationInterval, "aggregated APIService did not deregister within timeout") } +// newStateRepo creates a throwaway git repository with no remote for `rad shutdown` / `rad startup` +// to persist state into. gitstate resolves the repo from the rad process's working directory and +// pushes to `origin` when a remote exists; running from this remote-less repo exercises the +// design's supported local/test case (commit-only, no push) instead of trying to push to the +// checkout's GitHub origin, which has no credentials in CI. +func newStateRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, args := range [][]string{ + {"init"}, + {"config", "user.email", "statestore-test@radapp.io"}, + {"config", "user.name", "statestore-test"}, + {"commit", "--allow-empty", "-m", "init"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "git %v failed: %s", args, out) + } + return dir +} + // Test_StateStore_ShutdownStartup_TerraformCrossDeploy exercises every state path: // install, deploy a Terraform resource, shut down (backup), tear down, start up (restore), then // deploy an update to the same resource. @@ -202,6 +225,12 @@ func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { cli := radcli.NewCLI(t, "") + // shutdown/startup persist state into a remote-less git repo so the backup commits locally + // without trying to push to the checkout's GitHub origin (no credentials in CI). Both commands + // must use the same repo so the state committed by shutdown survives into startup. + stateCLI := radcli.NewCLI(t, "") + stateCLI.WorkingDirectory = newStateRepo(t) + appName := "statestore-tf-redis-app" envName := "statestore-tf-redis-env" resourceName := "statestore-tf-redis" @@ -250,7 +279,7 @@ func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { require.True(t, secretExists(), "Terraform state secret should exist after the first deploy") // 3. Back up all durable state. - out, err = cli.RunCommand(ctx, []string{"shutdown"}) + out, err = stateCLI.RunCommand(ctx, []string{"shutdown"}) require.NoErrorf(t, err, "rad shutdown failed: %s", out) // 4. Tear the control plane down completely (ephemeral teardown). @@ -261,7 +290,7 @@ func Test_StateStore_ShutdownStartup_TerraformCrossDeploy(t *testing.T) { require.False(t, secretExists(), "Terraform state secret must be gone after reinstall (teardown was real)") // 6. Restore the saved state. - out, err = cli.RunCommand(ctx, []string{"startup"}) + out, err = stateCLI.RunCommand(ctx, []string{"startup"}) require.NoErrorf(t, err, "rad startup failed: %s", out) // 7. Both stores must be restored: the Terraform state Secret is back, and the control-plane