Test infra plan/apply with multiple projects

.github/workflows/code_review_infra.yaml

@@ -21,7 +21,8 @@ permissions:
 
 jobs:
   code_review_dev:
-    uses: pagopa/dx/.github/workflows/infra_plan.yaml@main
+    # uses: pagopa/dx/.github/workflows/infra_plan.yaml@main
+    uses: ./.github/workflows/infra_plan.yaml
     name: Code Review Infra Plan
     secrets: inherit
     with:

.github/workflows/infra_apply.yaml

@@ -0,0 +1,357 @@
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: Environment where the resources will be deployed.
+        type: string
+        required: true
+      base_path:
+        description: The base path on which the script will look for Terraform projects
+        type: string
+        required: true
+      env_vars:
+        description: List of environment variables to set up, given in env=value format.
+        type: string
+        required: false
+      use_private_agent:
+        description: Use a private agent to run the Terraform plan.
+        type: boolean
+        required: false
+        default: false
+      override_github_environment:
+        description: Set a value if GitHub Environment name is different than the TF environment folder
+        type: string
+        required: false
+        default: ""
+      use_labels:
+        description: Use labels to start the right environment's GitHub runner. If use_labels is true, also use_private_agent must be set to true
+        type: boolean
+        required: false
+        default: false
+      override_labels:
+        description: Needed for special cases where the environment alone is not sufficient as a distinguishing label
+        type: string
+        required: false
+        default: ""
+
+env:
+  ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+  ARM_USE_OIDC: true
+  ARM_USE_AZUREAD: true
+  ARM_STORAGE_USE_AZUREAD: true
+  ROLE_ARN: ${{ secrets.ROLE_ARN }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  TF_IN_AUTOMATION: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.environment }}-${{ inputs.base_path }}-cd
+  cancel-in-progress: false
+
+jobs:
+  get-terraform-version:
+    name: Get Terraform Version
+    runs-on: ubuntu-latest
+    outputs:
+      terraform_version: ${{ steps.get-version.outputs.terraform_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Get terraform version from .terraform-version file
+        id: get-version
+        uses: pagopa/dx/.github/actions/get-terraform-version@main
+        with:
+          default_version: "1.10.4"
+
+  detect-projects:
+    name: Detect Terraform Projects
+    runs-on: ubuntu-latest
+    needs: [get-terraform-version]
+    outputs:
+      matrix: ${{ steps.detect.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: Detect Terraform projects
+        id: detect
+        env:
+          BASE_PATH: ${{ inputs.base_path }}
+          ENVIRONMENT: ${{ inputs.environment }}
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
+          BEFORE: ${{ github.event.before }}
+          SHA: ${{ github.sha }}
+        run: |
+          set -euo pipefail
+          BASE_PATH="${BASE_PATH%/}"
+          DIR="$BASE_PATH/$ENVIRONMENT"
+          MODULES_DIR="$BASE_PATH/_modules"
+
+          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
+            CHANGED=$(find "$DIR" -maxdepth 2 -name '*.tf' -not -path "*/_modules/*" 2>/dev/null || true)
+          elif [ "$EVENT_NAME" = "pull_request" ]; then
+            CHANGED=$(git diff --name-only "origin/${BASE_REF}...HEAD" -- "$DIR" || true)
+            MODULES_CHANGED=$(git diff --name-only "origin/${BASE_REF}...HEAD" -- "$MODULES_DIR" || true)
+          else
+            CHANGED=$(git diff --name-only "$BEFORE" "$SHA" -- "$DIR" || true)
+            MODULES_CHANGED=$(git diff --name-only "$BEFORE" "$SHA" -- "$MODULES_DIR" || true)
+          fi
+
+          if [ "$EVENT_NAME" != "workflow_dispatch" ] && [ -n "${MODULES_CHANGED:-}" ]; then
+            CHANGED=$(find "$DIR" -maxdepth 2 -name '*.tf' -not -path "*/_modules/*" 2>/dev/null || true)
+          fi
+
+          # The two layouts are mutually exclusive:
+          # - flat:          a .tf exists directly in $DIR → $DIR is the single project root
+          # - multi-project: no .tf directly in $DIR → each first-level subdir is a project root
+          IS_FLAT=$(printf '%s\n' "$CHANGED" | while IFS= read -r file; do
+            rel="${file#"${DIR}/"}"
+            case "$rel" in
+              */*) ;;                    # file in a subdir: not a flat indicator
+              *.tf) echo 1; break ;;    # .tf directly in $DIR: flat project
+            esac
+          done)
+
+          if [ -n "$IS_FLAT" ]; then
+            MATRIX=$(jq -cn --arg d "$DIR" '[$d]')
+          else
+            MATRIX=$(printf '%s\n' "$CHANGED" \
+              | while IFS= read -r file; do
+                  [ -z "$file" ] && continue
+                  rel="${file#"${DIR}/"}"
+                  case "$rel" in */*)  echo "${DIR}/${rel%%/*}" ;; esac
+                done \
+              | sort -u \
+              | jq -R . | jq -sc .)
+          fi
+
+          MATRIX=${MATRIX:-[]}
+          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
+          echo "Detected matrix: $MATRIX"
+
+  tf-modules-check:
+    uses: pagopa/dx/.github/workflows/static_analysis.yaml@main
+    name: Check terraform registry modules hashes
+    needs: [get-terraform-version, detect-projects]
+    if: ${{ needs.detect-projects.outputs.matrix != '[]' }}
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        dir: ${{ fromJSON(needs.detect-projects.outputs.matrix) }}
+    with:
+      terraform_version: ${{ needs.get-terraform-version.outputs.terraform_version }}
+      check_to_run: lock_modules
+      folder: ${{ matrix.dir }}/
+      verbose: true
+
+  tf_plan:
+    name: "Terraform Plan"
+    strategy:
+      fail-fast: false
+      matrix:
+        dir: ${{ fromJSON(needs.detect-projects.outputs.matrix) }}
+    # Use inputs.override_labels if set; otherwise, fall back to inputs.environment.
+    # When inputs.use_labels and inputs.use_private_agent are true, apply the selected labels.
+    # Default to 'self-hosted' if inputs.use_private_agent is true, or 'ubuntu-latest' otherwise.
+    runs-on: ${{ inputs.use_labels && inputs.use_private_agent && (inputs.override_labels != '' && inputs.override_labels || inputs.environment) || inputs.use_private_agent && 'self-hosted' || 'ubuntu-latest' }}
+    needs: [get-terraform-version, tf-modules-check, detect-projects]
+    if: ${{ needs.detect-projects.outputs.matrix != '[]' }}
+    environment: ${{ inputs.override_github_environment == '' && inputs.environment || inputs.override_github_environment}}-ci
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    env:
+      ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+      ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+      TFPLAN_FILE: tf-outcome-${{ github.sha }}
+      TF_HTTP_USERNAME: ${{ secrets.STATEGRAPH_USERNAME }}
+      TF_HTTP_PASSWORD: ${{ secrets.STATEGRAPH_TOKEN }}
+    outputs:
+      terraform_version: ${{ needs.get-terraform-version.outputs.terraform_version }}
+
+    steps:
+      # The working directory comes from the matrix; available in ${{ steps.directory.outputs.dir }}
+      - name: Set directory
+        id: directory
+        env:
+          WORKING_DIR: ${{ matrix.dir }}
+        run: |
+          set -euo pipefail
+          printf "dir=%q" "$WORKING_DIR" >> "$GITHUB_OUTPUT"
+
+      - name: Set artifact name
+        id: artifact
+        env:
+          WORKING_DIR: ${{ matrix.dir }}
+        run: |
+          set -euo pipefail
+          SLUG=$(echo "$WORKING_DIR" | tr '/' '-')
+          echo "name=terraform-bundle-${SLUG}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        name: Checkout
+
+      - name: Set Environment Variables
+        if: ${{ inputs.env_vars }}
+        env:
+          ENV_VARS: ${{ inputs.env_vars }}
+        run: |
+          set -euo pipefail
+          for i in "$ENV_VARS[@]"
+          do
+            printf "%q\n" "$i" >> "$GITHUB_ENV"
+          done
+
+      - name: Cloud Login
+        uses: pagopa/dx/actions/csp-login@main
+
+      - name: Terraform Setup
+        uses: pagopa/dx/.github/actions/terraform-setup@main
+        id: set-terraform-version
+        with:
+          terraform_version: ${{ needs.get-terraform-version.outputs.terraform_version }}
+
+      - name: Terraform Init
+        working-directory: ${{ steps.directory.outputs.dir }}
+        run: |
+          terraform init -input=false
+
+      - name: Terraform Plan
+        working-directory: ${{ steps.directory.outputs.dir }}
+        run: |
+          set -o pipefail
+          terraform plan \
+            -lock-timeout=120s \
+            -out="$TFPLAN_FILE" \
+            -input=false \
+            | grep -v "hidden-link:"
+      # Bundle the plan file together with .terraform.lock.hcl and .terraform/modules/
+      # and upload it to the same storage backend used for the Terraform state.
+      # The bundle path is derived automatically from the backend config.
+      # Outputs (provider, plan-path, credentials) are passed to the apply job.
+      - name: Upload Terraform Plan
+        id: upload_plan
+        uses: pagopa/dx/actions/terraform-plan-storage-upload@main
+        with:
+          plan-file: ${{ env.TFPLAN_FILE }}
+          working-directory: ${{ steps.directory.outputs.dir }}
+
+      - name: Save plan outputs to artifact
+        env:
+          PROVIDER: ${{ steps.upload_plan.outputs.provider }}
+          PLAN_PATH: ${{ steps.upload_plan.outputs['plan-path'] }}
+          AZURE_STORAGE_ACCOUNT: ${{ steps.upload_plan.outputs['azure-storage-account'] }}
+          AZURE_CONTAINER: ${{ steps.upload_plan.outputs['azure-container'] }}
+          AWS_BUCKET: ${{ steps.upload_plan.outputs['aws-bucket'] }}
+          AWS_REGION: ${{ steps.upload_plan.outputs['aws-region'] }}
+        run: |
+          set -euo pipefail
+          jq -n \
+            --arg provider "$PROVIDER" \
+            --arg plan_path "$PLAN_PATH" \
+            --arg azure_storage_account "$AZURE_STORAGE_ACCOUNT" \
+            --arg azure_container "$AZURE_CONTAINER" \
+            --arg aws_bucket "$AWS_BUCKET" \
+            --arg aws_region "$AWS_REGION" \
+            '{provider: $provider, plan_path: $plan_path, azure_storage_account: $azure_storage_account, azure_container: $azure_container, aws_bucket: $aws_bucket, aws_region: $aws_region}' \
+            > plan-outputs.json
+
+      - name: Upload plan outputs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.artifact.outputs.name }}-outputs
+          path: plan-outputs.json
+
+  tf_apply:
+    name: "Terraform Apply"
+    strategy:
+      fail-fast: false
+      matrix:
+        dir: ${{ fromJSON(needs.detect-projects.outputs.matrix) }}
+    runs-on: ${{ inputs.use_labels && inputs.use_private_agent && (inputs.override_labels != '' && inputs.override_labels || inputs.environment) || inputs.use_private_agent && 'self-hosted' || 'ubuntu-latest' }}
+    needs: [get-terraform-version, tf_plan, detect-projects]
+    if: ${{ needs.detect-projects.outputs.matrix != '[]' }}
+    environment: ${{ inputs.override_github_environment == '' && inputs.environment || inputs.override_github_environment}}-cd
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+      ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      TFPLAN_FILE: tf-outcome-${{ github.sha }}
+      TF_HTTP_USERNAME: ${{ secrets.STATEGRAPH_USERNAME }}
+      TF_HTTP_PASSWORD: ${{ secrets.STATEGRAPH_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        name: Checkout
+
+      - name: Set artifact name
+        id: artifact
+        env:
+          WORKING_DIR: ${{ matrix.dir }}
+        run: |
+          set -euo pipefail
+          SLUG=$(echo "$WORKING_DIR" | tr '/' '-')
+          echo "name=terraform-bundle-${SLUG}" >> "$GITHUB_OUTPUT"
+
+      - name: Cloud Login
+        uses: pagopa/dx/actions/csp-login@main
+
+      - name: Terraform Setup
+        uses: pagopa/dx/.github/actions/terraform-setup@main
+        with:
+          terraform_version: ${{ needs.get-terraform-version.outputs.terraform_version }}
+
+      # Download plan outputs artifact from tf_plan
+      - name: Download plan outputs artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.artifact.outputs.name }}-outputs
+          path: /tmp/plan-outputs
+
+      - name: Read plan outputs
+        id: plan_outputs
+        run: |
+          set -euo pipefail
+          FILE=/tmp/plan-outputs/plan-outputs.json
+          echo "provider=$(jq -r .provider "$FILE")" >> "$GITHUB_OUTPUT"
+          echo "plan_path=$(jq -r .plan_path "$FILE")" >> "$GITHUB_OUTPUT"
+          echo "azure_storage_account=$(jq -r .azure_storage_account "$FILE")" >> "$GITHUB_OUTPUT"
+          echo "azure_container=$(jq -r .azure_container "$FILE")" >> "$GITHUB_OUTPUT"
+          echo "aws_bucket=$(jq -r .aws_bucket "$FILE")" >> "$GITHUB_OUTPUT"
+          echo "aws_region=$(jq -r .aws_region "$FILE")" >> "$GITHUB_OUTPUT"
+
+      # Download and extract the plan bundle (plan file + .terraform.lock.hcl + .terraform/modules/).
+      # Storage coordinates come from the plan outputs artifact.
+      - name: Download Terraform Plan
+        uses: pagopa/dx/actions/terraform-plan-storage-download@main
+        with:
+          provider: ${{ steps.plan_outputs.outputs.provider }}
+          plan-path: ${{ steps.plan_outputs.outputs.plan_path }}
+          working-directory: ${{ matrix.dir }}
+          azure-storage-account: ${{ steps.plan_outputs.outputs.azure_storage_account }}
+          azure-container: ${{ steps.plan_outputs.outputs.azure_container }}
+          aws-bucket: ${{ steps.plan_outputs.outputs.aws_bucket }}
+          aws-region: ${{ steps.plan_outputs.outputs.aws_region }}
+
+      - name: Terraform Init
+        working-directory: ${{ matrix.dir }}
+        run: |
+          terraform init -input=false
+
+      - name: Terraform Apply
+        working-directory: ${{ matrix.dir }}
+        run: |
+          set -o pipefail
+          terraform apply \
+            -lock-timeout=120s \
+            -auto-approve \
+            -input=false \
+            "$TFPLAN_FILE" \
+            | grep -v "hidden-link:"