VED-1223: Update permissions to auto-ops role so the pipeline can apply terraform changes at account level

.github/workflows/account-terraform.yml

@@ -0,0 +1,183 @@
+name: Account Terraform
+
+on:
+  workflow_call:
+    inputs:
+      base_sha:
+        required: true
+        type: string
+      head_sha:
+        required: true
+        type: string
+      environment:
+        required: true
+        type: string
+      artifact_name:
+        required: true
+        type: string
+
+concurrency:
+  group: account-terraform-${{ github.repository }}-${{ inputs.environment }}
+  cancel-in-progress: false
+
+jobs:
+  detect-account-infra-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      account_infra_changed: ${{ steps.diff.outputs.account_infra_changed }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          fetch-depth: 0
+
+      - name: Detect account terraform changes
+        id: diff
+        run: |
+          base_sha="${{ inputs.base_sha }}"
+          if [ -z "$base_sha" ] || [ "$base_sha" = "0000000000000000000000000000000000000000" ]; then
+            base_sha=$(git rev-parse HEAD~1)
+          fi
+
+          changed_files=$(git diff --name-only "$base_sha" "${{ inputs.head_sha }}")
+          if echo "$changed_files" | grep -q '^infrastructure/account/'; then
+            echo "account_infra_changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "account_infra_changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  account-terraform-plan:
+    permissions:
+      id-token: write
+      contents: read
+    needs: [detect-account-infra-changes]
+    if: ${{ needs.detect-account-infra-changes.outputs.account_infra_changed == 'true' }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85
+        with:
+          terraform_version: "1.12.2"
+
+      - name: Resolve account terraform state bucket
+        id: account-state-bucket
+        env:
+          CONFIGURED_ACCOUNT_TERRAFORM_STATE_BUCKET: ${{ vars.ACCOUNT_TERRAFORM_STATE_BUCKET }}
+        run: |
+          bucket_name="$(bash ./utilities/scripts/resolve_account_terraform_state_bucket.sh)"
+          if [ -z "$bucket_name" ]; then
+            echo "Resolved empty account terraform state bucket." >&2
+            exit 1
+          fi
+          echo "bucket_name=$bucket_name" >> "$GITHUB_OUTPUT"
+
+      - name: Terraform Init (account)
+        working-directory: infrastructure/account
+        run: make init ENVIRONMENT=${{ inputs.environment }} BUCKET_NAME=${{ steps.account-state-bucket.outputs.bucket_name }}
+
+      - name: Terraform Plan (account)
+        # Ignore cancellations to prevent Terraform from being killed while it holds a state lock
+        # A stuck process can still be killed with the force-cancel API operation
+        if: ${{ !failure() }}
+        working-directory: infrastructure/account
+        run: make plan-ci ENVIRONMENT=${{ inputs.environment }}
+
+      - name: Save Account Terraform Plan
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: infrastructure/account/tfplan
+
+  account-terraform-manual-approval:
+    needs: [account-terraform-plan]
+    if: ${{ !cancelled() && needs.account-terraform-plan.result == 'success' }}
+    runs-on: ubuntu-latest
+    environment:
+      name: account-level-infra-approval
+    steps:
+      - name: Await manual approval
+        run: echo "Waiting for account-level infrastructure approval."
+
+  account-terraform-apply:
+    permissions:
+      id-token: write
+      contents: read
+    needs: [account-terraform-manual-approval]
+    if: ${{ !cancelled() && needs.account-terraform-manual-approval.result == 'success' }}
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85
+        with:
+          terraform_version: "1.12.2"
+
+      - name: Resolve account terraform state bucket
+        id: account-state-bucket
+        env:
+          CONFIGURED_ACCOUNT_TERRAFORM_STATE_BUCKET: ${{ vars.ACCOUNT_TERRAFORM_STATE_BUCKET }}
+        run: |
+          bucket_name="$(bash ./utilities/scripts/resolve_account_terraform_state_bucket.sh)"
+          if [ -z "$bucket_name" ]; then
+            echo "Resolved empty account terraform state bucket." >&2
+            exit 1
+          fi
+          echo "bucket_name=$bucket_name" >> "$GITHUB_OUTPUT"
+
+      - name: Retrieve Account Terraform Plan
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: infrastructure/account
+
+      - name: Terraform Init (account)
+        working-directory: infrastructure/account
+        run: make init ENVIRONMENT=${{ inputs.environment }} BUCKET_NAME=${{ steps.account-state-bucket.outputs.bucket_name }}
+
+      - name: Terraform Apply (account)
+        # Ignore cancellations to prevent Terraform from being killed while it holds a state lock
+        # A stuck process can still be killed with the force-cancel API operation
+        if: ${{ !failure() }}
+        working-directory: infrastructure/account
+        run: make apply-ci ENVIRONMENT=${{ inputs.environment }}
+
+  account-terraform-not-required:
+    needs: [detect-account-infra-changes]
+    if: ${{ needs.detect-account-infra-changes.outputs.account_infra_changed != 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skip account terraform
+        run: echo "No account-level infrastructure changes detected."
+
+  account-terraform-ready:
+    needs:
+      - account-terraform-plan
+      - account-terraform-manual-approval
+      - account-terraform-apply
+      - account-terraform-not-required
+    if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Account terraform stage complete
+        run: echo "Account terraform stage complete."

.github/workflows/continuous-deployment.yml

@@ -15,8 +15,16 @@ jobs:
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
+  account-terraform:
+    uses: ./.github/workflows/account-terraform.yml
+    with:
+      base_sha: ${{ github.event.before }}
+      head_sha: ${{ github.sha }}
+      environment: dev
+      artifact_name: dev-account-tfplan
+
   deploy-internal-dev-backend:
-    needs: [run-quality-checks]
+    needs: [run-quality-checks, account-terraform]
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev

.github/workflows/pr-deploy-and-test.yml

@@ -14,14 +14,22 @@ jobs:
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
+  account-terraform:
+    uses: ./.github/workflows/account-terraform.yml
+    with:
+      base_sha: ${{ github.event.pull_request.base.sha }}
+      head_sha: ${{ github.event.pull_request.head.sha }}
+      environment: dev
+      artifact_name: pr-dev-account-tfplan
+
   deploy-pr-backend:
-    needs: [run-quality-checks]
+    needs: [run-quality-checks, account-terraform]
     if: ${{ always() && !failure() && !cancelled() }}
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
       build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' }}
-      diff_base_sha: ${{ github.event.before }}
+      diff_base_sha: ${{ github.event.pull_request.base.sha }}
       diff_head_sha: ${{ github.event.pull_request.head.sha }}
       run_diff_check: ${{ github.event.action == 'synchronize' }}
       create_mns_subscription: true

infrastructure/account/Makefile

@@ -6,6 +6,13 @@ tf_cmd = AWS_PROFILE=$(AWS_PROFILE) terraform
 tf_state= -backend-config="bucket=$(BUCKET_NAME)"
 tf_vars= -var-file=environments/$(ENVIRONMENT)/variables.tfvars
 
+define require_bucket_name
+	@if [ -z "$(strip $(BUCKET_NAME))" ]; then \
+		echo "BUCKET_NAME variable not set. Use 'make init ENVIRONMENT=... BUCKET_NAME=...'"; \
+		exit 1; \
+	fi
+endef
+
 .PHONY: lock-provider workspace init plan apply clean destroy output tf-%
 
 lock-provider:
@@ -17,17 +24,25 @@ workspace:
 	$(tf_cmd) workspace select -or-create $(ENVIRONMENT) && echo "Switched to workspace/environment: $(ENVIRONMENT)"
 
 init:
+	$(require_bucket_name)
 	$(tf_cmd) init $(tf_state) -upgrade $(tf_vars)
 
 init-reconfigure:
+	$(require_bucket_name)
 	$(tf_cmd) init $(tf_state) -upgrade $(tf_vars) -reconfigure
 
 plan: workspace
 	 $(tf_cmd) plan $(tf_vars)
 
+plan-ci: workspace
+	$(tf_cmd) plan $(tf_vars) -out=tfplan -input=false
+
 apply: workspace
 	$(tf_cmd) apply $(tf_vars) -auto-approve
 
+apply-ci: workspace
+	$(tf_cmd) apply $(tf_vars) -input=false tfplan
+
 clean:
 	rm -rf build .terraform upload-key
 

infrastructure/account/auto_ops_policy.json

@@ -214,6 +214,7 @@
         "iam:DeleteGroupPolicy",
         "iam:ListMFADeviceTags",
         "elasticache:*",
+        "shield:*",
         "iam:DeletePolicyVersion",
         "chatbot:*"
       ],

utilities/scripts/resolve_account_terraform_state_bucket.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -o nounset errexit pipefail
+
+configured_bucket="$(printf '%s' "${CONFIGURED_ACCOUNT_TERRAFORM_STATE_BUCKET:-}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+
+if [ -n "$configured_bucket" ]; then
+  printf '%s\n' "$configured_bucket"
+  exit 0
+fi
+
+mapfile -t buckets < <(
+  aws s3api list-buckets --query 'Buckets[].Name' --output text |
+  tr '\t' '\n' |
+  grep -E '^immunisation-dev-terraform-state(-files)?$'
+)
+
+if [ "${#buckets[@]}" -ne 1 ]; then
+  echo "Expected exactly 1 dev account terraform state bucket, found ${#buckets[@]}." >&2
+  echo "Set repo/environment variable ACCOUNT_TERRAFORM_STATE_BUCKET to remove ambiguity." >&2
+  printf '%s\n' "${buckets[@]}" >&2
+  exit 1
+fi
+
+printf '%s\n' "${buckets[0]}"