Fix: [AEA-0000] - new stack for monitoring

.github/scripts/describe_cdk_changes.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -e
+
+{
+  echo "<details><summary>$SUMMARY</summary>"
+  echo ""
+  echo "- **Stack Name:** $STACK_NAME"
+  echo ""
+  echo "#### Parameters:"
+  echo "~~~ json"
+  cat "$PARAMETERS_FILE"
+  echo "~~~"
+  echo "#### Changes:"
+  echo "~~~ text"
+  cat "$DIFF_FILE"
+  echo "~~~"
+  echo "</details>"
+} >> "$GITHUB_STEP_SUMMARY"

.github/scripts/fix_cdk_json.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -e
+
+# script used to set context key values in cdk.json pre deployment from environment variables
+
+# helper function to set string values
+fix_string_key() {
+    KEY_NAME=$1
+    KEY_VALUE=$2
+    if [ -z "${KEY_VALUE}" ]; then
+        echo "${KEY_NAME} value is unset or set to the empty string"
+        exit 1
+    fi
+    echo "Setting ${KEY_NAME}"
+    jq \
+        --arg key_value "${KEY_VALUE}" \
+        --arg key_name "${KEY_NAME}" \
+        '. += {($key_name): $key_value}' "$OUTPUT_FILE_NAME" > "${TEMP_FILE}"
+    mv "${TEMP_FILE}" "$OUTPUT_FILE_NAME"
+}
+
+fix_list_key() {
+    KEY_NAME=$1
+    KEY_VALUE=$2
+    if [ -z "${KEY_VALUE}" ]; then
+        echo "${KEY_NAME} value is unset or set to empty list"
+        exit 1
+    fi
+    echo "Setting ${KEY_NAME}"
+    jq \
+        --argjson key_value "${KEY_VALUE}" \
+        --arg key_name "${KEY_NAME}" \
+        '. += {($key_name): $key_value}' "$OUTPUT_FILE_NAME" > "${TEMP_FILE}"
+    mv "${TEMP_FILE}" "$OUTPUT_FILE_NAME"
+}
+
+# helper function to set boolean and number values (without quotes)
+fix_boolean_number_key() {
+    KEY_NAME=$1
+    KEY_VALUE=$2
+    # remove any surrounding quotes and convert to lowercase
+    KEY_VALUE=$(sed -e 's/^"//' -e 's/"$//' <<<"$KEY_VALUE")
+    KEY_VALUE=${KEY_VALUE,,}
+    if [ -z "${KEY_VALUE}" ]; then
+        echo "${KEY_NAME} value is unset or set to the empty string"
+        exit 1
+    fi
+    echo "Setting ${KEY_NAME} to ${KEY_VALUE}"
+    jq \
+        --argjson key_value "${KEY_VALUE}" \
+        --arg key_name "${KEY_NAME}" \
+        '. += {($key_name): $key_value}' "$OUTPUT_FILE_NAME" > "${TEMP_FILE}"
+    mv "${TEMP_FILE}" "$OUTPUT_FILE_NAME"
+}
+
+OUTPUT_FILE_NAME=$1
+if [ -z "${OUTPUT_FILE_NAME}" ]; then
+    echo "OUTPUT_FILE_NAME value is unset or set to the empty string"
+    exit 1
+fi
+echo "{}" > "$OUTPUT_FILE_NAME"
+TEMP_FILE=$(mktemp)
+
+
+CFN_DRIFT_DETECTION_GROUP="account-resources"
+if [[ "$IS_PULL_REQUEST" = "true" ]]; then
+  CFN_DRIFT_DETECTION_GROUP="account-resources-pull-request"
+fi
+
+# go through all the key values we need to set
+fix_string_key versionNumber "${VERSION_NUMBER}"
+fix_string_key commitId "${COMMIT_ID}"
+fix_string_key cfnDriftDetectionGroup "${CFN_DRIFT_DETECTION_GROUP}"
+fix_string_key stackName "${STACK_NAME}"
+fix_boolean_number_key isPullRequest "${IS_PULL_REQUEST}"
+fix_boolean_number_key lambdaConcurrencyThreshold "${LAMBDA_CONCURRENCY_THRESHOLD}"
+fix_boolean_number_key lambdaConcurrencyWarningThreshold "${LAMBDA_CONCURRENCY_WARNING_THRESHOLD}"
+fix_boolean_number_key enableAlerts "${ENABLE_ALERTS}"

.github/workflows/cdk_package_code.yml

@@ -0,0 +1,65 @@
+name: cdk package code
+
+on:
+  workflow_call:
+
+
+jobs:
+  package_cdk_code:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ env.BRANCH_NAME }}
+
+      # using git commit sha for version of action to ensure we have stable version
+      - name: Install asdf
+        uses: asdf-vm/actions/setup@1902764435ca0dd2f3388eea723a4f92a4eb8302
+        with:
+          asdf_branch: v0.14.1
+
+      - name: Cache asdf
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.asdf
+          key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }}
+          restore-keys: |
+            ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }}
+
+      - name: Install asdf dependencies in .tool-versions
+        uses: asdf-vm/actions/install@1902764435ca0dd2f3388eea723a4f92a4eb8302
+        with:
+          asdf_branch: v0.14.1
+        env:
+          PYTHON_CONFIGURE_OPTS: --enable-shared
+
+      - name: make install
+        run: |
+          make install
+
+      - name: 'Tar files'
+        run: |
+          tar -rf artifact.tar \
+            .tool-versions \
+            packages \
+            node_modules \
+            package.json \
+            package-lock.json \
+            tsconfig.defaults.json \
+            Makefile \
+            cdk.json \
+            cdk.context.json \
+            scripts \
+            environmentSettings
+
+      - uses: actions/upload-artifact@v4
+        name: upload build artifact
+        with:
+          name: cdk_build_artifact
+          path: artifact.tar

.github/workflows/cdk_release_code.yml

@@ -0,0 +1,203 @@
+name: cdk release code
+
+on:
+  workflow_call:
+    inputs:
+      STACK_NAME:
+        required: true
+        type: string
+      TARGET_ENVIRONMENT:
+        required: true
+        type: string
+      VERSION:
+        required: true
+        type: string
+      COMMIT_ID:
+        required: true
+        type: string
+      CDK_APP_NAME:
+        required: true
+        type: string
+      DEPLOY_CDK_CODE:
+        required: true
+        type: boolean
+      IS_PULL_REQUEST:
+        required: true
+        type: boolean
+      DEPLOYMENT_ENVIRONMENT:
+        required: true
+        type: string
+    secrets:
+      CLOUD_FORMATION_DEPLOY_ROLE:
+        required: true
+      CDK_PULL_IMAGE_ROLE:
+        required: true
+
+jobs:
+  deploy_cdk_code:
+    runs-on: ubuntu-22.04
+    name: "${{ inputs.DEPLOY_CDK_CODE && 'Deploy' || 'Diff' }} cdk app ${{ inputs.CDK_APP_NAME }}"
+    environment: ${{ inputs.DEPLOYMENT_ENVIRONMENT }}
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout local github actions
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ env.BRANCH_NAME }}
+          fetch-depth: 0
+          sparse-checkout: |
+            .github
+
+      - name: Configure AWS Credentials for pulling cdk deploy image
+        id: connect-aws-pull-image
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          aws-region: eu-west-2
+          role-to-assume: ${{ secrets.CDK_PULL_IMAGE_ROLE }}
+          role-session-name: account-resources-pull-image
+
+      - name: build_artifact download
+        uses: actions/download-artifact@v5
+        with:
+          name: cdk_build_artifact
+
+      - name: extract build_artifact
+        run: |
+          mkdir -p .build
+          tar -xf artifact.tar -C .build
+
+      - name: Retrieve AWS Account ID
+        id: retrieve-account-id
+        run: echo "ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_ENV"
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        run: |
+          aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ env.ACCOUNT_ID }}.dkr.ecr.eu-west-2.amazonaws.com
+
+      - name: Pull cdk-utils-build from Amazon ECR
+        run: |
+          docker pull "${{ env.ACCOUNT_ID }}.dkr.ecr.eu-west-2.amazonaws.com/cdk-utils-build-repo:latest"
+          docker tag "${{ env.ACCOUNT_ID }}.dkr.ecr.eu-west-2.amazonaws.com/cdk-utils-build-repo:latest" cdk-utils-build-repo:latest
+      - name: Configure AWS Credentials for diff or deploy
+        id: connect-aws-deploy
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          aws-region: eu-west-2
+          role-to-assume: ${{ secrets.CLOUD_FORMATION_DEPLOY_ROLE }}
+          role-session-name: account-resources-cdk-diff
+          output-credentials: true
+
+      - name: Parse Parameters
+        id: parse_parameters
+        shell: bash
+        working-directory: ./.build
+        env:
+          output_format: "env_vars"
+        run: |
+          PARAMETERS=$(python scripts/parse_parameters.py ${{ inputs.TARGET_ENVIRONMENT }} ${{ inputs.STACK_NAME }} )
+          echo "${PARAMETERS}" >> "$GITHUB_ENV"
+
+      - name: fix cdk.json for diff - existing tag
+        if: ${{ inputs.DEPLOY_CDK_CODE == false }}
+        run: |
+          VERSION_NUMBER=$(aws cloudformation describe-stacks --stack-name  "$STACK_NAME" --query "Stacks[0].Tags[?Key=='version'].Value" --output text)
+          COMMIT_ID=$(aws cloudformation describe-stacks --stack-name  "$STACK_NAME" --query "Stacks[0].Tags[?Key=='commit'].Value" --output text)
+          if [ "${VERSION_NUMBER}" == "" ]; then
+            echo "Can not find target tag. Using initial tag in repo"
+            export VERSION_NUMBER="v1.0.4-alpha"
+          fi
+          VERSION_NUMBER="${VERSION_NUMBER}" \
+            COMMIT_ID="${COMMIT_ID}" \
+            ./.github/scripts/fix_cdk_json.sh ".build/${{ inputs.STACK_NAME }}_existing_tag.json"
+        env:
+          STACK_NAME: "${{ inputs.STACK_NAME }}"
+          COMMIT_ID: "${{ inputs.COMMIT_ID }}"
+          IS_PULL_REQUEST: "${{ inputs.IS_PULL_REQUEST }}"
+
+      - name: "Run ${{ inputs.DEPLOY_CDK_CODE && 'deploy' || 'diff' }} for cdk app ${{ inputs.CDK_APP_NAME }} existing tag"
+        if: ${{ inputs.DEPLOY_CDK_CODE == false }}
+        run: |
+          docker run \
+          -v "$(pwd)/.build":/home/cdkuser/workspace/ \
+          -e AWS_ACCESS_KEY_ID=${{ steps.connect-aws-deploy.outputs.aws-access-key-id }} \
+          -e AWS_SECRET_ACCESS_KEY=${{ steps.connect-aws-deploy.outputs.aws-secret-access-key }} \
+          -e AWS_SESSION_TOKEN=${{ steps.connect-aws-deploy.outputs.aws-session-token }} \
+          -e AWS_REGION="eu-west-2" \
+          -e SHOW_DIFF="true" \
+          -e DEPLOY_CODE="${{ inputs.DEPLOY_CDK_CODE }}" \
+          -e CONFIG_FILE_NAME="${{ inputs.STACK_NAME }}_existing_tag.json" \
+          -e CDK_APP_PATH="packages/cdk/bin/${{ inputs.CDK_APP_NAME }}.ts" \
+          cdk-utils-build-repo:latest 2>&1 | tee .build/cdk_deploy_diff_existing_tag.log
+        shell: bash
+
+      - name: fix cdk.json for deployment
+        run: |
+          ./.github/scripts/fix_cdk_json.sh ".build/${{ inputs.STACK_NAME }}.json"
+        env:
+          STACK_NAME: "${{ inputs.STACK_NAME }}"
+          VERSION_NUMBER: "${{ inputs.VERSION }}"
+          COMMIT_ID: "${{ inputs.COMMIT_ID }}"
+          IS_PULL_REQUEST: "${{ inputs.IS_PULL_REQUEST }}"
+
+      - name: "Run ${{ inputs.DEPLOY_CDK_CODE && 'deploy' || 'diff' }} for cdk app ${{ inputs.CDK_APP_NAME }}"
+        run: |
+          docker run \
+          -v "$(pwd)/.build":/home/cdkuser/workspace/ \
+          -e AWS_ACCESS_KEY_ID=${{ steps.connect-aws-deploy.outputs.aws-access-key-id }} \
+          -e AWS_SECRET_ACCESS_KEY=${{ steps.connect-aws-deploy.outputs.aws-secret-access-key }} \
+          -e AWS_SESSION_TOKEN=${{ steps.connect-aws-deploy.outputs.aws-session-token }} \
+          -e AWS_REGION="eu-west-2" \
+          -e SHOW_DIFF="true" \
+          -e DEPLOY_CODE="${{ inputs.DEPLOY_CDK_CODE }}" \
+          -e CONFIG_FILE_NAME="${{ inputs.STACK_NAME }}.json" \
+          -e CDK_APP_PATH="packages/cdk/bin/${{ inputs.CDK_APP_NAME }}.ts" \
+          cdk-utils-build-repo:latest 2>&1 | tee .build/cdk_deploy_diff_new_tag.log
+        shell: bash
+
+      - name: Header output
+        if: ${{ inputs.DEPLOY_CDK_CODE == false }}
+        shell: bash
+        working-directory: .github/scripts
+        run: ./header_output.sh
+        env:
+          TARGET_ENVIRONMENT: ${{ inputs.TARGET_ENVIRONMENT }}
+          STACK_NAME: ${{ inputs.STACK_NAME }}
+          VERSION: ${{ inputs.VERSION }}
+      - name: Describe existing tag CDK Changes
+        if: ${{ inputs.DEPLOY_CDK_CODE == false }}
+        shell: bash
+        env:
+          SUMMARY: "[${{ inputs.target_environment }}] CDK Changes for existing tag ${{ inputs.stack_name }}"
+          STACK_NAME: ${{ inputs.STACK_NAME }}
+          PARAMETERS_FILE: ".build/${{ inputs.STACK_NAME }}_existing_tag.json"
+          DIFF_FILE: .build/cdk_deploy_diff_existing_tag.log
+        run: ./.github/scripts/describe_cdk_changes.sh
+
+      - name: Describe Full CDK Changes
+        if: ${{ inputs.DEPLOY_CDK_CODE == false }}
+        shell: bash
+        env:
+          SUMMARY: "[${{ inputs.target_environment }}] FULL CDK Changes for ${{ inputs.stack_name }}"
+          STACK_NAME: ${{ inputs.STACK_NAME }}
+          PARAMETERS_FILE: ".build/${{ inputs.STACK_NAME }}.json"
+          DIFF_FILE: .build/cdk_deploy_diff_new_tag.log
+        run: ./.github/scripts/describe_cdk_changes.sh
+
+      - name: Update release tag in github pages
+        if: ${{ inputs.DEPLOY_CDK_CODE == true }}
+        run: |
+          cd gh-pages
+          NOW=$(date +'%Y-%m-%dT%H:%M:%S')
+          echo "tag,release_datetime" > _data/${{ inputs.TARGET_ENVIRONMENT }}_${{ inputs.STACK_NAME }}_latest.csv
+          echo "${{ inputs.VERSION }},${NOW}" >> _data/${{ inputs.TARGET_ENVIRONMENT }}_${{ inputs.STACK_NAME }}_latest.csv
+          echo "${{ inputs.VERSION }},${NOW}" >> _data/${{ inputs.TARGET_ENVIRONMENT }}_${{ inputs.STACK_NAME }}_deployments.csv
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git add _data/${{ inputs.TARGET_ENVIRONMENT }}_${{ inputs.STACK_NAME }}_latest.csv
+          git add _data/${{ inputs.TARGET_ENVIRONMENT }}_${{ inputs.STACK_NAME }}_deployments.csv
+          git commit -m 'update releases for ${{ inputs.TARGET_ENVIRONMENT }} and stack ${{ inputs.STACK_NAME }}'
+          parallel --retries 10 --delay 3 ::: "git pull --rebase && git push"