Deploy next to performance #3503
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy | |
| run-name: Deploy ${{ inputs.git_ref_to_deploy || github.ref_name }} to ${{ inputs.environment }} | |
| on: | |
| workflow_call: | |
| inputs: | |
| environment: | |
| required: true | |
| type: string | |
| server_types: | |
| required: true | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| git_ref_to_deploy: | |
| description: | |
| | # Use blank unicode character (U+2800) to force line-break | |
| Use code from: ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ | |
| (Git ref to deploy, for example, a tag, branch name or commit SHA. Will use workflow ref if not provided.) | |
| type: string | |
| environment: | |
| description: Deployment environment | |
| required: true | |
| type: choice | |
| options: | |
| - qa | |
| - test | |
| - preview | |
| - training | |
| - production | |
| - sandbox-alpha | |
| - sandbox-beta | |
| - performance | |
| - pentest | |
| server_types: | |
| description: Server types to deploy | |
| required: true | |
| type: choice | |
| options: | |
| - all | |
| - web | |
| - sidekiq | |
| - ops | |
| default: all | |
| concurrency: | |
| group: deploy-${{ inputs.environment }} | |
| permissions: {} | |
| env: | |
| git_ref_to_deploy: ${{ inputs.git_ref_to_deploy }} | |
| environment: ${{ inputs.environment }} | |
| server_types: ${{ inputs.server_types }} | |
| aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }} | |
| cluster_name: mavis-${{ inputs.environment }} | |
| app_version: ${{ inputs.git_ref_to_deploy == '' && github.ref_name || inputs.git_ref_to_deploy }} | |
| jobs: | |
| validate-inputs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Validate inputs | |
| run: | | |
| if [[ "$environment" == "preview" || "$environment" == "production" ]]; then | |
| if [[ -z "$git_ref_to_deploy" ]]; then | |
| echo "Error: git_ref_to_deploy is required for preview and production environments." | |
| exit 1 | |
| fi | |
| fi | |
| determine-git-sha: | |
| runs-on: ubuntu-latest | |
| needs: validate-inputs | |
| outputs: | |
| git-sha: ${{ steps.get-git-sha.outputs.git-sha }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.git_ref_to_deploy || github.sha }} | |
| - name: Get git sha | |
| id: get-git-sha | |
| run: echo "git-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | |
| build-and-push-images: | |
| permissions: | |
| id-token: write | |
| needs: determine-git-sha | |
| uses: ./.github/workflows/build-and-push-image.yml | |
| with: | |
| git_sha: ${{ needs.determine-git-sha.outputs.git-sha }} | |
| prepare-deployment: | |
| name: Prepare deployment | |
| runs-on: ubuntu-latest | |
| needs: [ determine-git-sha, build-and-push-images ] | |
| permissions: | |
| id-token: write | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| service: ${{ inputs.server_types == 'all' && fromJSON('["web", "sidekiq", "ops"]') || fromJSON(format('["{0}"]', inputs.server_types)) }} | |
| env: | |
| repository_name: ${{ matrix.service == 'ops' && 'mavis/ops' || 'mavis/webapp' }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| id: checkout-code | |
| with: | |
| ref: ${{ needs.determine-git-sha.outputs.git-sha }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ env.aws_account_id }}:role/GithubDeployECSService | |
| aws-region: eu-west-2 | |
| - name: Get image digest | |
| id: get-image-digest | |
| env: | |
| image_tag: ${{ needs.determine-git-sha.outputs.git-sha }} | |
| run: | | |
| digest=$(aws ecr describe-images \ | |
| --repository-name $repository_name \ | |
| --image-ids imageTag=$image_tag \ | |
| --query 'imageDetails[0].imageDigest' \ | |
| --output text) | |
| echo "digest=$digest" >> $GITHUB_OUTPUT | |
| - name: Populate task definition | |
| id: create-task-definition | |
| uses: aws-actions/amazon-ecs-render-task-definition@v1 | |
| with: | |
| task-definition-family: "mavis-${{ matrix.service }}-task-definition-${{ inputs.environment }}-template" | |
| container-name: "application" | |
| image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/${{ env.repository_name }}@${{ steps.get-image-digest.outputs.digest }}" | |
| environment-variables: | | |
| APP_VERSION=${{ env.app_version }} | |
| - name: Rename task definition file | |
| run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/${{ matrix.service }}-task-definition.json | |
| - name: Upload artifact for ${{ matrix.service }} task definition | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ inputs.environment }}-${{ matrix.service }}-task-definition | |
| path: ${{ runner.temp }}/${{ matrix.service }}-task-definition.json | |
| notify-approval-required: | |
| name: Notify approval required | |
| if: ${{ inputs.environment == 'production' }} | |
| runs-on: ubuntu-latest | |
| needs: prepare-deployment | |
| steps: | |
| - name: Notify approval required | |
| uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a | |
| with: | |
| webhook: ${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: ":hourglass: Approval required :hourglass:" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "${{ github.workflow }} requires approval" | |
| - type: "section" | |
| fields: | |
| - type: "mrkdwn" | |
| text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>" | |
| approve-deployments: | |
| name: Wait for approval if required | |
| runs-on: ubuntu-latest | |
| needs: prepare-deployment | |
| environment: ${{ inputs.environment }} | |
| steps: | |
| - run: echo "Proceeding with deployment to $environment" | |
| run-migrations: | |
| name: Run migrations | |
| runs-on: ubuntu-latest | |
| needs: [ approve-deployments ] | |
| if: ${{ inputs.server_types == 'all' || inputs.server_types == 'ops' }} | |
| permissions: | |
| id-token: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ env.aws_account_id }}:role/GithubDeployECSService | |
| aws-region: eu-west-2 | |
| - name: Download ops task definition artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: ${{ runner.temp }} | |
| name: ${{ inputs.environment }}-ops-task-definition | |
| - name: Register migration task definition | |
| id: register-migration-task-definition | |
| run: | | |
| family_name="mavis-migration-task-definition-$environment" | |
| file_path="${{ runner.temp }}/migration-task-definition.json" | |
| echo "$(jq --arg f "$family_name" '.family = $f' "${{ runner.temp }}/ops-task-definition.json")" > "$file_path" | |
| task_definition_arn=$(aws ecs register-task-definition \ | |
| --cli-input-json file://$file_path \ | |
| --query 'taskDefinition.taskDefinitionArn' \ | |
| --output text | |
| ) | |
| echo "task_definition_arn=$task_definition_arn" >> $GITHUB_OUTPUT | |
| - name: Run schema migrations | |
| id: run-schema-migrations | |
| env: | |
| SLACK_MAVIS_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }} | |
| run: | | |
| TASK_DEFINITION_ARN=${{ steps.register-migration-task-definition.outputs.task_definition_arn }} | |
| SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=private-subnet-$environment-a --query 'Subnets[0].SubnetId' --output text) | |
| SECURITY_GROUP_ID=$(aws ec2 describe-security-groups --filters Name=group-name,Values=ops-service-$environment --query 'SecurityGroups[0].GroupId' --output text) | |
| MAX_ATTEMPTS=3 | |
| ATTEMPT=1 | |
| while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do | |
| TASK_ARN=$(aws ecs run-task \ | |
| --cluster $cluster_name \ | |
| --task-definition $TASK_DEFINITION_ARN \ | |
| --launch-type FARGATE \ | |
| --network-configuration "awsvpcConfiguration={subnets=[$SUBNET_ID],securityGroups=[$SECURITY_GROUP_ID]}" \ | |
| --overrides '{ | |
| "containerOverrides": [{ | |
| "name": "application", | |
| "command": ["bin/rails","db:migrate"] | |
| }] | |
| }' \ | |
| --query 'tasks[0].taskArn' \ | |
| --output text) | |
| echo "Waiting for task to complete: $TASK_ARN" | |
| TASK_ID=$(sed 's:^.*/::' <<< $TASK_ARN) | |
| AWS_CONSOLE_URL="https://eu-west-2.console.aws.amazon.com/ecs/v2/clusters/$cluster_name/tasks/$TASK_ID/logs" | |
| echo "View logs in AWS Console: $AWS_CONSOLE_URL" | |
| if [ $environment = 'production' ]; then | |
| ./.github/send_slack_notification.sh "${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }}" "$AWS_CONSOLE_URL" "Running schema migrations attempt $ATTEMPT/$MAX_ATTEMPTS" | |
| fi | |
| MAX_WAIT_TIME=3600 | |
| POLL_INTERVAL=10 # Poll every 10 seconds | |
| ELAPSED=0 | |
| while [ $ELAPSED -lt $MAX_WAIT_TIME ]; do | |
| TASK_STATUS=$(aws ecs describe-tasks \ | |
| --cluster $cluster_name \ | |
| --tasks $TASK_ID \ | |
| --query 'tasks[0].lastStatus' \ | |
| --output text) | |
| if [ "$TASK_STATUS" = "STOPPED" ]; then | |
| echo "Task has stopped" | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| ELAPSED=$((ELAPSED + POLL_INTERVAL)) | |
| done | |
| if [ $ELAPSED -ge $MAX_WAIT_TIME ]; then | |
| echo "ERROR: Migration task did not complete within $MAX_WAIT_TIME seconds." | |
| exit 1 | |
| fi | |
| EXIT_CODE=$(aws ecs describe-tasks \ | |
| --cluster $cluster_name \ | |
| --tasks $TASK_ARN \ | |
| --query 'tasks[0].containers[0].exitCode' \ | |
| --output text) | |
| echo "Container exit code: $EXIT_CODE" | |
| if [ "$EXIT_CODE" = "0" ]; then | |
| echo "Migrations completed" | |
| break | |
| else | |
| echo "ECS task failed with exit code: $EXIT_CODE" | |
| if [ "$ATTEMPT" = "$MAX_ATTEMPTS" ]; then | |
| exit 1 | |
| fi | |
| ATTEMPT=$((ATTEMPT+1)) | |
| fi | |
| done | |
| - name: Notify migrations completed | |
| if: ${{ env.environment == 'production' }} | |
| uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a | |
| with: | |
| webhook: ${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "Schema migrations finished successfully :white_check_mark:" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "Schema migrations finished successfully :white_check_mark:" | |
| deploy-service: | |
| name: Deploy service | |
| runs-on: ubuntu-latest | |
| needs: [ prepare-deployment, approve-deployments, run-migrations ] | |
| if: ${{ !cancelled() && needs.run-migrations.result != 'failure' }} | |
| permissions: | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| service: ${{ inputs.server_types == 'all' && fromJSON('["web", "sidekiq", "ops"]') || fromJSON(format('["{0}"]', inputs.server_types)) }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ env.aws_account_id }}:role/GithubDeployECSService | |
| aws-region: eu-west-2 | |
| - name: Download ${{ matrix.service }} task definition artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: ${{ runner.temp }} | |
| name: ${{ inputs.environment }}-${{ matrix.service }}-task-definition | |
| - name: Change family of task definition | |
| run: | | |
| file_path="${{ runner.temp }}/${{ matrix.service }}-task-definition.json" | |
| family_name="mavis-${{ matrix.service }}-task-definition-$environment" | |
| echo "$(jq --arg f "$family_name" '.family = $f' "$file_path")" > "$file_path" | |
| - name: Deploy ${{ matrix.service }} service | |
| id: ecs-deploy | |
| uses: aws-actions/amazon-ecs-deploy-task-definition@v2 | |
| with: | |
| task-definition: ${{ runner.temp }}/${{ matrix.service }}-task-definition.json | |
| cluster: ${{ env.cluster_name }} | |
| service: mavis-${{ inputs.environment }}-${{ matrix.service }} | |
| force-new-deployment: true | |
| wait-for-service-stability: true | |
| - name: Check if deployment was successful | |
| run: | | |
| current_task_definition_arn=$(aws ecs describe-services --cluster mavis-$environment --services mavis-$environment-${{ matrix.service }} --query services[0].deployments[0].taskDefinition | jq -r ".") | |
| new_task_definition_arn=${{ steps.ecs-deploy.outputs.task-definition-arn }} | |
| echo "Current task definition arn: $current_task_definition_arn" | |
| echo "Expected task definition arn after deployment: $new_task_definition_arn" | |
| if [ "$current_task_definition_arn" != "$new_task_definition_arn" ]; then | |
| echo "Task definition arns don't match, likely due to a rollback to the previous version. Deployment failed." | |
| exit 1 | |
| fi | |
| notify-deployment-complete: | |
| name: Notify deployment status | |
| runs-on: ubuntu-latest | |
| needs: [ deploy-service, run-migrations ] | |
| if: ${{ !cancelled() && inputs.environment == 'production' }} | |
| steps: | |
| - name: Notify deployment success | |
| if: ${{ needs.deploy-service.result == 'success' }} | |
| uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a | |
| with: | |
| webhook: ${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "*${{ env.app_version }}* is deployed :ship:" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "*${{ env.app_version }}* is deployed :ship:" | |
| - name: Notify deployment failure | |
| if: ${{ needs.deploy-service.result == 'failure' || needs.run-migrations.result == 'failure' }} | |
| uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a | |
| with: | |
| webhook: ${{ secrets.SLACK_MAVIS_RELEASES_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "Deployment of *${{ env.app_version }}* failed :x:" | |
| blocks: | |
| - type: "section" | |
| text: | |
| type: "mrkdwn" | |
| text: "Deployment of *${{ env.app_version }}* failed :x:" | |
| - type: "section" | |
| fields: | |
| - type: "mrkdwn" | |
| text: "*Failed job:*\n${{ needs.deploy-service.result == 'failure' && 'deploy-service' || 'run-migrations' }}" | |
| - type: "mrkdwn" | |
| text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>" |