dataflow-ops-aws-eks/.github/workflows/main.yaml at main · anna-geller/dataflow-ops-aws-eks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
name: 2 Regular CD to AWS EKS, S3 and Prefect Cloud

on:
  workflow_dispatch:
#  push:
#    branches:
#      - main

env:
  NAMESPACE_ENV: prod
  PREFECT_VERSION: 2.*
  AWS_DEFAULT_REGION: us-east-1

jobs:
  changes:
    name: Flow and code dependency changes
    runs-on: ubuntu-latest
    outputs:
      prefect_flows: ${{ steps.filter.outputs.flows_files }}
      prefect_flows_changed: ${{ steps.filter.outputs.flows }}
      code_dependencies_changed: ${{ steps.filter.outputs.code }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          list-files: json
          filters: |
            flows:
              - added|modified: 'flows/*.py'
            code:
              - added|modified: 'requirements.txt'
              - added|modified: 'Dockerfile'
      - name: Generate Markdown Summary
        run: |
          echo Flow changes: ${{ steps.filter.outputs.flows_files }} >> $GITHUB_STEP_SUMMARY
          echo Code dependency changes: ${{ steps.filter.outputs.code_files }} >> $GITHUB_STEP_SUMMARY

  ecr-build:
    name: ECR image - rebuild and push
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.code_dependencies_changed == 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
          mask-aws-account-id: 'no'

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1

      - name: Build, tag, and push image to Amazon ECR
        id: build-image
        env:
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          IMAGE_TAG: ${{ github.sha }}
        run: |
          docker build -t $ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG .
          docker push $ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG
          echo "::set-output name=image::$ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG"
          image: ${{ steps.build-image.outputs.image }}

  k8-block:
    name: Create blocks
    runs-on: ubuntu-latest
    needs: ecr-build
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Python dependencies
        run: pip install -U "prefect==$PREFECT_VERSION"

      - name: Prefect Cloud login
        run: |
          prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}
          prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}

      - name: Update block's image
        run: |
          cat <<EOF > k8s_block.py
          from prefect.infrastructure import KubernetesJob
          k8s = KubernetesJob.load("${{ env.NAMESPACE_ENV }}")
          k8s.image = "${{ steps.build-image.outputs.image }}"
          k8s.save("${{ env.NAMESPACE_ENV }}", overwrite=True)
          EOF
          python k8s_block.py

  deploy:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.prefect_flows_changed == 'true' }}
    strategy:
      matrix:
        flows: ${{ fromJson(needs.changes.outputs.prefect_flows) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Python dependencies
        run: |
          pip install -U "prefect==$PREFECT_VERSION"
          pip install s3fs
          pip install .

      - name: Prefect Cloud login
        run: |
          prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}
          prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}

      - name: AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
          mask-aws-account-id: 'no'

      - name: Deploy flows to S3
        id: build
        run: |
          FLOW_NAME=$(basename ${{ matrix.flows }} .py)
          prefect deployment build ${{ matrix.flows }}:$FLOW_NAME --name ${{ env.NAMESPACE_ENV }} -q $NAMESPACE_ENV \
          -sb s3/$NAMESPACE_ENV -ib kubernetes-job/$NAMESPACE_ENV -a -v $GITHUB_SHA -o $FLOW_NAME.yaml
          echo ::set-output name=flow_manifest::$FLOW_NAME.yaml
      - name: Upload deployment manifest as artifact
        uses: actions/upload-artifact@v3
        with:
          name: Deployment YAML manifests
          path: ${{ steps.build.outputs.flow_manifest }}

      - name: Summary
        run: echo 'Prefect flows successfully redeployed! :rocket:' >> $GITHUB_STEP_SUMMARY