VED-1013 Introduce new e2e tests from automation testers (#1146)

Co-authored-by: Krupasindhu Baral <krupasindhu.baral@nhs.net>
Co-authored-by: James W. <218585646+JamesW1-NHS@users.noreply.github.com>
Co-authored-by: fimran08 <90399910+fimran08@users.noreply.github.com>

Author: 4 people

.github/dependabot.yml

@@ -62,8 +62,7 @@ updates:
       - "/lambdas/recordprocessor"
       - "/lambdas/redis_sync"
       - "/lambdas/shared"
-      - "/tests/e2e"
-      - "/tests/e2e_batch"
+      - "/tests/e2e_automation"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1

.github/workflows/continuous-deployment.yml

@@ -18,41 +18,50 @@ jobs:
     # Technically the first step is not a pre-requisite - sandbox backend deployment is handled by APIM
     # Stipulating this condition simply makes it more likely the environment will be ready when tests are invoked
     needs: [deploy-internal-dev-backend]
-    uses: ./.github/workflows/run-e2e-tests.yml
+    uses: ./.github/workflows/run-e2e-automation-tests.yml
     with:
       apigee_environment: internal-dev-sandbox
       environment: dev
       sub_environment: internal-dev-sandbox
+      service_under_test: all
+      suite_to_run: sandbox
     secrets:
       APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
       APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
       APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+      CIS2_E2E_USERNAME: ${{ secrets.CIS2_E2E_USERNAME }}
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}
 
   run-sandbox-tests:
     needs: [run-internal-dev-sandbox-tests]
-    uses: ./.github/workflows/run-e2e-tests.yml
+    uses: ./.github/workflows/run-e2e-automation-tests.yml
     with:
       apigee_environment: sandbox
       environment: dev
       sub_environment: sandbox
+      service_under_test: all
+      suite_to_run: sandbox
     secrets:
       APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
       APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
       APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+      CIS2_E2E_USERNAME: ${{ secrets.CIS2_E2E_USERNAME }}
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}
 
   run-internal-dev-tests:
     needs: [deploy-internal-dev-backend]
-    uses: ./.github/workflows/run-e2e-tests.yml
+    uses: ./.github/workflows/run-e2e-automation-tests.yml
     with:
       apigee_environment: internal-dev
       environment: dev
       sub_environment: internal-dev
+      service_under_test: all
+      suite_to_run: smoke
     secrets:
       APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
       APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
       APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+      CIS2_E2E_USERNAME: ${{ secrets.CIS2_E2E_USERNAME }}
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}
 
   deploy-higher-dev-envs:
@@ -72,13 +81,21 @@ jobs:
     strategy:
       matrix:
         sub_environment_name: [ref, internal-qa]
-    uses: ./.github/workflows/run-e2e-tests.yml
+        include:
+          - sub_environment_name: ref
+            required_test_suite: proxy_smoke
+          - sub_environment_name: internal-qa
+            required_test_suite: smoke
+    uses: ./.github/workflows/run-e2e-automation-tests.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
       environment: dev
       sub_environment: ${{ matrix.sub_environment_name }}
+      service_under_test: all
+      suite_to_run: ${{ matrix.required_test_suite }}
     secrets:
       APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
       APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
       APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+      CIS2_E2E_USERNAME: ${{ secrets.CIS2_E2E_USERNAME }}
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}

.github/workflows/pr-deploy-and-test.yml

@@ -13,18 +13,26 @@ jobs:
       environment: dev
       sub_environment: pr-${{github.event.pull_request.number}}
 
-  run-e2e-tests:
+  run-e2e-automation-tests:
     needs: [deploy-pr-backend]
     strategy:
       matrix:
         apigee_environment_name: [internal-dev, internal-dev-sandbox]
-    uses: ./.github/workflows/run-e2e-tests.yml
+        include:
+          - apigee_environment_name: internal-dev
+            required_test_suite: smoke
+          - apigee_environment_name: internal-dev-sandbox
+            required_test_suite: sandbox
+    uses: ./.github/workflows/run-e2e-automation-tests.yml
     with:
       apigee_environment: ${{ matrix.apigee_environment_name }}
       environment: dev
       sub_environment: pr-${{github.event.pull_request.number}}
+      service_under_test: all
+      suite_to_run: ${{ matrix.required_test_suite }}
     secrets:
       APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
       APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
       APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+      CIS2_E2E_USERNAME: ${{ secrets.CIS2_E2E_USERNAME }}
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}

.github/workflows/run-e2e-tests.yml …b/workflows/run-e2e-automation-tests.yml.github/workflows/run-e2e-tests.yml renamed to .github/workflows/run-e2e-automation-tests.yml .github/workflows/run-e2e-tests.yml renamed to .github/workflows/run-e2e-automation-tests.yml

@@ -1,4 +1,4 @@
-name: Run e2e Tests
+name: FHIR API and Batch Automation - E2E Tests
 
 on:
   workflow_call:
@@ -12,13 +12,21 @@ on:
       sub_environment:
         required: true
         type: string
+      service_under_test:
+        required: true
+        type: string
+      suite_to_run:
+        required: true
+        type: string
     secrets:
       APIGEE_PASSWORD:
         required: true
       APIGEE_BASIC_AUTH_TOKEN:
         required: true
       APIGEE_OTP_KEY:
         required: true
+      CIS2_E2E_USERNAME:
+        required: true
       STATUS_API_KEY:
         required: true
   workflow_dispatch:
@@ -33,19 +41,35 @@ on:
           - internal-qa
           - int
           - ref
-          - prod
       environment:
         type: string
         description: Select the backend environment
         options:
           - dev
           - preprod
-          - prod
       sub_environment:
         type: string
         description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
+      service_under_test:
+        description: Select the service you want to test
+        required: true
+        type: choice
+        options:
+          - fhir_api
+          - batch
+          - all
+      suite_to_run:
+        description: Select the suite you would like to run
+        default: "functional"
+        type: choice
+        options:
+          - smoke
+          - functional
+          - sandbox
+          - proxy_smoke
 
 env:
+  APIGEE_AUTH_ENV: ${{ inputs.apigee_environment == 'int' && inputs.apigee_environment || 'internal-dev' }}
   APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
   ENVIRONMENT: ${{ inputs.environment }}
   SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
@@ -60,11 +84,9 @@ jobs:
       contents: read
     runs-on: ubuntu-latest
     environment: ${{ inputs.apigee_environment }}
-    outputs:
-      # Workaround for environment-level variables being unavailable in `jobs.<job-id>.if`.
-      RUN_BATCH_E2E_TESTS: ${{ vars.RUN_BATCH_E2E_TESTS }}
     steps:
       - name: Wait for API to be available
+        if: github.event_name != 'workflow_dispatch'
         run: |
           endpoint=""
           if [[ ${APIGEE_ENVIRONMENT} =~ "prod" ]]; then
@@ -106,16 +128,18 @@ jobs:
               exit 1
           fi
 
-  e2e-tests:
+  e2e-automation-tests:
     permissions:
       id-token: write
-      contents: read
+      checks: write
+      contents: write
     runs-on: ubuntu-latest
     needs: [wait-for-deployment]
     environment: ${{ inputs.apigee_environment }}
     env:
       APIGEE_USERNAME: ${{ vars.APIGEE_USERNAME }}
-      TF_OUTPUTS_REQUIRED: ${{ vars.RUN_FULL_E2E_TESTS == 'true' || vars.RUN_PROXY_E2E_TESTS == 'true' }}
+      TF_OUTPUTS_REQUIRED: ${{ inputs.suite_to_run != 'sandbox' }}
+
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
@@ -146,11 +170,7 @@ jobs:
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
         working-directory: infrastructure/instance
         run: |
-          echo "IMMS_DELTA_TABLE_NAME=$(make -s output name=imms_delta_table_name)" >> $GITHUB_ENV
           echo "AWS_DOMAIN_NAME=$(make -s output name=service_domain_name)" >> $GITHUB_ENV
-          echo "DYNAMODB_TABLE_NAME=$(make -s output name=dynamodb_table_name)" >> $GITHUB_ENV
-          echo "AWS_SQS_QUEUE_NAME=$(make -s output name=aws_sqs_queue_name)" >> $GITHUB_ENV
-          echo "AWS_SNS_TOPIC_NAME=$(make -s output name=aws_sns_topic_name)" >> $GITHUB_ENV
 
       - name: Install poetry
         run: pip install poetry==2.1.4
@@ -159,21 +179,21 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
-          cache-dependency-path: tests/e2e/poetry.lock
+          cache-dependency-path: tests/e2e_automation/poetry.lock
 
       - name: Install e2e test dependencies
-        working-directory: tests/e2e
+        working-directory: tests/e2e_automation
         run: poetry install --no-root
 
       - name: Get Apigee access token
-        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: tests/e2e
+        if: inputs.apigee_environment != 'int'
+        working-directory: tests/e2e_automation
         env:
           APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
           APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
           APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
         run: |
-          CODE=$(poetry run python utils/compute_totp_code.py "$APIGEE_OTP_KEY")
+          CODE=$(poetry run python utilities/compute_totp_code.py "$APIGEE_OTP_KEY")
           echo "::add-mask::$CODE"
 
           echo "Requesting access token from Apigee..."
@@ -187,55 +207,73 @@ jobs:
           echo "::add-mask::$token"
           echo "APIGEE_ACCESS_TOKEN=$token" >> $GITHUB_ENV
 
-      - name: Run proxy e2e test suite
-        if: ${{ vars.RUN_PROXY_E2E_TESTS == 'true' }}
-        working-directory: tests/e2e
-        run: poetry run python -m unittest test_proxy
-
-      - name: Run sandbox e2e test suite
-        if: ${{ vars.RUN_SANDBOX_E2E_TESTS == 'true' }}
-        working-directory: tests/e2e
-        run: poetry run python -m unittest test_proxy.TestProxyHealthcheck
-
-      - name: Run full e2e test suite
-        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: tests/e2e
-        run: poetry run python -m unittest
-
-  batch-e2e-tests:
-    permissions:
-      id-token: write
-      contents: read
-    needs: [wait-for-deployment, e2e-tests]
-    # Only actually depend on wait-for-deployment, but run after e2e-tests
-    if: ${{ !cancelled() && needs.wait-for-deployment.result == 'success' && needs.wait-for-deployment.outputs.RUN_BATCH_E2E_TESTS == 'true' }}
-    runs-on: ubuntu-latest
-    environment: ${{ inputs.apigee_environment }}
-    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+      - name: Run Pytest-BDD ${{ inputs.service_under_test }} tests with the ${{ inputs.suite_to_run }} filter
+        working-directory: tests/e2e_automation
+        env:
+          S3_env: ${{ inputs.sub_environment }}
+          auth_url: https://${{ env.APIGEE_AUTH_ENV }}.api.service.nhs.uk/oauth2-mock/authorize
+          token_url: https://${{ env.APIGEE_AUTH_ENV }}.api.service.nhs.uk/oauth2-mock/token
+          baseUrl: https://${{ inputs.apigee_environment }}.api.service.nhs.uk/${{env.SERVICE_BASE_PATH}}
+          callback_url: "https://oauth.pstmn.io/v1/callback"
+          username: ${{ secrets.CIS2_E2E_USERNAME }}
+          scope: "nhs-cis2"
+          USE_STATIC_APPS: ${{ inputs.apigee_environment == 'int' && 'True' || 'False' }}
+          Postman_Auth_client_Id: ${{ secrets.Postman_Auth_client_Id }} # The following static app values are only needed in INT
+          Postman_Auth_client_Secret: ${{ secrets.Postman_Auth_client_Secret }}
+          RAVS_client_Id: ${{ secrets.RAVS_client_Id }}
+          RAVS_client_Secret: ${{ secrets.RAVS_client_Secret }}
+          MAVIS_client_Id: ${{ secrets.MAVIS_client_Id }}
+          MAVIS_client_Secret: ${{ secrets.MAVIS_client_Secret }}
+          EMIS_client_Id: ${{ secrets.OPTUM_client_Id }}
+          EMIS_client_Secret: ${{ secrets.OPTUM_client_Secret }}
+          TPP_client_Id: ${{ secrets.TPP_client_Id }}
+          TPP_client_Secret: ${{ secrets.TPP_client_Secret }}
+          SONAR_client_Id: ${{ secrets.SONAR_client_Id }}
+          SONAR_client_Secret: ${{ secrets.SONAR_client_Secret }}
+          MEDICUS_client_Id: ${{ secrets.MEDICUS_client_Id }}
+          MEDICUS_client_Secret: ${{ secrets.MEDICUS_client_Secret }}
+          aws_token_refresh: "False"
+          TEST_PATH: ${{ inputs.service_under_test == 'batch' && 'features/batchTests' || inputs.service_under_test == 'fhir_api' && 'features/APITests' || 'features' }}
+          TEST_FILTER: ${{ inputs.suite_to_run == 'proxy_smoke' && 'Status_feature' || inputs.suite_to_run }}
+        run: poetry run pytest "$TEST_PATH" -m "$TEST_FILTER" --junitxml=output/test-results.xml --alluredir=output/allure-results
 
-      - name: Connect to AWS
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
+      - uses: dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5
+        if: always()
         with:
-          aws-region: eu-west-2
-          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
-          role-session-name: github-actions
+          name: BDD Test Summary
+          path: "**/output/test-results.xml"
+          reporter: java-junit
+          fail-on-error: false
 
-      - name: Install poetry
-        run: pip install poetry==2.1.4
+      - name: Load test report history
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        if: always()
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
 
-      - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548
+      - name: Build Allure report
+        if: always()
+        uses: simple-elf/allure-report-action@53ebb757a2097edc77c53ecef4d454fc2f2f774c
         with:
-          python-version: 3.11
-          cache: "poetry"
-          cache-dependency-path: tests/e2e_batch/poetry.lock
+          allure_results: tests/e2e_automation/output/allure-results
+          gh_pages: gh-pages
+          allure_report: allure-report
+          allure_history: allure-history
 
-      - name: Install e2e test dependencies
-        working-directory: tests/e2e_batch
-        run: poetry install --no-root
+      - name: Publish Allure report to GitHub Pages
+        if: always()
+        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: allure-history
 
-      - name: Run batch e2e test suite
-        working-directory: tests/e2e_batch
-        env:
-          ENVIRONMENT: ${{ inputs.sub_environment }}
-        run: poetry run python -m unittest -c -v
+      - name: Add link to Allure report
+        if: always()
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const url = `https://${context.repo.owner}.github.io/${context.repo.repo}/`;
+            core.summary.addHeading('Allure Report').addLink('View Report', url).write();

.gitignore

@@ -12,6 +12,7 @@ newman/
 **/*.iml
 
 __pycache__/
+.pytest_cache/
 .venv/
 .env
 .envrc
@@ -30,3 +31,8 @@ openapi.json
 
 devtools/volume/
 **/.coverage
+**/test-results.xml
+allure-results/
+allure-report/
+**/debugLog.log
+batch_files_directory/