CCM-13478: Improve Pipeline Times (#191)

* CCM-13478: Initial commit with various improvements

* CCM-13478: Further changes

* CCM-13478: Fix npm ci

* CCM-13478: Add read permission

* CCM-13478: Componenet test parallel test

* CCM-13478: Update lint/typecheck

* CCM-13478: Revert typecheck change

* CCM-13478: Revert parallel change

* CCM-13478: Add read permission to schema check

* CCM-13478: Fix missing permission

* CCM-13478: Remove echo steps and increase workers

* CCM-13478: Increase workers to 35

* CCM-13478: Reset workers to 8

Author: ScottFullerton-NHSE

.github/actions/acceptance-tests/action.yaml

@@ -34,7 +34,7 @@ runs:
       with:
         node-version: ${{ steps.nodejs_version.outputs.nodejs_version }}
         GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }}
-    - name: "Repo setup"
+    - name: "Install dependencies"
       shell: bash
       run: |
         npm ci

.github/actions/build-docs/action.yml

@@ -27,8 +27,8 @@ runs:
       uses: ruby/setup-ruby@v1.267.0
       with:
         ruby-version: "3.4.7" # Not needed with a .ruby-version file
-        bundler-cache: false # runs 'bundle install' and caches installed gems automatically
-        #cache-version: 0 # Increment this number if you need to re-download cached gems
+        bundler-cache: true # Enable automatic gem caching
+        cache-version: 0 # Increment this number if you need to re-download cached gems
         working-directory: "./docs"
     - name: Setup Pages
       id: pages

.github/actions/node-install/action.yaml

@@ -16,6 +16,8 @@ runs:
       uses: actions/setup-node@v6
       with:
         node-version: '${{ inputs.node-version }}'
+        cache: 'npm'
+        cache-dependency-path: '**/package-lock.json'
 
     - name: "Configure npm for GitHub Packages"
       shell: bash

.github/workflows/stage-1-commit.yaml

@@ -55,8 +55,6 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
-        with:
-          fetch-depth: 0 # Full history is needed to compare branches
       - name: "Check file format"
         uses: ./.github/actions/check-file-format
   check-markdown-format:
@@ -66,8 +64,6 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
-        with:
-          fetch-depth: 0 # Full history is needed to compare branches
       - name: "Check Markdown format"
         uses: ./.github/actions/check-markdown-format
   terraform-docs:
@@ -101,8 +97,6 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
-        with:
-          fetch-depth: 0 # Full history is needed to compare branches
       - name: "Check English usage"
         uses: ./.github/actions/check-english-usage
   check-todo-usage:
@@ -112,8 +106,6 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
-        with:
-          fetch-depth: 0 # Full history is needed to compare branches
       - name: "Check TODO usage"
         uses: ./.github/actions/check-todo-usage
   detect-terraform-changes:

.github/workflows/stage-2-test.yaml

@@ -41,10 +41,34 @@ env:
   TERM: xterm-256color
 
 jobs:
+  detect-schema-changes:
+    name: "Detect Schema Changes"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      schemas_changed: ${{ steps.filter.outputs.schemas }}
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@v5
+
+      - name: "Check for schema changes"
+        uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            schemas:
+              - 'src/cloudevents/**'
+              - 'src/typescript-schema-generator/**'
+              - 'src/python-schema-generator/**'
+              - 'src/digital-letters-events/**'
+
   check-generated-dependencies:
     name: "Check generated dependencies"
+    needs: [detect-schema-changes]
+    if: needs.detect-schema-changes.outputs.schemas_changed == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 4
     permissions:
       contents: read
       packages: read
@@ -80,6 +104,8 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: ${{ inputs.python_version }}
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
       - name: "Run unit test suite"
         run: |
           make test-unit
@@ -106,7 +132,7 @@ jobs:
   test-lint:
     name: "Linting"
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 4
     permissions:
       contents: read
       packages: read
@@ -117,6 +143,8 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: ${{ inputs.python_version }}
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
       - uses: ./.github/actions/node-install
         with:
           node-version: ${{ inputs.nodejs_version }}
@@ -127,7 +155,7 @@ jobs:
   test-typecheck:
     name: "Typecheck"
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 4
     permissions:
       contents: read
       packages: read
@@ -143,30 +171,14 @@ jobs:
       - name: "Run typecheck"
         run: |
           make test-typecheck
-  test-coverage:
-    name: "Test coverage"
-    needs: [test-unit]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    permissions:
-      contents: read
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@v5
-      - name: "Run test coverage check"
-        run: |
-          make test-coverage
-      - name: "Save the coverage check result"
-        run: |
-          echo "Nothing to save"
   perform-static-analysis:
     name: "Perform static analysis"
     needs: [test-unit]
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: read
-    timeout-minutes: 5
+    timeout-minutes: 4
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5

.github/workflows/stage-3-build.yaml

@@ -33,10 +33,45 @@ on:
         type: string
 
 jobs:
+  detect-doc-changes:
+    name: "Detect Documentation Changes"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      docs_changed: ${{ steps.filter.outputs.docs }}
+      schemas_changed: ${{ steps.filter.outputs.schemas }}
+      eventcatalog_changed: ${{ steps.filter.outputs.eventcatalog }}
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@v5
+
+      - name: "Check for relevant changes"
+        uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            docs:
+              - 'docs/**'
+            schemas:
+              - 'src/cloudevents/**'
+              - 'src/cloudeventjekylldocs/**'
+            eventcatalog:
+              - 'src/eventcatalog/**'
+              - 'src/asyncapigenerator/**'
+              - 'src/eventcatalogasyncapiimporter/**'
+
   artefact-jekyll-docs:
     name: "Build Docs"
+    needs: [detect-doc-changes]
+    if: |
+      needs.detect-doc-changes.outputs.docs_changed == 'true' ||
+      needs.detect-doc-changes.outputs.schemas_changed == 'true' ||
+      needs.detect-doc-changes.outputs.eventcatalog_changed == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    permissions:
+      contents: read
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5

.github/workflows/stage-4-acceptance.yaml

@@ -22,26 +22,9 @@ on:
         type: string
 
 jobs:
-  environment-set-up:
-    name: "Environment set up"
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@v5
-      - name: "Create infractructure"
-        run: |
-          echo "Creating infractructure..."
-      - name: "Update database"
-        run: |
-          echo "Updating database..."
-      - name: "Deploy application"
-        run: |
-          echo "Deploying application..."
   test-contract:
     name: "Contract test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -55,7 +38,6 @@ jobs:
   test-security:
     name: "Security test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -69,7 +51,6 @@ jobs:
   test-ui:
     name: "UI test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -83,7 +64,6 @@ jobs:
   test-ui-performance:
     name: "UI performance test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -97,7 +77,6 @@ jobs:
   test-integration:
     name: "Integration test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     steps:
       - uses: actions/checkout@v5.0.0
       # Calls out to the nhs-notify-internal repo.
@@ -122,7 +101,6 @@ jobs:
   test-accessibility:
     name: "Accessibility test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -136,7 +114,6 @@ jobs:
   test-load:
     name: "Load test"
     runs-on: ubuntu-latest
-    needs: environment-set-up
     timeout-minutes: 10
     steps:
       - name: "Checkout code"
@@ -147,24 +124,3 @@ jobs:
       - name: "Save result"
         run: |
           echo "Nothing to save"
-  environment-tear-down:
-    name: "Environment tear down"
-    runs-on: ubuntu-latest
-    needs:
-      [
-        test-accessibility,
-        test-contract,
-        test-integration,
-        test-load,
-        test-security,
-        test-ui-performance,
-        test-ui,
-      ]
-    if: always()
-    timeout-minutes: 5
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@v5
-      - name: "Tear down environment"
-        run: |
-          echo "Tearing down environment..."

scripts/tests/lint.sh

@@ -5,5 +5,4 @@ set -euo pipefail
 cd "$(git rev-parse --show-toplevel)"
 
 npm ci
-npm run generate-dependencies
 npm run lint