Update: [AEA-6053] - document preprocessing (#221)

## Summary

- 🤖 Operational or Infrastructure Change
- ✨ New Feature

### Details

- add preprocessing lambda to convert documents to markdown before KB
ingestion
- two-stage S3 pipeline: `raw/` -> preprocessing -> `processed/` -> sync
- Sonar fix suggestion: replace `/tmp` security issue with secure temp
directory handling
- cfn Guard suppressions for Lambda permission
- cdk-nag suppressions for preprocessing policie
- add excel-specific filtering for NHS scal documents
- excluding cli.py and magika_shim.py from SonarCloud coverage

---------

Co-authored-by: Beenyaa <bencegadanyi1@hotmail.com>

Author: bencegadanyi1-nhs

.github/actions/sync_documents/action.yml

@@ -37,7 +37,7 @@ runs:
       shell: bash
       run: |
         mkdir -p ./s3-content
-        aws s3 sync s3://${{ steps.find-source-bucket.outputs.BUCKET_NAME }} ./s3-content
+        aws s3 sync s3://${{ steps.find-source-bucket.outputs.BUCKET_NAME }} ./s3-content --exclude "raw/*" --exclude "processed/*"
 
     - name: Connect to Target Account
       uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
@@ -65,9 +65,9 @@ runs:
         if [ -z "$DIFFS" ]; then
           echo -e "\033[0;32m✔ NO DISCREPANCIES FOUND.\033[0m"
         else
-          echo -e "\033[0;33m⚠ WARNING: DISCREPANCIES FOUND:" 
+          echo -e "\033[0;33m⚠ WARNING: DISCREPANCIES FOUND:"
 
-          echo "$DIFFS" 
+          echo "$DIFFS"
           echo "--------------------------------------------------\033[0m"
 
           CLEAN_DIFFS="${DIFFS//$'\n'/'%0A'}"
@@ -78,5 +78,5 @@ runs:
     - name: Upload Files to Target S3
       shell: bash
       run: |
-        echo "Updating s3://${{ steps.find-destination-bucket.outputs.BUCKET_NAME }}..."
-        aws s3 sync ./s3-content s3://${{ steps.find-destination-bucket.outputs.BUCKET_NAME }} --delete
+        echo "Updating s3://${{ steps.find-destination-bucket.outputs.BUCKET_NAME }}/processed/..."
+        aws s3 sync ./s3-content s3://${{ steps.find-destination-bucket.outputs.BUCKET_NAME }}/processed/ --delete

.github/workflows/cdk_package_code.yml

@@ -68,6 +68,7 @@ jobs:
         run: |
           poetry show --only=slackBotFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > requirements_slackBotFunction
           poetry show --only=syncKnowledgeBaseFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > requirements_syncKnowledgeBaseFunction
+          poetry show --only=preprocessingFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > requirements_preprocessingFunction
           poetry show --only=bedrockLoggingConfigFunction | grep -E "^[a-zA-Z]" | awk '{print $1"=="$2}' > requirements_bedrockLoggingConfigFunction
           if [ ! -s requirements_slackBotFunction ] || [ "$(grep -c -v '^[[:space:]]*$' requirements_slackBotFunction)" -eq 0 ]; then \
             echo "Error: requirements_slackBotFunction is empty or contains only blank lines"; \
@@ -77,12 +78,29 @@ jobs:
             echo "Error: requirements_syncKnowledgeBaseFunction is empty or contains only blank lines"; \
             exit 1; \
           fi
+          if [ ! -s requirements_preprocessingFunction ] || [ "$(grep -c -v '^[[:space:]]*$' requirements_preprocessingFunction)" -eq 0 ]; then \
+            echo "Error: requirements_preprocessingFunction is empty or contains only blank lines"; \
+            exit 1; \
+          fi
           if [ ! -s requirements_bedrockLoggingConfigFunction ] || [ "$(grep -c -v '^[[:space:]]*$' requirements_bedrockLoggingConfigFunction)" -eq 0 ]; then \
             echo "Error: requirements_bedrockLoggingConfigFunction is empty or contains only blank lines"; \
             exit 1; \
           fi
           mkdir -p .dependencies/slackBotFunction/python
           mkdir -p .dependencies/syncKnowledgeBaseFunction/python
+          mkdir -p .dependencies/preprocessingFunction/python
+          pip3 install -r requirements_slackBotFunction -t .dependencies/slackBotFunction/python
+          pip3 install -r requirements_syncKnowledgeBaseFunction -t .dependencies/syncKnowledgeBaseFunction/python
+          pip3 install -r requirements_preprocessingFunction -t .dependencies/preprocessingFunction/python
+          rm -rf .dependencies/preprocessingFunction/python/magika* .dependencies/preprocessingFunction/python/onnxruntime*
+          cp packages/preprocessingFunction/magika_shim.py .dependencies/preprocessingFunction/python/magika.py
+          find .dependencies/preprocessingFunction/python -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true
+          find .dependencies/preprocessingFunction/python -type d -name "test" -exec rm -rf {} + 2>/dev/null || true
+          find .dependencies/preprocessingFunction/python -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+          find .dependencies/preprocessingFunction/python -type d -name "examples" -exec rm -rf {} + 2>/dev/null || true
+          find .dependencies/preprocessingFunction/python -type f \( -name "*.pyc" -o -name "*.pyo" -o -name "*.so.debug" \) -delete
+          find .dependencies/preprocessingFunction/python -type f -name "*.md" ! -name "README.md" -delete
+          find .dependencies/preprocessingFunction/python -name "*.txt" -size +10k -delete
           mkdir -p .dependencies/bedrockLoggingConfigFunction/python
           pip3 install -r requirements_slackBotFunction -t .dependencies/slackBotFunction/python
           pip3 install -r requirements_syncKnowledgeBaseFunction -t .dependencies/syncKnowledgeBaseFunction/python

.github/workflows/pull_request.yml

@@ -8,12 +8,6 @@ env:
   BRANCH_NAME: ${{ github.event.pull_request.head.ref }}
 
 jobs:
-  dependabot-auto-approve-and-merge:
-    needs: quality_checks
-    uses: NHSDigital/eps-common-workflows/.github/workflows/dependabot-auto-approve-and-merge.yml@b933ef1bb3527fd7e7d5a7629fbd4e4dd94bf1b4
-    secrets:
-      AUTOMERGE_APP_ID: ${{ secrets.AUTOMERGE_APP_ID }}
-      AUTOMERGE_PEM: ${{ secrets.AUTOMERGE_PEM }}
   get_asdf_version:
     runs-on: ubuntu-22.04
     outputs:
@@ -32,20 +26,105 @@ jobs:
           TAG_FORMAT=$(yq '.TAG_FORMAT' .github/config/settings.yml)
           echo "TAG_FORMAT=$TAG_FORMAT" >> "$GITHUB_OUTPUT"
 
+  get_commit_message:
+    runs-on: ubuntu-22.04
+    outputs:
+      commit_message: ${{ steps.commit_message.outputs.commit_message }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        with:
+          ref: ${{ env.BRANCH_NAME }}
+          fetch-depth: 0
+      - name: Get Commit message
+        id: commit_message
+        run: |
+          echo "commit_message=$(git show -s --format=%s)" >> "$GITHUB_OUTPUT"
+
   quality_checks:
     uses: NHSDigital/eps-common-workflows/.github/workflows/quality-checks.yml@b933ef1bb3527fd7e7d5a7629fbd4e4dd94bf1b4
-    needs: [get_asdf_version]
+    needs: [get_asdf_version, get_commit_message]
+    if: ${{ ! contains(needs.get_commit_message.outputs.commit_message, '#skip-qc') }}
     with:
       asdfVersion: ${{ needs.get_asdf_version.outputs.asdf_version }}
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
+  quality_gate:
+    needs: get_commit_message
+    runs-on: ubuntu-22.04
+    if: always()
+    steps:
+      - name: Wait for quality checks to succeed
+        if: ${{ ! contains(needs.get_commit_message.outputs.commit_message, '#skip-qc') }}
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-encoding: json
+          script: |
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const runId = context.runId;
+
+            // How many times to poll
+            const pollTime = 10000; // 10 seconds
+            const maxRetries = 120; // 20 minutes at 10 seconds each
+            let attempts = 0;
+
+            async function fetchQCJob() {
+              const { data } = await github.rest.actions.listJobsForWorkflowRun({
+                owner, repo, run_id: runId
+              });
+              return data.jobs.find(job => job.name === 'quality_checks / quality_checks');
+            }
+
+            let qc = await fetchQCJob();
+            while ((!qc || qc.status !== 'completed') && attempts < maxRetries) {
+              attempts++;
+              console.log(`Attempt #${attempts}: ` +
+                (qc
+                  ? `found job "${qc.name}" with status=${qc.status}`
+                  : 'no matching quality_checks job yet'));
+              await new Promise(r => setTimeout(r, pollTime));
+              qc = await fetchQCJob();
+            }
+
+            if (!qc) {
+              core.setFailed(
+                `Timed out waiting for a "quality_checks" job (after ${attempts} polls).`
+              );
+              return;
+            }
+
+            if (qc.status !== 'completed') {
+              core.setFailed(
+                `Quality checks job never completed (last status=${qc.status}).`
+              );
+              return;
+            }
+
+            if (qc.conclusion !== 'success') {
+              core.setFailed(
+                `Quality checks failed (conclusion=${qc.conclusion}).`
+              );
+            }
+
+      - name: Bypass QC gate
+        if: ${{ contains(needs.get_commit_message.outputs.commit_message, '#skip-qc') }}
+        run: echo "Skipping QC gate per commit message."
+
+  dependabot-auto-approve-and-merge:
+    needs: quality_gate
+    uses: NHSDigital/eps-common-workflows/.github/workflows/dependabot-auto-approve-and-merge.yml@b933ef1bb3527fd7e7d5a7629fbd4e4dd94bf1b4
+    secrets:
+      AUTOMERGE_APP_ID: ${{ secrets.AUTOMERGE_APP_ID }}
+      AUTOMERGE_PEM: ${{ secrets.AUTOMERGE_PEM }}
+
   pr_title_format_check:
     uses: NHSDigital/eps-common-workflows/.github/workflows/pr_title_check.yml@b933ef1bb3527fd7e7d5a7629fbd4e4dd94bf1b4
 
   get_issue_number:
     runs-on: ubuntu-22.04
-    needs: quality_checks
     outputs:
       issue_number: ${{steps.get_issue_number.outputs.result}}
 
@@ -71,7 +150,11 @@ jobs:
           result-encoding: string
 
   package_code:
-    needs: [get_issue_number]
+    needs: [get_issue_number, quality_gate]
+    if: |
+      always() &&
+      ! contains(needs.*.result, 'failure') &&
+      ! contains(needs.*.result, 'cancelled')
     uses: ./.github/workflows/cdk_package_code.yml
     with:
       STACK_NAME: epsam-pr-${{needs.get_issue_number.outputs.issue_number}}
@@ -80,6 +163,10 @@ jobs:
 
   release_code:
     needs: [get_issue_number, package_code]
+    if: |
+      always() &&
+      ! contains(needs.*.result, 'failure') &&
+      ! contains(needs.*.result, 'cancelled')
     uses: ./.github/workflows/release_all_stacks.yml
     with:
       STACK_NAME: epsam-pr-${{needs.get_issue_number.outputs.issue_number}}

Makefile

@@ -48,6 +48,7 @@ lint-flake8:
 test:
 	cd packages/slackBotFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage poetry run python -m pytest
 	cd packages/syncKnowledgeBaseFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage poetry run python -m pytest
+	cd packages/preprocessingFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage poetry run python -m pytest
 	cd packages/bedrockLoggingConfigFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage poetry run python -m pytest
 
 clean:
@@ -108,6 +109,7 @@ cdk-synth: cdk-synth-pr cdk-synth-non-pr
 cdk-synth-non-pr:
 	mkdir -p .dependencies/slackBotFunction
 	mkdir -p .dependencies/syncKnowledgeBaseFunction
+	mkdir -p .dependencies/preprocessingFunction
 	mkdir -p .dependencies/bedrockLoggingConfigFunction
 	mkdir -p .local_config
 	STACK_NAME=epsam \
@@ -127,6 +129,7 @@ cdk-synth-non-pr:
 cdk-synth-pr:
 	mkdir -p .dependencies/slackBotFunction
 	mkdir -p .dependencies/syncKnowledgeBaseFunction
+	mkdir -p .dependencies/preprocessingFunction
 	mkdir -p .dependencies/bedrockLoggingConfigFunction
 	mkdir -p .local_config
 	STACK_NAME=epsam-pr-123 \
@@ -159,14 +162,14 @@ sync-docs:
 	./scripts/sync_docs.sh
 
 convert-docs:
-	poetry run python scripts/convert_docs_to_markdown.py
+	cd packages/preprocessingFunction && poetry run python -m app.cli
 
 convert-docs-file:
 	@if [ -z "$$FILE" ]; then \
 		echo "usage: FILE=your_doc.pdf make convert-docs-file"; \
 		exit 1; \
 	fi
-	poetry run python scripts/convert_docs_to_markdown.py --file "$$FILE"
+	cd packages/preprocessingFunction && poetry run python -m app.cli --file "$$FILE"
 
 
 compile:

packages/cdk/assets/s3-folders/processed/.gitkeep

@@ -51,7 +51,7 @@ const addSuppressions = (resources: Array<CfnResource>, rules: Array<string>): v
       resource.cfnOptions.metadata = {}
     }
     const existing = resource.cfnOptions.metadata.guard?.SuppressedRules || []
-    const combined = [...new Set([...existing, ...rules])]
+    const combined = Array.from(new Set([...existing, ...rules]))
     resource.cfnOptions.metadata.guard = {SuppressedRules: combined}
   })
 }
@@ -63,9 +63,10 @@ export const applyCfnGuardSuppressions = (stack: Stack): void => {
   // Suppress all cfn-guard checks for all Lambda functions (including implicit CDK-generated ones)
   const allLambdas = findResourcesByType(stack, "AWS::Lambda::Function")
   addSuppressions(allLambdas, ["LAMBDA_DLQ_CHECK", "LAMBDA_INSIDE_VPC", "LAMBDA_CONCURRENCY_CHECK"])
-  const permissionResources = findResourcesByPattern(stack, [
-    "ApiPermission.Test.EpsAssistMeStackApisEpsAssistApiGateway1E1CF19C.POST..slack.events",
-    "AllowBucketNotificationsToEpsAssistMeStackFunctionsSyncKnowledgeBaseFunctionepsamSyncKnowledgeBaseFunction94D011F3"
-  ])
-  addSuppressions(permissionResources, ["LAMBDA_FUNCTION_PUBLIC_ACCESS_PROHIBITED"])
+
+  const apiGatewayPermissions = findResourcesByPattern(stack, ["ApiPermission"])
+  addSuppressions(apiGatewayPermissions, ["LAMBDA_FUNCTION_PUBLIC_ACCESS_PROHIBITED"])
+
+  const s3NotificationPermissions = findResourcesByPattern(stack, ["AllowBucketNotifications"])
+  addSuppressions(s3NotificationPermissions, ["LAMBDA_FUNCTION_PUBLIC_ACCESS_PROHIBITED"])
 }

packages/cdk/assets/s3-folders/raw/.gitkeep

@@ -58,6 +58,7 @@ export class S3Bucket extends Construct {
       principals: [props.deploymentRole],
       actions: [
         "s3:Abort*",
+        "s3:DeleteObject",
         "s3:GetBucket*",
         "s3:GetObject*",
         "s3:List*",

packages/cdk/bin/utils/appUtils.ts

@@ -6,6 +6,7 @@ import {Function as LambdaFunction} from "aws-cdk-lib/aws-lambda"
 export interface S3LambdaNotificationProps {
   bucket: Bucket
   lambdaFunction: LambdaFunction
+  prefix?: string
 }
 
 export class S3LambdaNotification extends Construct {
@@ -18,18 +19,23 @@ export class S3LambdaNotification extends Construct {
     const supportedExtensions = [".pdf", ".txt", ".md", ".csv", ".doc", ".docx", ".xls", ".xlsx", ".html", ".json"]
 
     supportedExtensions.forEach(ext => {
+      const filter: {suffix: string; prefix?: string} = {suffix: ext}
+      if (props.prefix) {
+        filter.prefix = props.prefix
+      }
+
       // Handle all file creation/modification events
       props.bucket.addEventNotification(
         EventType.OBJECT_CREATED,
         lambdaDestination,
-        {suffix: ext}
+        filter
       )
 
       // Handle all file deletion events
       props.bucket.addEventNotification(
         EventType.OBJECT_REMOVED,
         lambdaDestination,
-        {suffix: ext}
+        filter
       )
     })
   }