diff --git a/.gitignore b/.gitignore index 7879dae0e2..ce9a5b041f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,6 @@ openapi.json **/.vscode/**/* !**/.vscode/settings.json.default -devtools/volume/ **/.coverage **/test-results.xml allure-results/ diff --git a/README.md b/README.md index 49c1fc70c4..3f9695142b 100644 --- a/README.md +++ b/README.md @@ -14,39 +14,49 @@ See https://nhsd-confluence.digital.nhs.uk/display/APM/Glossary. **Note:** Each Lambda has its own `README.md` file for detailed documentation. For non-Lambda-specific folders, refer to `README.specification.md`. -### Lambdas - -| Folder | Description | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| `backend` | **Imms API** – Handles CRUD operations for the Immunisation API. | -| `delta_backend` | **Imms Sync** – Lambda function that reacts to events in the Immunisation database. | -| `ack_backend` | **Imms Batch** – Generates the final Business Acknowledgment (BUSACK) file from processed messages and writes it to the designated S3 location. | -| `filenameprocessor` | **Imms Batch** – Processes batch file names. | -| `mesh_processor` | **Imms Batch** – MESH-specific batch processing functionality. | -| `recordprocessor` | **Imms Batch** – Handles batch record processing. | -| `redis_sync` | **Imms Redis** – Handles sync s3 to REDIS. | -| `id_sync` | **Imms Redis** – Handles sync SQS to IEDS. | -| `shared` | **Imms Redis** – Not a lambda but Shared Code for lambdas | +### Lambdas (compute microservices) + +| Folder | Description | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ack_backend` | **Imms Batch** – Generates the final Business Acknowledgment (BUSACK) file from processed messages and writes it to the designated destination. | +| `backend` | **Imms API** – Handles CRUD operations for the Immunisation API. | +| `batch_processor_filter` | **Imms Batch** – Controller function that consumes from a queue and forwards file event for processing if filter conditions are met. | +| `delta_backend` | **Imms Sync** – Lambda function that reacts to events in the Immunisation Event Data Store (IEDS). | +| `filenameprocessor` | **Imms Batch** – Validates and processes new batch file events. | +| `id_sync` | **Imms Cross-cutting** – Handles [MNS](https://digital.nhs.uk/developer/api-catalogue/multicast-notification-service) NHS Number Change events and applies updates to affected records. | +| `mesh_processor` | **Imms Batch** – Triggered when new files are received via MESH. Moves them into the Imms Batch processing system. | +| `mns_subscription` | **Imms Cross-cutting** – Simple helper Lambda which sets up our required MNS subscription. Used in pipelines in DEV. | +| `recordforwarder` | **Imms Batch** – Consumes from the stream and applies the processed batch file row operations (CUD) to IEDS. | +| `recordprocessor` | **Imms Batch** – ECS Task - **not** a Lambda function - responsible for processing batch file rows and forwarding to the stream. | +| `redis_sync` | **Imms Cross-cutting** – Handles config file updates. E.g. disease mapping or permission files. | +| `shared` | **Imms Cross-cutting** – Shared `common` code that can be shared by other Lambda functions. This is not a standalone Lambda. | --- ### Pipelines -| Folder | Description | -| ------- | ------------------------------------------- | -| `azure` | Pipeline definition and orchestration code. | +Due to the timing of the project's inception, Azure pipelines have been inherited from the API Management team for deploying +the Apigee proxy and sandbox. The new way to manage and deploy said resources is [Proxygen](https://digital.nhs.uk/developer/api-catalogue/proxy-generator). + +In future a migration plan will be provided by the API Management team so we can move to the new process and use purely +GitHub Actions for our entire pipeline. + +| Folder | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `azure` | Pipeline definition and orchestration code concerned purely with the management and deployment of the Apigee proxy and sandbox. | +| `.github` | Pipeline definition and orchestration code for deploying and testing our backend project. | --- ### Infrastructure -| Folder | Description | -| ---------------------- | ------------------------------------------------------------------------------------------------------ | -| `account` | Base infrastructure components. | -| `grafana` | Terraform configuration for Grafana, built on top of core infra. | -| `instance` | Core Terraform infrastructure code. This is run in each PR and sets up lambdas associated with the PR. | -| `terraform_aws_backup` | Streamlined backup processing with AWS. | -| `proxies` | Apigee API proxy definitions. | +| Folder | Description | +| ---------------------- | ---------------------------------------------------------------- | +| `account` | Base infrastructure components deployed on a per account basis. | +| `grafana` | Terraform configuration for Grafana, built on top of core infra. | +| `instance` | Core Terraform app infrastructure. | +| `terraform_aws_backup` | Streamlined backup processing with AWS. | +| `proxies` | Apigee API proxy definitions. | --- @@ -60,13 +70,12 @@ See https://nhsd-confluence.digital.nhs.uk/display/APM/Glossary. ### Utilities -| Folder | Description | -| ---------------- | ------------------------------------------------------------- | -| `devtools` | Helper tools and utilities for local development | -| `quality_checks` | Dependencies for linting and formatting Python code | -| `scripts` | Standalone or reusable scripts for development and automation | -| `specification` | Specification files to document API and related definitions | -| `sandbox` | Simple sandbox API | +| Folder | Description | +| ------------------- | ------------------------------------------------------------- | +| `quality_checks` | Dependencies for linting and formatting Python code | +| `utilities/scripts` | Standalone or reusable scripts for development and automation | +| `specification` | Specification files to document API and related definitions | +| `sandbox` | Simple sandbox API | --- @@ -268,11 +277,3 @@ run a different set of tests. To do this: Please note that this project requires that all commits are verified using a GPG key. To set up a GPG key please follow the instructions specified here: https://docs.github.com/en/authentication/managing-commit-signature-verification - -## AWS configuration: Getting credentials for AWS federated user account - -In the 'Access keys' popup menu under AWS Access Portal: - -**NOTE** that AWS's 'Recommended' method of getting credentials **(AWS IAM Identity Center credentials)** will break mocking in unit tests; specifically any tests calling `dynamodb_client.create_table()` will fail with `botocore.errorfactory.ResourceInUseException: Table already exists`. - -Instead, use **Option 2 (Add a profile to your AWS credentials file)**. diff --git a/lambdas/delta_backend/README.md b/lambdas/delta_backend/README.md index a2ae4da482..51b9451699 100644 --- a/lambdas/delta_backend/README.md +++ b/lambdas/delta_backend/README.md @@ -8,9 +8,9 @@ This project is designed to convert FHIR-compliant JSON data (e.g., Immunization | File Name | What It Does | | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| **`converter.py`** | 🧠 The main brain — applies the schema, runs conversions, handles errors. | +| **`converter.py`** | The main brain — applies the schema, runs conversions, handles errors. | | **`conversion_layout.py`** | A plain Python list that defines which fields you want, and how they should be formatted (e.g. date format, renaming rules). | -| **`delta.py`** | Holds the function called by AWS Lambda | +| **`delta.py`** | Holds the function called by AWS Lambda. | | **`extractor.py`** | Tailored functionality to extract target fields from immunization record received by the delta handler. | | **`exception_messages.py`** | Holds reusable error messages and codes for clean debugging and validation feedback. | | **`log_firehose.py`** | Firehose logging functionality. | diff --git a/lambdas/id_sync/README.md b/lambdas/id_sync/README.md index cdb5da7891..9398bfa566 100644 --- a/lambdas/id_sync/README.md +++ b/lambdas/id_sync/README.md @@ -2,12 +2,12 @@ ## Overview -**Id Sync** is an AWS Lambda function designed to trigger from SQS. It receives a list of NHS Numbers and checks for changes in PDS. If change found, it updates the Events Table.. +**Id Sync** is an AWS Lambda function designed to trigger from SQS. It receives a list of NHS Numbers and checks for changes in PDS. If change found, it updates the Events Table. ## Features - **SQS Event Driven:** Automatically triggered by SQS event. -- **DynamoDb Integration:** Reviews contents of DynbamoDb Events table and updates where required.. +- **DynamoDb Integration:** Reviews contents of DynamoDb Events table and updates where required. - **Logging:** Provides detailed logging for monitoring and troubleshooting. ## How It Works diff --git a/lambdas/recordforwarder/README.md b/lambdas/recordforwarder/README.md index d537ac6954..6f09f33c9e 100644 --- a/lambdas/recordforwarder/README.md +++ b/lambdas/recordforwarder/README.md @@ -1,6 +1,14 @@ # About -This document describes the environment setup for the recordforwarder Lambda. +This document describes the purpose and environment setup for the recordforwarder Lambda. + +## Overview + +The Record Forwarder Lambda consumes from an AWS Kinesis Stream, and is responsible for applying updates to the Immunisation +Event Data Store. It will receive up to 100 records per batch. Each record will have been processed and formatted as a +FHIR Immunization by the ECS Record Processor in the prior batch step. + +The Record Forwarder Lambda will then execute the requested operation (Create, Update or Delete) on the given record. ## Setting up the recordforwarder lambda diff --git a/sonar-project.properties b/sonar-project.properties index 6e7479ae02..0c41e33275 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -3,7 +3,7 @@ sonar.projectKey=NHSDigital_immunisation-fhir-api sonar.organization=nhsdigital sonar.host.url=https://sonarcloud.io sonar.python.version=3.11 -sonar.exclusions=**/devtools/**,**/proxies/**,**/utilities/scripts/**,**/infrastructure/account/**,**/infrastructure/instance/**,**/infrastructure/grafana/**,**/terraform_aws_backup/**,**/tests/** +sonar.exclusions=**/proxies/**,**/utilities/scripts/**,**/infrastructure/account/**,**/infrastructure/instance/**,**/infrastructure/grafana/**,**/terraform_aws_backup/**,**/tests/** sonar.coverage.exclusions=lambdas/shared/src/common/models/batch_constants.py sonar.python.coverage.reportPaths=backend-coverage.xml,delta-coverage.xml,ack-lambda-coverage.xml,filenameprocessor-coverage.xml,recordforwarder-coverage.xml,recordprocessor-coverage.xml,mesh_processor-coverage.xml,redis_sync-coverage.xml,mns_subscription-coverage.xml,id_sync-coverage.xml,shared-coverage.xml,batchprocessorfilter-coverage.xml sonar.cpd.exclusions=**/Dockerfile diff --git a/utilities/devtools/.gitignore b/utilities/devtools/.gitignore deleted file mode 100644 index f9dd0525f7..0000000000 --- a/utilities/devtools/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -docker -*.iml -sample_data -test_* diff --git a/utilities/devtools/.terraform.lock.hcl b/utilities/devtools/.terraform.lock.hcl deleted file mode 100644 index f301f9b2c3..0000000000 --- a/utilities/devtools/.terraform.lock.hcl +++ /dev/null @@ -1,10 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "5.22.0" - constraints = "~> 5.0" - hashes = [ - "h1:XuU3tsGzElMt4Ti8SsM05pFllNMwSC4ScUxcfsOS140=", - ] -} diff --git a/utilities/devtools/Makefile b/utilities/devtools/Makefile deleted file mode 100644 index ff559dbc92..0000000000 --- a/utilities/devtools/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -test: - python -m unittest - -init: - terraform init -apply: - terraform apply -auto-approve - -delete-table: - aws dynamodb delete-table --no-cli-pager --table-name imms-default-imms-events --region us-east-1 --endpoint-url http://localhost:4566 || true -gen_data: - python generate_data.py -seed: delete-table apply gen_data - # seed database with the latest file that was added to the sample_data directory - python seed.py sample_data/$(shell ls -t sample_data | head -n1) - -localstack: - docker run -d --rm -it -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:latest - -.PHONY: test test-debug localstack apply init diff --git a/utilities/devtools/README.md b/utilities/devtools/README.md deleted file mode 100644 index 0c78e979e7..0000000000 --- a/utilities/devtools/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# Devtools - Localstack - -## About - -LocalStack is a fully functional local cloud service emulator that allows developers to run AWS services locally without connecting to the actual AWS cloud. It is especially useful for testing, development, and CI/CD pipelines where real AWS resources are not needed or too costly. - -## Setup: - -1. Install aws cli & awslocal (for localstack) for local testing of the infrastructure, might need to install unzip. AWSLocal is a wrapper for aws that simplifies interaction with LocalStack. - - ``` - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install - - pip install awscli-local - ``` - -2. Install terraform by following the instructions from `https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli` - -3. Navigate to `devtools`. -4. Create a virtual environment in devtools: `python -m venv .venv`. -5. Activate virtual environment: `source .venv/bin/activate`. You should see a `(.venv)` as a prefix in your terminal. -6. Upgrade pip in the new environment: `pip install --upgrade pip`. -7. Run `pip install -r requirements.txt` to install packages. -8. Run `make localstack` to setup localstack to run in docker. -9. Run `make init` or `terraform init`. If you get an error about failing to install providers, then remove the `.terraform.lock.hcl` and try again. -10. Run `make seed` to create a dynamodb table in localstack and add some data into it. -11. Run the following command to get a list of 10 items from dynamodb from localstack: - ``` - awslocal dynamodb query \ - --table-name imms-default-imms-events \ - --key-condition-expression "PK = :pk" \ - --expression-attribute-values '{":pk": {"S": "Immunization#e3e70682-c209-4cac-629f-6fbed82c07cd"}}'. - ``` -12. If you want to delete the table run `terraform apply -destroy` - -## Interacting with localstack - -The idea with localstack in regards to our project is to have a dynamodb table or an s3 bucket to interact with. We can't setup all the infrastructure on localstack because of the high complexity and lack of certain features such as AWS networking and IAM enforcement. - -1. Check if localstack is running in docker by calling: `docker ps`. The output should display a running container with the image localstack/localstack -2. Ensure that you have a dynamodb table set up and some data inside the table provided by the `make seed` command. -3. You can install dynamodb-admin which is npm web app that connects to localstack, to set it up: - - ``` - npm install -g dynamodb-admin - - export DYNAMO_ENDPOINT=http://localhost:4566 - export AWS_REGION=us-east-1 - - dynamodb-admin - - # Navigate to the url provided, typically: http://localhost:8001/ where you should see the table imms-default-imms-events - ``` - -4. To interact with the code there are 2 options. You can either persist the application lambda via docker or you can directly run / debug the code directly in vscode. Please note some modifications are needed to configure the code to run successfuly. - - 4.1 To run it via vs code, we can try the get_imms_handler, but first we should ensure that the request is correct so ensure that `event` has the folowing details. Note: that we are trying to retrieve the following immunisation record form the sample data `e3e70682-c209-4cac-629f-6fbed82c07cd` hence why we hardcoded the `VaccineTypePermissions`. - - ``` - event = { - "pathParameters": {"id": args.id}, - "headers": { - "Content-Type": "application/x-www-form-urlencoded", - "AuthenticationType": "ApplicationRestricted", - "Permissions": (",".join([Permission.READ])), - "VaccineTypePermissions": "covidcode2:read" - }, - } - ``` - - 4.2 If you want to run it via docker make the following changes in the lambda.Dockerfile: - - Set the dynamo db table name env variable to `imms-default-imms-events` - - Add `ENV DYNAMODB_TABLE_NAME=imms-default-imms-events` into the base section of the file - - Add the following line at the end in the lambda.Dockerfile: `CMD ["get_status_handler.get_status_handler"]` - - Test by sending a request via Postman to `http://localhost:8080/2015-03-31/functions/function/invocations` and add the event data into the body section. - -## Access Pattern - -We receive Immunisation Events in json format via our API. Each message contains inlined resources i.e. it doesn't -contain a reference/link to a preexisting resource. For example, a vaccination event contains the full patient resource -embedded in it. This means our backend doesn't assume Patient as a separate resource but rather, part of the message -itself. This is the same for other resources included in the resource like address, location etc. - -- **Creating an event:** Add the entire message in an attribute so, it can be retrieved. This attribute has the entire - original message with no changes. The event-id must be contained in the message itself. Our backend won't create the - id. **This is our main index.** and it doesn't contain any sort-key -- **Retrieve an event:** The simplest form is by id; `GET /event/{id}`. This access pattern should always result in - either one resource or none -- **Search:** This pattern can be broken down into two main categories. Queries that retrieve events with a known - patient and, queries that retrieve events with particular set of search criteria. - -### Patient - -One index is dedicated to search patient. This will satisfy -the `/event?NhsNumber=1234567,dateOfBirth=01/01/1970,diseaseTypeFilter=covid|flu|mmr|hpv|polio` endpoint. -The `NhsNumber` is our PK and the SK has `##` format. \*\*This means, in order to -filter based on `diseaseType`, `dateOfBirth` must be known. We can filter based on only `nhsNumber` and `diseaseType` -but that requires an attribute filter. - -**Q:** Do we need to retrieve events based on only NHS number and Disease Type? i.e. is this a valid -request? `GET /event?NhsNumber=1234567,diseaseTypeFilter=covid` -**Q:** What is LocalPatient? In our sample data we have both ID and System values, but we don't have any access pattern -for it. - -### Vaccination - -The provided object relational model, has a few highlighted fields related to vaccination but, we don't have any search -criteria for them. We can create one or more indices to address different search requests but, we need to know in -advance what they are. - -For example, one access pattern can be `PK: DiseaseType` and `SK: ##`. This will -be similar to the patient access pattern. - -**Q:** What are search criteria for vaccination related fields? - -## Field mappings - -Given the relational model (below image) and `sample_event.json` below is our field mappings for highlighted fields: -![img](img/relational-model.png) - -- `UNIQUE_ID -> $["identifier"][0]["value"]` -- `UNIQUE_ID_URI -> $["identifier"][0]["system"]` -- `ACTION_FLAG -> ?????` -- `VACCINATION_PROCEDURE_CODE -> $["extension"][0]["valueCodeableConcept"]["coding"][0]["code"]` -- `VACCINATION_PRODUCT_CODE -> $["vaccineCode"]["coding"][0]["code"]` -- `DISEASE_TYPE -> $["protocolApplied"][0]["targetDisease"][0]["coding"][0]["code"]` -- `LOCAL_PATIENT_ID -> $["contained"][0]["item"][3]["answer"][0]["valueCoding"]["code"]` -- `LOCAL_PATIENT_TYPE_URI -> $["contained"][0]["item"][3]["answer"][0]["valueCoding"]["system"]` - -TODO: write the rest of the mappings. - -**Q**: What does this -mean: [[first point in Overview]](https://nhsd-confluence.digital.nhs.uk/display/Vacc/Immunisation+FHIR+API+-+IEDS+Data+Model) - -> This data model must include backwards compatibility with the legacy CSV process to account for business continuity - -## Error Scenarios - -We assume all error responses will be of type `OperationOutcome`. We can break down scenarios in three categories. One -group is errors that are generated and dealt with in the Proxy before reaching to our backend. -Authentication/authorisation errors belong to this group. - -A second group is related to both fine and coarse validation errors. - -- **Q:** What is expected in the response message? A detailed diagnostics of the validation or just a generic error? - -The third group is anything related to the backend itself. `404` and, catch-all exceptions in the source code (like -malformed json for example) diff --git a/utilities/devtools/__init__.py b/utilities/devtools/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/utilities/devtools/dynamodb.tf b/utilities/devtools/dynamodb.tf deleted file mode 100644 index aca0b4ae8d..0000000000 --- a/utilities/devtools/dynamodb.tf +++ /dev/null @@ -1,63 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5" - } - } - backend "local" { - path = ".terraform/local.tfstate" - } -} - -provider "aws" { - region = "us-east-1" - skip_region_validation = true - - endpoints { - sts = "http://localhost:4566" - dynamodb = "http://localhost:4566" - } -} - -locals { - short_prefix = "imms-default" -} - -resource "aws_dynamodb_table" "events-dynamodb-table" { - name = "${local.short_prefix}-imms-events" - billing_mode = "PAY_PER_REQUEST" - hash_key = "PK" - stream_enabled = true - stream_view_type = "NEW_IMAGE" - - attribute { - name = "PK" - type = "S" - } - attribute { - name = "PatientPK" - type = "S" - } - attribute { - name = "PatientSK" - type = "S" - } - attribute { - name = "IdentifierPK" - type = "S" - } - - global_secondary_index { - name = "PatientGSI" - hash_key = "PatientPK" - range_key = "PatientSK" - projection_type = "ALL" - } - - global_secondary_index { - name = "IdentifierGSI" - hash_key = "IdentifierPK" - projection_type = "ALL" - } -} diff --git a/utilities/devtools/generate_data.py b/utilities/devtools/generate_data.py deleted file mode 100644 index 25562aab07..0000000000 --- a/utilities/devtools/generate_data.py +++ /dev/null @@ -1,93 +0,0 @@ -import copy -import json -import os -import random -import uuid -from datetime import datetime - -# generate reproducible random UUIDs -rd = random.Random() -rd.seed(0) - - -def make_rand_id(): - return uuid.UUID(int=rd.getrandbits(128)) - - -patient_pool = [ - {"nhs_number": "9999999999"}, - {"nhs_number": "1111111111"}, - {"nhs_number": "1212233445"}, -] -suppliers = [ - "https://supplierABC/ODSCode145", - "https://supplierSDF/ODSCode123", - "https://supplierXYZ/ODSCode735", -] -disease_type = ["covid", "flu", "mmr", "hpv", "polio"] -vaccine_code = [ - "covidCode1", - "covidCode2", - "fluCode1", - "fluCode2", - "fluCode3", - "mmrCode1", - "mmrCode2", - "hpvCode1", - "polioCode1", -] -vaccine_procedure = [ - "vac_procedure-oral", - "vac_procedure-injection", - "vac_procedure-123", -] -local_patient_pool = [ - {"code": "ACME-Patient12345", "system": "https://supplierABC/identifiers/patient"}, - {"code": "ACME-Patient23455", "system": "https://supplierCSB/identifiers/patient"}, - {"code": "ACME-Patient35623", "system": "https://supplierXYZ/identifiers/patient"}, -] -action_flag = ["flagA", "flagB"] - - -def pick_rand(pool): - idx = random.randint(0, len(pool) - 1) - return pool[idx] - - -def generate_immunization(num): - with open("sample_imms.json", "r") as template: - imms = json.loads(template.read()) - - all_imms = [] - for _ in range(num): - _imms = copy.deepcopy(imms) - # ID - _imms["id"] = str(make_rand_id()) - _imms["identifier"][0]["system"] = pick_rand(suppliers) - _imms["identifier"][0]["value"] = str(make_rand_id()) - # Patient - patient = pick_rand(patient_pool) - _imms["patient"]["identifier"]["value"] = patient["nhs_number"] - # Vaccination - _imms["protocolApplied"][0]["targetDisease"][0]["coding"][0]["code"] = pick_rand(vaccine_code) - _imms["vaccineCode"]["coding"][0]["code"] = pick_rand(vaccine_code) - - all_imms.append(_imms) - - return all_imms - - -def write(_data, resource_type): - path = "sample_data" - if not os.path.exists(path): - os.makedirs(path) - - name = f"{datetime.today().strftime('%Y-%m-%dT%H:%M:%S')}_{resource_type}-{len(_data)}.json" - with open(f"{path}/{name}", "w") as sample_file: - json_events = json.dumps(_data, indent=2) - sample_file.write(json_events) - - -if __name__ == "__main__": - imms = generate_immunization(30) - write(imms, resource_type="immunization") diff --git a/utilities/devtools/img/relational-model.png b/utilities/devtools/img/relational-model.png deleted file mode 100644 index 8659d55141..0000000000 Binary files a/utilities/devtools/img/relational-model.png and /dev/null differ diff --git a/utilities/devtools/localstack-compose.yml b/utilities/devtools/localstack-compose.yml deleted file mode 100644 index 5235e24e87..0000000000 --- a/utilities/devtools/localstack-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: "3.8" - -services: - localstack: - container_name: "localstack_main" - image: localstack/localstack:latest - environment: - - SERVICES=s3,dynamodb,sts - - DEFAULT_REGION=eu-west-2 - - DEBUG=1 - - PERSISTENCE=1 - ports: - - "4566:4566" - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" diff --git a/utilities/devtools/pytest.ini b/utilities/devtools/pytest.ini deleted file mode 100644 index 0da43c35a4..0000000000 --- a/utilities/devtools/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -;python_files = test_*.py -norecursedirs = .venv .eggs docker sample_data build dist utils -markers = - query: query - debug: debug diff --git a/utilities/devtools/requirements.txt b/utilities/devtools/requirements.txt deleted file mode 100644 index 7ae26195bc..0000000000 --- a/utilities/devtools/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto3 -pytest diff --git a/utilities/devtools/sample_event.json b/utilities/devtools/sample_event.json deleted file mode 100644 index 041b48ad27..0000000000 --- a/utilities/devtools/sample_event.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "resourceType": "Immunization", - "contained": [ - { - "resourceType": "QuestionnaireResponse", - "questionnaire": "Questionnaire/1", - "status": "completed", - "item": [ - { - "linkId": "SiteCode", - "answer": [ - { - "valueCoding": { - "system": "snomed", - "code": "M242ND" - } - } - ] - }, - { - "linkId": "SiteName", - "answer": [ - { - "valueCoding": { - "code": "dummy" - } - } - ] - }, - { - "linkId": "NhsNumberStatus", - "answer": [ - { - "valueCoding": { - "code": "snomed", - "display": "test description" - } - } - ] - }, - { - "linkId": "LocalPatient", - "answer": [ - { - "valueCoding": { - "system": "https://supplierABC/identifiers/patient", - "code": "ACME-patient123456" - } - } - ] - }, - { - "linkId": "Consent", - "answer": [ - { - "valueCoding": { - "code": "snomed", - "display": "free text" - } - } - ] - }, - { - "linkId": "CareSetting", - "answer": [ - { - "valueCoding": { - "code": "1127531000000102", - "display": "SNOMED-CT Term description Community health services (qualifier value)" - } - } - ] - }, - { - "linkId": "IpAddress", - "answer": [ - { - "valueCoding": { - "code": "1.0.0.0" - } - } - ] - }, - { - "linkId": "UserId", - "answer": [ - { - "valueCoding": { - "code": "test123" - } - } - ] - }, - { - "linkId": "UserName", - "answer": [ - { - "valueCoding": { - "code": "test" - } - } - ] - }, - { - "linkId": "UserEmail", - "answer": [ - { - "valueCoding": { - "code": "test@gmail.com" - } - } - ] - }, - { - "linkId": "SubmittedTimeStamp", - "answer": [ - { - "valueCoding": { - "code": "2020-12-14T10:08:15" - } - } - ] - }, - { - "linkId": "ReduceValidation", - "answer": [ - { - "valueCoding": { - "code": "TRUE", - "display": "test" - } - } - ] - } - ] - } - ], - "extension": [ - { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "snomed", - "display": "snomed" - } - ] - } - }, - { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationSituation", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "snomed", - "display": "snomed" - } - ] - } - } - ], - "identifier": [ - { - "system": "https://supplierABC/ODSCode", - "value": "e045626e-4dc5-4df3-bc35-da25263f901e" - } - ], - "status": "completed", - "statusReason": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "snomed", - "display": "snomed" - } - ] - }, - "vaccineCode": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "snomed", - "display": "snomed" - } - ] - }, - "lotNumber": "AAJN11K", - "expirationDate": "2020-05-06", - "patient": { - "resourceType": "Patient", - "identifier": [ - { - "system": "https//fhir.nhs.uk/Id/nhs-number", - "value": "1234567891" - } - ], - "name": [ - { - "family": "test", - "given": ["test"] - } - ], - "gender": "1", - "birthDate": "1999-10-03", - "address": [ - { - "postalCode": "LS1 5HT" - } - ] - }, - "occurrenceDateTime": "2020-12-14T10:08:15+00:00", - "primarySource": true, - "location": { - "identifier": { - "system": "https://fhir.nhs.uk/Id/ods-organization-code", - "value": "B0C4P" - }, - "resourceType": "Location" - }, - "site": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "LA", - "display": "left arm" - } - ] - }, - "route": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "IM", - "display": "injection, intramuscular" - } - ] - }, - "doseQuantity": { - "value": 5, - "unit": "mg", - "system": "http://unitsofmeasure.org", - "code": "snomed" - }, - "protocolApplied": [ - { - "targetDisease": [ - { - "coding": [ - { - "code": "40468003" - } - ] - } - ], - "doseNumber": "5" - } - ], - "reportOrigin": { - "text": "sample" - }, - "reasonCode": [ - { - "coding": [ - { - "code": "snomed", - "display": "test" - } - ] - } - ], - "recorded": "2010-05-06", - "manufacturer": { - "resourceType": "Organization", - "name": "org" - }, - "performer": { - "actor": { - "resourceType": "Practitioner", - "identifier": [ - { - "type": { - "coding": [ - { - "display": "GP" - } - ] - }, - "system": "https://fhir.hl7.org.uk/Id/gmc-number", - "value": "OP" - } - ], - "name": [ - { - "family": "test", - "given": ["test"] - } - ] - } - } -} diff --git a/utilities/devtools/sample_imms.json b/utilities/devtools/sample_imms.json deleted file mode 100644 index 7ede1d5ac6..0000000000 --- a/utilities/devtools/sample_imms.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "id": "d11c69d8-7a50-4a54-a848-7648121e9953", - "resourceType": "Immunization", - "extension": [ - { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "1324681000000101", - "display": "Administration of first dose of severe acute respiratory syndrome coronavirus 2 vaccine (procedure)" - } - ] - } - } - ], - "identifier": [ - { - "use": "secondary", - "system": "https://supplierABC/identifiers/vacc", - "value": "1324761000000100" - } - ], - "status": "completed", - "vaccineCode": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "39114911000001105", - "display": "COVID-19 Vaccine AstraZeneca (ChAdOx1 S [recombinant]) 5x10,000,000,000 viral particles/0.5ml dose solution for injection multidose vials (AstraZeneca UK Ltd) (product)" - } - ] - }, - "patient": { - "reference": "urn:uuid:124fcb63-669c-4a3c-af2b-caf55de167ec", - "type": "Patient", - "identifier": { - "system": "https://fhir.nhs.uk/Id/nhs-number", - "value": "9000000009" - } - }, - "occurrenceDateTime": "2020-12-10T13:00:08.476000+00:00", - "recorded": "2020-12-10T00:00:00+00:00", - "primarySource": true, - "manufacturer": { - "display": "AstraZeneca Ltd" - }, - "lotNumber": "R04X", - "expirationDate": "2021-04-29", - "site": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "368208006", - "display": "Left upper arm structure (body structure)" - } - ] - }, - "route": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "78421000", - "display": "Intramuscular route (qualifier value)" - } - ] - }, - "doseQuantity": { - "system": "http://snomed.info/sct", - "value": 1, - "unit": "pre-filled disposable injection", - "code": "3318611000001103" - }, - "performer": [ - { - "actor": { - "type": "Organization", - "identifier": { - "system": "https://fhir.nhs.uk/Id/ods-organization-code", - "value": "RX809" - }, - "display": "TEST-SITE" - } - } - ], - "reasonCode": [ - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "443684005", - "display": "Disease outbreak (event)" - } - ] - } - ], - "protocolApplied": [ - { - "doseNumberPositiveInt": 1, - "targetDisease": [ - { - "coding": [ - { - "code": "443684005", - "display": "Disease outbreak (event)" - } - ] - } - ] - } - ], - "location": { - "identifier": { - "system": "urn:iso:std:iso:3166", - "value": "GB" - } - } -} diff --git a/utilities/devtools/seed.py b/utilities/devtools/seed.py deleted file mode 100644 index 29c1899cfe..0000000000 --- a/utilities/devtools/seed.py +++ /dev/null @@ -1,66 +0,0 @@ -import json -import sys - -import boto3 - -sample_file = "sample_data/2023-11-29T19:04:37_immunisation-30.json" - -dynamodb_url = "http://localhost:4566" -table_name = "imms-default-imms-events" - - -class DynamoTable: - def __init__(self, endpoint_url, _table_name): - db = boto3.resource("dynamodb", endpoint_url=endpoint_url, region_name="us-east-1") - self.table = db.Table(_table_name) - - def create_immunization(self, immunization): - # When seeding, we preserve the original ID, instead of creating new one - new_id = immunization["id"] - patient_id = immunization["patient"]["identifier"]["value"] - disease_type = immunization["protocolApplied"][0]["targetDisease"][0]["coding"][0]["code"] - - patient_sk = f"{disease_type}#{new_id}" - - response = self.table.put_item( - Item={ - "PK": self._make_immunization_pk(new_id), - "Resource": json.dumps(immunization), - "PatientPK": self._make_patient_pk(patient_id), - "PatientSK": patient_sk, - "Version": 1, - } - ) - - if response["ResponseMetadata"]["HTTPStatusCode"] == 200: - return immunization - else: - raise Exception("Non-200 response from dynamodb") - - @staticmethod - def _make_immunization_pk(_id: str): - return f"Immunization#{_id}" - - @staticmethod - def _make_patient_pk(_id: str): - return f"Patient#{_id}" - - -def seed_immunization(table, _sample_file): - with open(_sample_file, "r") as raw_data: - imms_list = json.loads(raw_data.read()) - - for imms in imms_list: - table.create_immunization(imms) - - print(f"{len(imms_list)} resources added successfully") - - -if __name__ == "__main__": - _table = DynamoTable(dynamodb_url, table_name) - - seed_file = sample_file - if len(sys.argv) > 1: - seed_file = sys.argv[1] - - seed_immunization(_table, seed_file)