CCM-12896: Generate Python Models

.gitignore

@@ -26,6 +26,19 @@ output
 /schemas
 .env
 
+# Python
+__pycache__/
+.Python
+.venv/
+venv/
+*.egg-info/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+coverage.xml
+
 # Coverage reports
 htmlcov/
 .coverage

Makefile

@@ -32,6 +32,7 @@ clean:: # Clean-up project resources (main) @Operations
 	$(MAKE) -C src/cloudevents clean
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean-output
+	$(MAKE) -C src/python-schema-generator clean
 	rm -f .version
 	npm run clean
 

package.json

@@ -42,8 +42,10 @@
     "react": "^19.0.0"
   },
   "scripts": {
+    "build-schemas": "make -C src/cloudevents/domains/digital-letters build-no-bundle publish-bundled-json",
     "clean": "npm run clean --workspaces --if-present",
-    "generate-dependencies": "make -C src/cloudevents/domains/digital-letters build-no-bundle publish-bundled-json && npm run generate-dependencies --workspaces --if-present",
+    "generate-dependencies": "npm run build-schemas && npm run generate-dependencies --workspaces --if-present && npm run generate-python-dependencies",
+    "generate-python-dependencies": "make -C src/python-schema-generator generate",
     "lint": "npm run lint --workspaces",
     "lint:fix": "npm run lint:fix --workspaces",
     "start": "npm run start --workspace frontend",
@@ -61,6 +63,7 @@
     "utils/sender-management",
     "src/cloudevents",
     "src/digital-letters-events",
+    "src/python-schema-generator",
     "src/typescript-schema-generator",
     "tests/playwright"
   ]

project.code-workspace

@@ -79,10 +79,10 @@
     },
     "terminal.integrated.scrollback": 10000,
     "jest.virtualFolders": [
-
       { "name": "key-generation", "rootPath": "lambdas/key-generation" },
       { "name": "mesh-poll", "rootPath": "lambdas/mesh-poll" },
       { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" },
+      { "name": "python-schema-generator", "rootPath": "src/python-schema-generator" },
       { "name": "ttl-create-lambda", "rootPath": "lambdas/ttl-create-lambda/" },
       { "name": "ttl-handle-expiry-lambda", "rootPath": "lambdas/ttl-handle-expiry-lambda" },
       { "name": "ttl-poll-lambda", "rootPath": "lambdas/ttl-poll-lambda" },

scripts/config/sonar-scanner.properties

@@ -4,12 +4,12 @@ sonar.host.url=https://sonarcloud.io
 sonar.qualitygate.wait=true
 sonar.sourceEncoding=UTF-8
 sonar.sources=.
-sonar.tests=tests/, src/asyncapigenerator/tests, src/cloudeventjekylldocs/tests, src/eventcatalogasyncapiimporter/tests, src/cloudevents/tools/builder/__tests__, src/cloudevents/tools/cache/__tests__, src/cloudevents/tools/generator/__tests__, lambdas/mesh-poll/src/__tests__, lambdas/ttl-create-lambda/src/__tests__, lambdas/ttl-poll-lambda/src/__tests__, utils/utils/src/__tests__, utils/sender-management/src/__tests__
+sonar.tests=tests/, src/asyncapigenerator/tests, src/cloudeventjekylldocs/tests, src/eventcatalogasyncapiimporter/tests, src/python-schema-generator/tests, src/cloudevents/tools/builder/__tests__, src/cloudevents/tools/cache/__tests__, src/cloudevents/tools/generator/__tests__, lambdas/mesh-poll/src/__tests__, lambdas/ttl-create-lambda/src/__tests__, lambdas/ttl-poll-lambda/src/__tests__, utils/utils/src/__tests__, utils/sender-management/src/__tests__
 sonar.test.inclusions=tests/**, src/**/tests/**, src/**/__tests__/**, lambdas/**/src/__tests__/**, utils/utils/src/__tests__/**, utils/sender-management/src/__tests__/**
 sonar.terraform.provider.aws.version=5.54.1
 sonar.cpd.exclusions=**.test.*
-sonar.coverage.exclusions=tests/**, src/**/tests/**, src/**/__tests__/**, **/*.dev.*, lambdas/**/src/__tests__/**, **/jest.config.ts, **/jest.config.cjs, scripts/**/*.*, docs/**/*.*, utils/utils/src/__tests__/**, src/asyncapigenerator/example_usage.py, src/asyncapigenerator/test_generator.py, src/eventcatalogasyncapiimporter/examples.py
+sonar.coverage.exclusions=tests/**, src/**/tests/**, src/**/__tests__/**, **/*.dev.*, lambdas/**/src/__tests__/**, **/jest.config.ts, **/jest.config.cjs, scripts/**/*.*, docs/**/*.*, utils/utils/src/__tests__/**, src/asyncapigenerator/example_usage.py, src/asyncapigenerator/test_generator.py, src/eventcatalogasyncapiimporter/examples.py, src/digital-letters-events/**
 
-sonar.python.coverage.reportPaths=src/asyncapigenerator/coverage.xml,src/cloudeventjekylldocs/coverage.xml,src/eventcatalogasyncapiimporter/coverage.xml
+sonar.python.coverage.reportPaths=src/asyncapigenerator/coverage.xml,src/cloudeventjekylldocs/coverage.xml,src/eventcatalogasyncapiimporter/coverage.xml,src/python-schema-generator/coverage.xml
 sonar.javascript.lcov.reportPaths=lcov.info,src/cloudevents/coverage/lcov.info
 sonar.typescript.lcov.reportPaths=lcov.info,src/cloudevents/coverage/lcov.info

scripts/config/vale/styles/config/vocabularies/words/accept.txt

@@ -69,6 +69,7 @@ Podman
 producedby
 projectRoot
 Protobuf
+Pydantic
 pylint
 Python
 quotingType

scripts/tests/unit.sh

@@ -34,6 +34,11 @@ echo "Setting up and running eventcatalogasyncapiimporter tests..."
 make -C ./src/eventcatalogasyncapiimporter install-dev
 make -C ./src/eventcatalogasyncapiimporter coverage  # Run with coverage to generate coverage.xml for SonarCloud
 
+# Python projects - python-schema-generator
+echo "Setting up and running python-schema-generator tests..."
+make -C ./src/python-schema-generator install-dev
+make -C ./src/python-schema-generator coverage  # Run with coverage to generate coverage.xml for SonarCloud
+
 # TypeScript/JavaScript projects (npm workspace)
 # Note: src/cloudevents is included in workspaces, so it will be tested here
 npm ci

src/digital-letters-events/.gitignore

@@ -1,2 +1,3 @@
 validators
 types
+models

src/digital-letters-events/README.md

@@ -1,16 +1,24 @@
 # digital-letters-events
 
+<!-- vale Vale.Avoid = NO -->
+<!-- vale Vale.Terms = NO -->
 This package contains the automatically-generated code that the
-[typescript-schema-generator](../typescript-schema-generator/) tool produces.
+[typescript-schema-generator](../typescript-schema-generator/) and the
+[python-schema-generator](../python-schema-generator/) tools produce.
+<!-- vale Vale.Avoid = YES -->
+<!-- vale Vale.Terms = YES -->
 
 The source files in this package should not be edited directly. If changes are
 required, update the schemas in the
 [src/cloudevents/domains](../cloudevents/domains) directory and use the
-`typescript-schema-generator` tool to regenerate them.
+`typescript-schema-generator` and `python-schema-generator` tools to regenerate
+them.
 
 ## Using this Package
 
-### Using Event Types
+### TypeScript
+
+#### Using Event Types
 
 The event types can be used by simply installing the
 `digital-letters-events` package as a dependency and then importing
@@ -51,7 +59,7 @@ const pdmResourceSubmittedEvent: PDMResourceSubmitted = {
   };
 ```
 
-### Using Event Validator Functions
+#### Using Event Validator Functions
 
 Validator functions for an event can be used by importing the default export
 from the relevant JS file in
@@ -74,3 +82,54 @@ if (isEventValid) {
 Note: You will need to make sure the
 [`allowJs`](https://www.typescriptlang.org/tsconfig/#allowJs) option is set in
 your package's `tsconfig.json` in order to import the JS files.
+
+### Python
+
+#### Using Event Models
+
+The Pydantic models can be used by installing the `digital-letters-events` package and importing the desired model:
+
+```python
+from digital_letters_events import PDMResourceSubmitted
+
+try:
+    # Validate and parse an event
+    event_data = {
+        "type": "uk.nhs.notify.digital.letters.pdm.resource.submitted.v1",
+        "source": "/nhs/england/notify/staging/dev-647563337/data-plane/digitalletters/pdm",
+        "dataschema": "https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-submitted-data.schema.json",
+        "specversion": "1.0",
+        "id": "0249e529-f947-4012-819e-b634eb71be79",
+        "subject": "customer/7ff8ed41-cd5f-20e4-ef4e-34f96d8cc8ac/75027ace-9b8c-bcfe-866e-6c24242cffc3/q58dnxk5e/4cbek805wwx/yiaw7bl0d/her/1ccb7eb8-c6fe-0a42-279a-2a0e48ff1ca9/zk",
+        "time": "2025-11-21T16:01:52.268Z",
+        "datacontenttype": "application/json",
+        "traceparent": "00-ee4790eb6821064c645406abe918b3da-3a4e6957ce2a15de-01",
+        "tracestate": "nisi quis",
+        "partitionkey": "customer-7ff8ed41",
+        "recordedtime": "2025-11-21T16:01:53.268Z",
+        "sampledrate": 1,
+        "sequence": "00000000000350773861",
+        "severitytext": "INFO",
+        "severitynumber": 2,
+        "dataclassification": "restricted",
+        "dataregulation": "ISO-27001",
+        "datacategory": "non-sensitive",
+        "data": {
+            "messageReference": "incididunt Ut aute laborum",
+            "senderId": "officia voluptate culpa Ut dolor",
+            "resourceId": "a2bcbb42-ab7e-42b6-88d6-74f8d3ca4a09",
+            "retryCount": 97_903_257,
+        },
+    }
+
+    # Create and validate the event
+    event = PDMResourceSubmitted(**event_data)
+
+    # Access validated fields
+    print(event.id)
+    print(event.type)
+    print(event.data.messageReference)
+except Exception as e:
+    print(e)
+    raise ValueError("Error processing event") from e
+```

src/digital-letters-events/digital_letters_events/__init__.py

@@ -0,0 +1,13 @@
+"""Digital Letters Events package.
+
+This package contains automatically-generated Pydantic v2 models for NHS Notify
+Digital Letters events. These models are generated from JSON schemas by the
+python-schema-generator tool.
+
+DO NOT EDIT: Files in this package are auto-generated. Any manual changes will
+be overwritten when the models are regenerated.
+"""
+
+from .models import *
+
+__all__ = ["models"]

src/digital-letters-events/requirements.txt

@@ -0,0 +1 @@
+pydantic>=2.0.0,<3.0.0

src/digital-letters-events/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="digital-letters-events",
+    version="0.1.0",
+    packages=find_packages(),
+)

src/python-schema-generator/.gitignore

@@ -0,0 +1 @@
+schemas

src/python-schema-generator/Makefile

@@ -0,0 +1,68 @@
+# Variables
+SCHEMA_SRC_DIR := ../../schemas/digital-letters/2025-10-draft/events
+OUTPUT_DIR := ../digital-letters-events/digital_letters_events/models
+SRC_DIR := src
+SCHEMAS_DIR := $(SRC_DIR)/schemas
+
+# Default target
+.PHONY: all clean generate install install-dev test coverage help
+
+all: generate
+
+# Install production dependencies
+install:
+	@echo "Installing Python production dependencies..."
+	@pip install -r requirements.txt
+	@echo "Production dependencies installed!"
+
+# Install development dependencies
+install-dev:
+	@echo "Installing development dependencies..."
+	@pip install -r requirements-dev.txt
+	@echo "Development dependencies installed!"
+
+# Generate Pydantic models from JSON schemas
+generate: install
+	@echo "Generating Pydantic models from JSON schemas..."
+	@mkdir -p $(OUTPUT_DIR)
+	@npm run merge
+	@python $(SRC_DIR)/generate_models.py \
+		--input-dir $(SCHEMAS_DIR) \
+		--output-dir $(OUTPUT_DIR)
+	@echo "Pydantic models generated in $(OUTPUT_DIR)/"
+
+# Run tests
+test: install-dev
+	@echo "Running tests..."
+	@cd ../.. && PYTHONPATH=src/python-schema-generator/src:$$PYTHONPATH pytest src/python-schema-generator/tests/
+
+# Generate coverage report
+coverage: install-dev
+	@echo "Generating coverage report..."
+	@cd ../.. && PYTHONPATH=src/python-schema-generator/src:$$PYTHONPATH pytest src/python-schema-generator/tests/ \
+		--cov=src/python-schema-generator \
+		--cov-config=src/python-schema-generator/pytest.ini \
+		--cov-report=html:src/python-schema-generator/htmlcov \
+		--cov-report=term-missing \
+		--cov-report=xml:src/python-schema-generator/coverage.xml \
+		--cov-branch
+	@echo "Coverage report generated in src/python-schema-generator/htmlcov/"
+
+# Clean output directory and generated files
+clean:
+	@echo "Cleaning generated models..."
+	@rm -rf $(OUTPUT_DIR) htmlcov .pytest_cache coverage.xml
+	@rm -rf $(SCHEMAS_DIR)
+	@echo "Clean complete!"
+
+# Show help
+help:
+	@echo "Available targets:"
+	@echo "  all          - Generate Pydantic models (default)"
+	@echo "  install      - Install production dependencies"
+	@echo "  install-dev  - Install development dependencies"
+	@echo "  generate     - Generate Pydantic models from JSON schemas"
+	@echo "  test         - Run tests"
+	@echo "  coverage     - Generate coverage report"
+	@echo "  clean        - Clean output directories"
+	@echo "  help         - Show this help message"

src/python-schema-generator/README.md

@@ -0,0 +1,110 @@
+<!-- vale Vale.Avoid = NO -->
+<!-- vale Vale.Terms = NO -->
+# python-schema-generator
+<!-- vale Vale.Avoid = YES -->
+<!-- vale Vale.Terms = YES -->
+
+This package provides a tool that generates Pydantic v2 models for the events
+that are defined in the JSON schemas that are built as part of this repository.
+The generated Pydantic models can be used to validate and parse NHS Notify
+Digital Letters events.
+
+## Setup
+
+Use `make install-dev` to install the necessary dependencies for this application.
+
+## Generating Models
+
+In order for this tool to function, you must first build the JSON schemas for
+the Digital Letters events. The simplest way to build these schemas is to run
+the `npm run generate-dependencies` command from the root of this repository.
+
+### Using the Generator
+
+Once the JSON schemas have been built, Pydantic models can be generated by running:
+
+```bash
+make generate
+```
+
+This will:
+
+- Read all JSON event schemas from `schemas/digital-letters/2025-10-draft/events/`
+- Generate Pydantic v2 models using `datamodel-code-generator`
+- Output the models to `../digital-letters-events/models/`
+
+### Configuration
+
+The generator can be configured through:
+
+- Command-line arguments
+- Environment variables
+- Configuration file (if needed)
+
+### Output Structure
+
+<!-- vale Vale.Terms = NO -->
+<!-- vale Vale.Avoid = NO -->
+Generated models are placed in the
+[digital-letters-events](../digital-letters-events/) package:
+<!-- vale Vale.Terms = YES -->
+<!-- vale Vale.Avoid = YES -->
+
+- `../digital-letters-events/models/` - Individual Pydantic model files
+- `../digital-letters-events/models/__init__.py` - Combined index for easy imports
+
+## Testing
+
+Run tests using:
+
+```bash
+make test
+```
+
+For coverage reporting:
+
+```bash
+make coverage
+```
+
+## Development
+
+### Prerequisites
+
+- Python 3.11+
+- pip or poetry for dependency management
+
+### Installing Development Dependencies
+
+```bash
+make install-dev
+```
+
+### Running Tests
+
+```bash
+make test
+```
+
+## Usage
+
+Once generated, the models can be used in your code as follows:
+
+```python
+from digital_letters_event_python import PrintLetterAvailable, CloudEventEnvelope
+
+# Validate incoming event
+event_data = CloudEventEnvelope(**raw_event)
+if event_data.type == "uk.nhs.notify.digital.letters.letter.available.v1":
+    letter_available = PrintLetterAvailable(**raw_event)
+    # Process the validated event
+```
+
+## Architecture
+
+The generator consists of:
+
+- `src/generate_models.py` - Main CLI script
+- `src/schema_processor.py` - Schema loading and processing
+- `src/model_generator.py` - Pydantic model generation logic
+- `src/file_utils.py` - File system utilities