pythonpath

zip exist

Use Map Cache

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -111,6 +111,8 @@ jobs:
       - name: Run unittest with redis_sync
         working-directory: redis_sync
         id: redis_sync
+        env:
+            PYTHONPATH: redis_sync/src:redis_sync/tests
         continue-on-error: true
         run: |
             poetry env use 3.11

filenameprocessor/src/clients.py

@@ -8,8 +8,7 @@
 import redis
 from boto3 import client as boto3_client, resource as boto3_resource
 
-# AWS Clients and Resources.
-REGION_NAME = "eu-west-2"
+REGION_NAME = os.getenv("AWS_REGION", "eu-west-2")
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 sqs_client = boto3_client("sqs", region_name=REGION_NAME)

filenameprocessor/src/file_name_processor.py

@@ -14,7 +14,6 @@
 from make_and_upload_ack_file import make_and_upload_the_ack_file
 from audit_table import upsert_audit_table, get_next_queued_file_details, ensure_file_is_not_a_duplicate
 from clients import logger
-from elasticache import upload_to_elasticache
 from logging_decorator import logging_decorator
 from supplier_permissions import validate_vaccine_type_permissions
 from errors import (
@@ -34,7 +33,7 @@
 @logging_decorator
 def handle_record(record) -> dict:
     """
-    Processes a single record based on whether it came from the 'data-sources' or 'config' bucket.
+    Processes a single record based on whether it came from the 'data-sources' bucket.
     Returns a dictionary containing information to be included in the logs.
     """
     try:
@@ -140,17 +139,6 @@ def handle_record(record) -> dict:
                 "supplier": supplier
             }
 
-    elif "config" in bucket_name:
-        try:
-            upload_to_elasticache(file_key, bucket_name)
-            logger.info("%s content successfully uploaded to cache", file_key)
-            message = "File content successfully uploaded to cache"
-            return {"statusCode": 200, "message": message, "file_key": file_key}
-        except Exception as error:  # pylint: disable=broad-except
-            logger.error("Error uploading to cache for file '%s': %s", file_key, error)
-            message = "Failed to upload file content to cache"
-            return {"statusCode": 500, "message": message, "file_key": file_key, "error": str(error)}
-
     else:
         try:
             vaccine_type, supplier = validate_file_key(file_key)

recordprocessor/src/clients.py

@@ -1,10 +1,13 @@
 """Initialise s3 and kinesis clients"""
 
 import logging
+import os
+
 from boto3 import client as boto3_client, resource as boto3_resource
 from botocore.config import Config
+from redis_cacher import RedisCacher
 
-REGION_NAME = "eu-west-2"
+REGION_NAME = os.getenv("AWS_REGION")
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 kinesis_client = boto3_client(
@@ -21,3 +24,7 @@
 logging.basicConfig(level="INFO")
 logger = logging.getLogger()
 logger.setLevel("INFO")
+
+REDIS_HOST = os.getenv("REDIS_HOST")
+REDIS_PORT = os.getenv("REDIS_PORT")
+redis_cacher = RedisCacher(REDIS_HOST, REDIS_PORT, logger)

recordprocessor/src/constants.py

@@ -84,3 +84,8 @@ class Urls:
     NHS_NUMBER = "https://fhir.nhs.uk/Id/nhs-number"
     NULL_FLAVOUR_CODES = "http://terminology.hl7.org/CodeSystem/v3-NullFlavor"
     VACCINATION_PROCEDURE = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure"
+
+
+class RedisCacheKeys:
+    PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+    DISEASE_MAPPING_FILE_KEY = "disease_mapping.json"

recordprocessor/src/mappings.py

@@ -2,7 +2,8 @@
 
 from enum import Enum
 from typing import Dict, List
-from constants import Urls
+from constants import Urls, RedisCacheKeys
+from redis_cacher import redis_cacher
 
 
 class Vaccine(Enum):
@@ -54,6 +55,8 @@ class DiseaseDisplayTerm(Enum):
     Vaccine.RSV: [Disease.RSV],
 }
 
+VACCINE_DISEASE_MAPPING = redis_cacher.get(RedisCacheKeys.DISEASE_MAPPING_FILE_KEY)
+
 
 def map_target_disease(vaccine: Vaccine) -> list:
     """Returns the target disease element for the given vaccine type using the vaccine_disease_mapping"""

recordprocessor/src/redis_cacher.py

@@ -0,0 +1,28 @@
+import json
+import redis
+
+
+class RedisCacher():
+    """ RedisCacher abstraction class to decouple application code
+    from direct use of Redis client.
+    Also centralised error handling & extensibility.
+    """
+
+    def __init__(self, redis_host, redis_port, logger):
+        try:
+            # Attempt to connect to Redis
+            self.redis_client = redis.StrictRedis(redis_host, redis_port, decode_responses=True)
+            # Check the connection with a PING command
+            if self.redis_client.ping():
+                logger.info("Successfully connected to Redis.")
+            else:
+                logger.error("Failed to connect to Redis.")
+        except Exception as e:
+            logger.exception(f"Connection to Redis failed: {e}")
+
+    def get(self, key: str) -> dict:
+        """Gets the value from Redis cache for the given key."""
+        value = self.redis_client.get(key)
+        if value is not None:
+            return json.loads(value)
+        return {}

redis_sync/Makefile

@@ -0,0 +1,14 @@
+
+test:
+	@PYTHONPATH=src python -m unittest
+
+coverage-run:
+	coverage run -m unittest discover -v
+
+coverage-report:
+	coverage report -m 
+
+coverage-html:
+	coverage html  
+
+.PHONY: build package