CCM-13278: Create Python Utility library

simonlabarere · simonlabarere · commit 98792ed31538 · 2026-01-22T10:02:36.000Z
diff --git a/lambdas/mesh-acknowledge/mesh_acknowledge/__tests__/test_handler.py b/lambdas/mesh-acknowledge/mesh_acknowledge/__tests__/test_handler.py
@@ -4,7 +4,7 @@
 from unittest.mock import Mock, MagicMock, patch, call
 
 import pytest
-from event_publisher import log
+from dl_utils import log
 from mesh_acknowledge.handler import handler
 
 
diff --git a/lambdas/mesh-acknowledge/mesh_acknowledge/config.py b/lambdas/mesh-acknowledge/mesh_acknowledge/config.py
@@ -1,7 +1,7 @@
 """
 Module for configuring MESH Acknowledger application
 """
-from event_publisher import BaseMeshConfig
+from dl_utils import BaseMeshConfig
 
 _REQUIRED_ENV_VAR_MAP = {
     "ssm_prefix": "SSM_PREFIX",
diff --git a/lambdas/mesh-acknowledge/mesh_acknowledge/handler.py b/lambdas/mesh-acknowledge/mesh_acknowledge/handler.py
@@ -3,7 +3,8 @@
 from typing import Dict, Any
 
 from boto3 import client
-from event_publisher import EventPublisher, log
+from event_publisher import EventPublisher
+from dl_utils import log
 from sender_management import SenderLookup
 from .acknowledger import MeshAcknowledger
 from .config import Config
diff --git a/lambdas/mesh-acknowledge/requirements.txt b/lambdas/mesh-acknowledge/requirements.txt
@@ -8,3 +8,4 @@ structlog>=21.5.0
 -e ../../utils/metric-publishers
 -e ../../utils/sender-management
 -e ../../utils/py-mock-mesh
+-e ../../utils/py-utils
diff --git a/lambdas/mesh-download/mesh_download/config.py b/lambdas/mesh-download/mesh_download/config.py
@@ -1,7 +1,7 @@
 """
 Module for configuring MESH Download application
 """
-from event_publisher import BaseMeshConfig, log
+from dl_utils import BaseMeshConfig, log
 from metric_publishers.metric_client import Metric
 
 
diff --git a/lambdas/mesh-download/requirements.txt b/lambdas/mesh-download/requirements.txt
@@ -10,4 +10,5 @@ requests>=2.32.0
 pyopenssl>=24.2.1
 -e ../../utils/event-publisher-py
 -e ../../utils/py-mock-mesh
+-e ../../utils/py-utils
 -e ../../utils/metric-publishers
diff --git a/lambdas/mesh-poll/mesh_poll/config.py b/lambdas/mesh-poll/mesh_poll/config.py
@@ -1,7 +1,7 @@
 """
 Module for configuring Mesh Poll application
 """
-from event_publisher import BaseMeshConfig, log
+from dl_utils import BaseMeshConfig, log
 from metric_publishers.metric_client import Metric
 
 
diff --git a/lambdas/mesh-poll/requirements.txt b/lambdas/mesh-poll/requirements.txt
@@ -11,4 +11,5 @@ pydantic>=2.0.0
 -e ../../utils/metric-publishers
 -e ../../utils/event-publisher-py
 -e ../../utils/py-mock-mesh
+-e ../../utils/py-utils
 -e ../../utils/sender-management
diff --git a/utils/event-publisher-py/event_publisher/__init__.py b/utils/event-publisher-py/event_publisher/__init__.py
@@ -7,20 +7,8 @@
 
 from .event_publisher import EventPublisher
 from . import models
-from .mesh_config import (
-    BaseMeshConfig,
-    InvalidMeshEndpointError,
-    InvalidEnvironmentVariableError,
-    store_file,
-    log
-)
 
 __all__ = [
     'EventPublisher',
-    'models',
-    'BaseMeshConfig',
-    'InvalidMeshEndpointError',
-    'InvalidEnvironmentVariableError',
-    'store_file',
-    'log'
+    'models'
 ]
diff --git a/utils/event-publisher-py/requirements.txt b/utils/event-publisher-py/requirements.txt
@@ -1,7 +1,5 @@
 boto3>=1.40.70
 pydantic>=2.0.0
 structlog>=21.5.0
-mesh-client>=3.2.3
 pyopenssl>=24.0.0
 -e ../metric-publishers
--e ../py-mock-mesh
diff --git a/utils/py-utils/Makefile b/utils/py-utils/Makefile
@@ -0,0 +1,24 @@
+.PHONY: install install-dev test coverage clean
+
+install:
+	pip install -r requirements.txt
+
+install-dev: install
+	pip install -r requirements-dev.txt
+
+test:
+	cd ../.. && PYTHONPATH=utils/py-utils:$$PYTHONPATH pytest utils/py-utils/dl_utils/__tests__/ -v
+
+coverage:
+	cd ../.. && PYTHONPATH=utils/py-utils:$$PYTHONPATH pytest utils/py-utils/dl_utils/__tests__/ \
+		--cov=utils/py-utils/dl_utils \
+		--cov-config=utils/py-utils/pytest.ini \
+		--cov-report=html:utils/py-utils/htmlcov \
+		--cov-report=term-missing \
+		--cov-report=xml:utils/py-utils/coverage.xml \
+		--cov-branch
+
+clean:
+	rm -rf dist/ .coverage htmlcov/ .pytest_cache/ coverage.xml
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -type f -name '*.pyc' -delete
diff --git a/utils/py-utils/dl_utils/__init__.py b/utils/py-utils/dl_utils/__init__.py
@@ -0,0 +1,21 @@
+"""
+Utility library for Python projects.
+
+"""
+
+from .mesh_config import (
+    BaseMeshConfig,
+    InvalidMeshEndpointError,
+    InvalidEnvironmentVariableError,
+)
+
+from .log_config import log
+from .store_file import store_file
+
+__all__ = [
+    'BaseMeshConfig',
+    'InvalidMeshEndpointError',
+    'InvalidEnvironmentVariableError',
+    'store_file',
+    'log'
+]
diff --git a/utils/py-utils/dl_utils/__tests__/test_mesh_config.py b/utils/py-utils/dl_utils/__tests__/test_mesh_config.py
@@ -0,0 +1,4 @@
+
+class TestMeshConfig:
+    def test_placeholder(self):
+        assert True
diff --git a/utils/py-utils/dl_utils/log_config.py b/utils/py-utils/dl_utils/log_config.py
@@ -0,0 +1,4 @@
+import structlog
+
+structlog.configure(processors=[structlog.processors.JSONRenderer()])
+log = structlog.get_logger()
diff --git a/utils/py-utils/dl_utils/mesh_config.py b/utils/py-utils/dl_utils/mesh_config.py
@@ -3,15 +3,11 @@
 """
 import json
 import os
-import tempfile
 import boto3
-import structlog
 import mesh_client
 from py_mock_mesh.mesh_client import MockMeshClient
 from metric_publishers.certificate_monitor import report_expiry_time
-
-structlog.configure(processors=[structlog.processors.JSONRenderer()])
-log = structlog.get_logger()
+from .log_config import log
 
 
 class InvalidMeshEndpointError(Exception):
@@ -26,16 +22,6 @@ class InvalidEnvironmentVariableError(Exception):
     """
 
 
-def store_file(content):
-    """
-    Writes a temp file and returns the name
-    """
-    with tempfile.NamedTemporaryFile(delete=False) as file:
-        file.write(content)
-        file.close()
-        return file.name
-
-
 class BaseMeshConfig:  # pylint: disable=too-many-instance-attributes
     """
     Base configuration class for MESH client applications.
@@ -50,8 +36,7 @@ def __init__(self, ssm=None, s3_client=None):
         Initialize base MESH configuration.
         """
         self.ssm = ssm if ssm is not None else boto3.client('ssm')
-        self.s3_client = s3_client if s3_client is not None else boto3.client(
-            's3')
+        self.s3_client = s3_client if s3_client is not None else boto3.client('s3')
 
         # MESH connection attributes
         self.mesh_endpoint = None
@@ -63,7 +48,7 @@ def __init__(self, ssm=None, s3_client=None):
         self.mesh_client = None
 
         # Common configuration attributes
-        self.ssm_prefix = None
+        self.ssm_mesh_prefix = None
         self.environment = None
         self.certificate_expiry_metric_name = None
         self.certificate_expiry_metric_namespace = None
@@ -102,15 +87,14 @@ def _load_optional_env_vars(self):
                 value = os.environ[key]
                 if attr == "use_mesh_mock":
                     # Convert string to boolean
-                    setattr(self, attr, value.lower()
-                            in ('true', '1', 'yes', 'on'))
+                    setattr(self, attr, value.lower() in ('true', '1', 'yes', 'on'))
                 else:
                     setattr(self, attr, value)
 
     def __enter__(self):
         # Load MESH configuration from SSM
         ssm_response = self.ssm.get_parameter(
-            Name=self.ssm_prefix + '/mesh/config',
+            Name=self.ssm_mesh_prefix + '/config',
             WithDecryption=True
         )
         mesh_config = json.loads(ssm_response['Parameter']['Value'])
@@ -122,11 +106,11 @@ def __enter__(self):
 
         # Load client certificates from SSM
         client_cert_parameter = self.ssm.get_parameter(
-            Name=self.ssm_prefix + '/mesh/client-cert',
+            Name=self.ssm_mesh_prefix + '/client-cert',
             WithDecryption=True
         )
         client_key_parameter = self.ssm.get_parameter(
-            Name=self.ssm_prefix + '/mesh/client-key',
+            Name=self.ssm_mesh_prefix + '/client-key',
             WithDecryption=True
         )
 
diff --git a/utils/py-utils/dl_utils/store_file.py b/utils/py-utils/dl_utils/store_file.py
@@ -0,0 +1,10 @@
+import tempfile
+
+def store_file(content):
+    """
+    Writes a temp file and returns the name
+    """
+    with tempfile.NamedTemporaryFile(delete=False) as file:
+        file.write(content)
+        file.close()
+        return file.name
diff --git a/utils/py-utils/pytest.ini b/utils/py-utils/pytest.ini
@@ -0,0 +1,16 @@
+[pytest]
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+testpaths = dl_utils/__tests__
+addopts = -v --tb=short
+
+[coverage:run]
+relative_files = True
+omit =
+    */dl_utils/__tests__/*
+    */test_*.py
+    */__pycache__/*
+    */venv/*
+    */.venv/*
+    */env/*
diff --git a/utils/py-utils/requirements-dev.txt b/utils/py-utils/requirements-dev.txt
@@ -0,0 +1,5 @@
+-r requirements.txt
+pytest>=8.0.0
+pytest-cov>=6.0.0
+pytest-mock>=3.14.0
+moto>=5.0.0
diff --git a/utils/py-utils/requirements.txt b/utils/py-utils/requirements.txt
@@ -0,0 +1,7 @@
+boto3>=1.40.70
+pydantic>=2.0.0
+structlog>=21.5.0
+mesh-client>=3.2.3
+pyopenssl>=24.0.0
+-e ../metric-publishers
+-e ../py-mock-mesh
diff --git a/utils/py-utils/setup.py b/utils/py-utils/setup.py
@@ -0,0 +1,7 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="utils-py",
+    version="0.1.0",
+    packages=find_packages(),
+)
