VED-789 Schema Validation Engine

lambdas/shared/Makefile

@@ -1,5 +1,10 @@
+TEST_ENV := @PYTHONPATH=src:tests:src/common
+
 test:
-	@PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v
+	$(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v
+
+testv:
+	$(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v
 
 test-list:
 	@PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_

lambdas/shared/pyproject.toml

@@ -29,7 +29,24 @@ pyjwt = "^2.10.1"
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.10.7"
+ruff = "^0.14.0"
 
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "B", "C4", "I", "UP"]  # Common sensible defaults
+ignore = ["E501"]  # Ignore line-too-long if needed
+
+[tool.ruff.lint.isort]
+force-single-line = true  # Optional: force single-line imports
+
+[tool.ruff.format]
+quote-style = "double"  # Or "single", depending on your code style
+indent-style = "space"
+line-ending = "lf"

lambdas/shared/src/common/aws_dynamodb.py

@@ -1,4 +1,5 @@
-from common.clients import dynamodb_resource, logger
+from common.clients import dynamodb_resource
+from common.clients import logger
 
 
 def get_dynamodb_table(table_name):

lambdas/shared/src/common/aws_lambda_event.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, Dict
+from typing import Any
 
 
 class AwsEventType(Enum):
@@ -10,7 +10,7 @@ class AwsEventType(Enum):
 
 
 class AwsLambdaEvent:
-    def __init__(self, event: Dict[str, Any]):
+    def __init__(self, event: dict[str, Any]):
         self.event_source = None
         self.event_type = AwsEventType.UNKNOWN
         self.event_source = event.get("eventSource")

lambdas/shared/src/common/cache.py

@@ -1,5 +1,4 @@
 import json
-from typing import Optional
 
 
 class Cache:
@@ -19,7 +18,7 @@ def put(self, key: str, value: dict):
         self.cache[key] = value
         self._overwrite()
 
-    def get(self, key: str) -> Optional[dict]:
+    def get(self, key: str) -> dict | None:
         return self.cache.get(key, None)
 
     def delete(self, key: str):

lambdas/shared/src/common/log_decorator.py

@@ -9,7 +9,8 @@
 from datetime import datetime
 from functools import wraps
 
-from common.clients import firehose_client, logger
+from common.clients import firehose_client
+from common.clients import logger
 
 
 def send_log_to_firehose(stream_name, log_data: dict) -> None:

lambdas/shared/src/common/s3_reader.py

@@ -1,4 +1,5 @@
-from common.clients import logger, s3_client
+from common.clients import logger
+from common.clients import s3_client
 
 
 class S3Reader:

lambdas/shared/src/common/validator/enums/error_levels.py

@@ -0,0 +1,11 @@
+# all error Levels
+CRITICAL_ERROR = 0
+WARNING = 1
+NOTIFICATION = 2
+
+
+MESSAGES = {
+    CRITICAL_ERROR: "Critical Validation Error [%s]: %s",
+    WARNING: "Non-Critical Validation Error [%s]: %s",
+    NOTIFICATION: "Quality Issue Found [%s]: %s",
+}

lambdas/shared/src/common/validator/enums/exception_messages.py

@@ -0,0 +1,30 @@
+# all exceptions and messgaes
+UNEXPECTED_EXCEPTION = 0
+VALUE_CHECK_FAILED = 1
+HEADER_CHECK_FAILED = 2
+RECORD_LENGTH_CHECK_FAILED = 3
+VALUE_PREDICATE_FALSE = 4
+RECORD_CHECK_FAILED = 5
+RECORD_PREDICATE_FALSE = 6
+UNIQUE_CHECK_FAILED = 7
+ASSERT_CHECK_FAILED = 8
+FINALLY_ASSERT_CHECK_FAILED = 9
+PARSING_ERROR = 10
+PARENT_FAILED = 11
+KEY_CHECK_FAILED = 12
+
+MESSAGES = {
+    UNEXPECTED_EXCEPTION: "Unexpected exception [%s]: %s",
+    VALUE_CHECK_FAILED: "Value check failed.",
+    HEADER_CHECK_FAILED: "Header check failed.",
+    RECORD_LENGTH_CHECK_FAILED: "Record length check failed.",
+    RECORD_CHECK_FAILED: "Record check failed.",
+    VALUE_PREDICATE_FALSE: "Value predicate returned false.",
+    RECORD_PREDICATE_FALSE: "Record predicate returned false.",
+    UNIQUE_CHECK_FAILED: "Unique check failed.",
+    ASSERT_CHECK_FAILED: "Assertion check failed.",
+    FINALLY_ASSERT_CHECK_FAILED: "Final assertion check failed.",
+    PARSING_ERROR: "Failed to parse data correctly.",
+    PARENT_FAILED: "The parent expression failed to validate",
+    KEY_CHECK_FAILED: "Value could not be found in the Key list",
+}