Release/4.0 (#561)

This PR covers releasing 4.0

Co-authored-by: JackPlowman jack.plowman1@nhs.net
Co-authored-by: Sam Clayton SLClayton@users.noreply.github.com

Author: JackPlowman

.github/PULL_REQUEST_TEMPLATE/task_pull_request_template.md

@@ -18,13 +18,9 @@ Delete not appropriate
 ## Development Checklist
 
 - [ ] I have performed a self-review of my own code
-- [ ] I have run the [code formatting checks](../README.md#code-quality)
-- [ ] I have run the [code quality checks](../README.md#code-quality)
-- [ ] New code meets [standards](https://nhsd-confluence.digital.nhs.uk/display/DI/DI+Ways+of+Working) agreed by the team
 - [ ] Tests have added that prove my fix is effective or that my feature works (Integration tests)
 - [ ] I have updated Dependabot to include my changes (if applicable)
 
 ## Code Reviewer Checklist
 
 - [ ] I can confirm the changes have been tested or approved by a tester
-- [ ] I have checked any ignore commands for code linting tools and I agree that the code is safe

Makefile

@@ -30,14 +30,17 @@ build-and-push: # Build lambda docker images and pushes them to ECR
 	done
 
 deploy: # Deploys whole project - mandatory: PROFILE
-	make terraform-apply-auto-approve STACKS=api-key,appconfig,before-lambda-deployment
 	eval "$$(make -s populate-deployment-variables)"
+	make terraform-apply-auto-approve STACKS=api-key,appconfig,before-lambda-deployment
+	eval "$$(make -s populate-serverless-variables)"
 	make serverless-deploy
 	make terraform-apply-auto-approve STACKS=after-lambda-deployment
 
 undeploy: # Undeploys whole project - mandatory: PROFILE
+	eval "$$(make -s populate-deployment-variables)"
 	make terraform-destroy-auto-approve STACKS=after-lambda-deployment
-	make serverless-remove VERSION="any" DB_PASSWORD="any" DB_SERVER="any" DB_USER_NAME="any" SLACK_WEBHOOK_URL="any" DB_READ_ONLY_USER_NAME="any" DB_READ_AND_WRITE_USER_NAME="any" DB_REPLICA_SERVER="any" PROJECT_TEAM_EMAIL_ADDRESS="any" DB_NAME="any" PROJECT_SYSTEM_EMAIL_ADDRESS="any"
+	eval "$$(make -s populate-serverless-variables)"
+	make serverless-remove VERSION="any"
 	make terraform-destroy-auto-approve STACKS=before-lambda-deployment,appconfig
 	if [ "$(PROFILE)" != "live" ]; then
 		make terraform-destroy-auto-approve STACKS=api-key
@@ -50,12 +53,21 @@ build-and-deploy: # Builds and Deploys whole project - mandatory: PROFILE
 populate-deployment-variables:
 	echo "export DB_SERVER=$$(make -s aws-rds-describe-instance-value DB_INSTANCE=$(DB_SERVER_NAME) KEY_DOT_PATH=Endpoint.Address)"
 	echo "export DB_REPLICA_SERVER=$$(make -s aws-rds-describe-instance-value DB_INSTANCE=$(DB_REPLICA_SERVER_NAME) KEY_DOT_PATH=Endpoint.Address)"
-	echo "export DB_READ_AND_WRITE_USER_NAME=$$(make -s secret-get-existing-value NAME=$(DB_USER_NAME_SECRET_NAME) KEY=$(DB_USER_NAME_SECRET_KEY))"
-	echo "export DB_READ_ONLY_USER_NAME=$$(make -s secret-get-existing-value NAME=$(DB_READ_ONLY_USER_NAME_SECRET_NAME) KEY=$(DB_READ_ONLY_USER_NAME_SECRET_KEY))"
-	echo "export SLACK_WEBHOOK_URL=$$(make -s secret-get-existing-value NAME=$(SLACK_WEBHOOK_SECRET_NAME) KEY=$(SLACK_WEBHOOK_SECRET_KEY))"
-	echo "export PROJECT_SYSTEM_EMAIL_ADDRESS=$$(make -s secret-get-existing-value NAME=$(EMAIL_SECRETS) KEY=$(SYSTEM_EMAIL_KEY))"
-	echo "export PROJECT_TEAM_EMAIL_ADDRESS=$$(make -s secret-get-existing-value NAME=$(EMAIL_SECRETS) KEY=$(TEAM_EMAIL_KEY))"
-	echo "export TERRAFORM_KMS_KEY_ID=$$(make terraform-output STACKS=before-lambda-deployment | grep -oP 'kms_key_id = "\K[^"]+')"
+	DEPLOYMENT_SECRETS=$$(make -s secret-get-existing-value NAME=$(DEPLOYMENT_SECRETS))
+	echo "export DB_READ_AND_WRITE_USER_NAME=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DB_USER_NAME_SECRET_KEY)')"
+	echo "export DB_READ_ONLY_USER_NAME=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DB_READ_ONLY_USER_NAME_SECRET_KEY)')"
+	echo "export SLACK_WEBHOOK_URL=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(SLACK_WEBHOOK_SECRET_KEY)')"
+	echo "export PROJECT_SYSTEM_EMAIL_ADDRESS=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(SYSTEM_EMAIL_KEY)')"
+	echo "export PROJECT_TEAM_EMAIL_ADDRESS=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(TEAM_EMAIL_KEY)')"
+	echo "export PROJECT_SERVICE_CATEGORY=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(SERVICE_CATEGORY_KEY)')"
+	echo "export PROJECT_DATA_CLASSIFICATION=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DATA_CLASSIFICATION_KEY)')"
+	echo "export PROJECT_DISTRIBUTION_LIST=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DISTRIBUTION_LIST_KEY)')"
+	echo "export TF_VAR_service_category=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(SERVICE_CATEGORY_KEY)')"
+	echo "export TF_VAR_data_classification=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DATA_CLASSIFICATION_KEY)')"
+	echo "export TF_VAR_distribution_list=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DISTRIBUTION_LIST_KEY)')"
+
+populate-serverless-variables:
+	echo "export TERRAFORM_KMS_KEY_ID=$$(make -s terraform-output STACKS=before-lambda-deployment OPTS='-raw kms_key_id' | tail -n1)"
 
 unit-test-local:
 	pyenv local .venv
@@ -239,6 +251,7 @@ send-email-build-and-deploy: ### Build and deploy send email lambda docker image
 
 sls-only-deploy: # Deploys all lambdas - mandatory: PROFILE, VERSION=[commit hash-timestamp/latest]
 	eval "$$(make -s populate-deployment-variables)"
+	eval "$$(make -s populate-serverless-variables)"
 	make serverless-deploy
 
 quick-build-and-deploy: # Build and deploy lambdas only (meant to for fast redeployment of existing lambdas) - mandatory: PROFILE, ENVIRONMENT
@@ -523,7 +536,7 @@ trigger-dos-deployment-pipeline:
 	--user $$JENKINS_USERNAME:$$JENKINS_PASSWORD \
 	-H "Jenkins-Crumb: $$JENKINS_CRUMB" \
 	-F "TARGET=\"regressiondi\"" \
-	-F "IMAGE_TAG=\"7.11.3_157c538\"" \
+	-F "IMAGE_TAG=\"7.12.1_d393439\"" \
 	-F "REFRESH=\"true\""
 	echo Jenkins Job has started
 	echo Sleeping for 3 minutes

application/pyproject.toml

@@ -29,5 +29,4 @@ ignore_names = [
   "metadata",
   "recipient_email_address",
   "recipient_id",
-  "format_address",
   ]

application/send_email/requirements.txt

@@ -1,2 +1,2 @@
 aws_embedded_metrics
-aws_lambda_powertools
+aws-lambda-powertools == 1.31.1

application/service_sync/changes_to_dos.py

@@ -3,7 +3,7 @@
 
 from aws_lambda_powertools.logging import Logger
 
-from .format import format_website
+from .format import format_address, format_website
 from .service_histories import ServiceHistories
 from .validation import validate_website
 from common.dos import DoSService, get_valid_dos_location
@@ -98,20 +98,23 @@ def check_for_specified_opening_times_changes(self) -> bool:
                 "Removing Specified opening times that occur in the past",
                 extra={"all_nhs": nhs_spec_open_dates, "future_nhs": future_nhs_spec_open_dates},
             )
-        if SpecifiedOpeningTime.equal_lists(dos_spec_open_dates, future_nhs_spec_open_dates):
-            logger.info(
-                "Specified opening times are equal, so no change",
-                extra={"dos": dos_spec_open_dates, "nhs": future_nhs_spec_open_dates},
-            )
-            return False
-        else:
+        equal_specified_opening_times = SpecifiedOpeningTime.equal_lists(
+            dos_spec_open_dates, future_nhs_spec_open_dates
+        )
+        if not equal_specified_opening_times or len(nhs_spec_open_dates) != len(future_nhs_spec_open_dates):
             logger.info(
                 "Specified opening times not equal",
                 extra={"dos": dos_spec_open_dates, "nhs": future_nhs_spec_open_dates},
             )
             self.current_specified_opening_times = dos_spec_open_dates
             self.new_specified_opening_times = future_nhs_spec_open_dates
             return True
+        else:
+            logger.info(
+                "Specified opening times are equal, so no change",
+                extra={"dos": dos_spec_open_dates, "nhs": future_nhs_spec_open_dates},
+            )
+            return False
 
     def check_for_address_and_postcode_for_changes(self) -> Tuple[bool, bool, Optional[DoSLocation]]:
         """Check if address and postcode have changed between dos_service and nhs_entity,
@@ -120,13 +123,12 @@ def check_for_address_and_postcode_for_changes(self) -> Tuple[bool, bool, Option
         Returns:
             Tuple[bool, bool]: Tuple of booleans, first is if address has changed, second is if postcode has changed, third is the DoSLocation object for the postcode
         """  # noqa: E501
-        # Address Formatting Turned off in DI-591
-        # before_title_case_address = self.nhs_entity.address_lines
-        # self.nhs_entity.address_lines = list(map(format_address, self.nhs_entity.address_lines))
-        # logger.info(
-        #     f"Address after title casing: {self.nhs_entity.address_lines}",
-        #     extra={"before": before_title_case_address, "after": self.nhs_entity.address_lines},
-        # )
+        before_title_case_address = self.nhs_entity.address_lines
+        self.nhs_entity.address_lines = list(map(format_address, self.nhs_entity.address_lines))
+        logger.info(
+            f"Address after title casing: {self.nhs_entity.address_lines}",
+            extra={"before": before_title_case_address, "after": self.nhs_entity.address_lines},
+        )
         nhs_uk_address_string = "$".join(self.nhs_entity.address_lines)
         dos_address = self.dos_service.address
         is_address_same = True

application/service_sync/service_histories.py

@@ -193,7 +193,7 @@ def save_service_histories(self, connection: connection) -> None:
                 "CURRENT_DATE_TIME": current_date_time,
                 "SERVICE_ID": self.service_id,
             },
-            log_vars=False
+            log_vars=False,
         )
         cursor.close()
         if self.history_already_exists:
@@ -204,7 +204,7 @@ def save_service_histories(self, connection: connection) -> None:
                     """UPDATE servicehistories SET history = %(SERVICE_HISTORY)s WHERE serviceid = %(SERVICE_ID)s;"""
                 ),
                 vars={"SERVICE_HISTORY": Json(self.service_history), "SERVICE_ID": self.service_id},
-                log_vars=False
+                log_vars=False,
             )
             logger.info(f"Service history updated for serviceid {self.service_id}")
             cursor.close()
@@ -217,7 +217,7 @@ def save_service_histories(self, connection: connection) -> None:
                     """VALUES (%(SERVICE_ID)s, %(SERVICE_HISTORY)s);"""
                 ),
                 vars={"SERVICE_ID": self.service_id, "SERVICE_HISTORY": Json(self.service_history)},
-                log_vars=False
+                log_vars=False,
             )
             cursor.close()
             logger.warning(f"Service history created in the database for serviceid {self.service_id}")

build/automation/init.mk

@@ -605,7 +605,7 @@ INFRASTRUCTURE_DIR_REL := $(shell echo $(INFRASTRUCTURE_DIR) | sed "s;$(PROJECT_
 JQ_DIR_REL := $(shell echo $(abspath $(LIB_DIR)/jq) | sed "s;$(PROJECT_DIR);;g")
 
 GIT_BRANCH_PATTERN_MAIN := ^(main|master|develop)$$
-GIT_BRANCH_PATTERN_PREFIX := ^(task|spike|automation|test|bugfix|hotfix|fix|release|migration|refactor)
+GIT_BRANCH_PATTERN_PREFIX := ^(task|spike|automation|test|bugfix|hotfix|fix|release|migration|refactor|documentation)
 GIT_BRANCH_PATTERN_SUFFIX := ([A-Z]{2,5}-([0-9]{1,5}|X{1,5})_[A-Z][a-z]+_[A-Za-z0-9]+_[A-Za-z0-9_]+)$$
 GIT_BRANCH_PATTERN_ADDITIONAL := ^(task/Update_(automation_scripts|dependencies|documentation|tests|versions)|task/Refactor|task/Refactor_[A-Za-z0-9_]+_[A-Za-z0-9_]+|refactor/[A-Z][a-z]+_[A-Za-z0-9_]+_[A-Za-z0-9_]+|devops/[A-Z][a-z]+_[A-Za-z0-9_]+_[A-Za-z0-9_]+|alignment/[A-Z][a-z]+_[A-Za-z0-9_]+_[A-Za-z0-9_]+|dependabot/.*)$$
 GIT_BRANCH_PATTERN := $(GIT_BRANCH_PATTERN_MAIN)|$(GIT_BRANCH_PATTERN_PREFIX)/$(GIT_BRANCH_PATTERN_SUFFIX)|$(GIT_BRANCH_PATTERN_ADDITIONAL)

build/automation/var/platform-texas/v1/account-live-k8s-nonprod.mk

@@ -8,12 +8,9 @@ TF_VAR_terraform_platform_state_store = nhsd-texasplatform-terraform-state-store
 
 TF_VAR_developer_role_name = UECPUServiceDeveloper
 
-EMAIL_SECRETS := uec-dos-int-dev/deployment
-PROJECT_EMAIL_SECRET_NAME := $(EMAIL_SECRETS)
+
 SYSTEM_EMAIL_KEY := DI_SYSTEM_MAILBOX_ADDRESS
 TEAM_EMAIL_KEY := DI_TEAM_MAILBOX_ADDRESS
-TF_VAR_email_secrets := $(EMAIL_SECRETS)
-TF_VAR_system_email_address_key := $(SYSTEM_EMAIL_KEY)
 
 # ==============================================================================
 

build/automation/var/platform-texas/v1/account-live-k8s-prod.mk

@@ -8,12 +8,8 @@ TF_VAR_terraform_platform_state_store = nhsd-texasplatform-terraform-state-store
 
 TF_VAR_developer_role_name = UECPUSupportServiceDeveloper
 
-EMAIL_SECRETS := uec-dos-int-live/deployment
-PROJECT_EMAIL_SECRET_NAME := $(EMAIL_SECRETS)
 SYSTEM_EMAIL_KEY := DI_SYSTEM_MAILBOX_ADDRESS
 TEAM_EMAIL_KEY := DI_TEAM_MAILBOX_ADDRESS
-TF_VAR_email_secrets := $(EMAIL_SECRETS)
-TF_VAR_system_email_address_key := $(SYSTEM_EMAIL_KEY)
 
 # ==============================================================================
 

build/automation/var/profile/demo.mk

@@ -2,6 +2,7 @@
 
 # ==============================================================================
 # Service variables
+DEPLOYMENT_SECRETS = $(PROJECT_ID)-demo/deployment # Move to project.mk when nonprod profiles are merged
 
 LOG_LEVEL:= INFO
 
@@ -26,6 +27,8 @@ SLACK_WEBHOOK_SECRET_NAME = uec-dos-int-$(PROFILE)/deployment
 SLACK_WEBHOOK_SECRET_KEY = SLACK_WEBHOOK
 SLACK_ALERT_CHANNEL := dos-integration-dev-status
 
+TAG_SECRET_MANAGER := uec-dos-int-live/deployment
+
 # ==============================================================================
 # Organisation Types Feature Flags