DS-2241 Fixing CW query consistency issue (#1147)

# Task Branch Pull Request

**<https://nhsd-jira.digital.nhs.uk/browse/DS-2241>**

## Description of Changes

This change is to fix the read-after-write eventual consistency issue
occuring during terraform apply operation in AWS codebuild jobs across
CI/CD pipelines.

## Type of change

Delete not appropriate

- Bug fix (Fixing CW query consistency issue during terraform apply)

## Development Checklist

- [x] I have performed a self-review of my own code
- [x] Tests have added that prove my fix is effective or that my feature
works (Integration tests)
- [x] I have updated Dependabot to include my changes (if applicable)

## Code Reviewer Checklist

- [x] I can confirm the changes have been tested or approved by a tester

Author: ajmu1

infrastructure/stacks/blue-green-link/cloudwatch-queries.tf

@@ -1,14 +1,4 @@
-// Introduce a short sleep BEFORE creating query definitions to allow
-// recently-created log groups (from other modules in the same apply)
-// to propagate. Note: This does NOT guarantee post-create consistency
-// of the query definitions themselves, but can reduce immediate read
-// failures when the provider plans many resources concurrently.
-resource "time_sleep" "wait_for_propagation" {
-  create_duration = "45s"
-}
-
 resource "aws_cloudwatch_query_definition" "search_for_errors" {
-  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-errors"
 
   log_group_names = [
@@ -27,13 +17,12 @@ fields @timestamp, correlation_id, ods_code, function_name, message
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_errors]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id"
 
   log_group_names = [
@@ -52,13 +41,12 @@ fields @timestamp, message
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id_expanded" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id-expanded"
 
   log_group_names = [
@@ -77,13 +65,12 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_odscode" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id_expanded]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode"
 
   log_group_names = [
@@ -102,13 +89,12 @@ fields @timestamp, message
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_odscode_expanded" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_odscode]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode-expanded"
 
   log_group_names = [
@@ -127,13 +113,12 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_invalid_postcode" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_odscode_expanded]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-postcode"
 
   log_group_names = [
@@ -146,13 +131,12 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_invalid_opening_times" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_invalid_postcode]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-opening-times"
 
   log_group_names = [
@@ -165,13 +149,12 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | sort @timestamp
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_email_correlation_id" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_invalid_opening_times]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-email-correlation-id"
 
   log_group_names = [
@@ -185,13 +168,12 @@ fields correlation_id
 | filter email_correlation_id == "ADD_EMAIL_CORRELATION_ID"
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_update_request_success" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_email_correlation_id]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-success"
 
   log_group_names = [
@@ -204,13 +186,12 @@ fields @timestamp, correlation_id
 | sort @timestamp desc
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_update_request_failed" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_update_request_success]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-failed"
 
   log_group_names = [
@@ -223,13 +204,12 @@ fields @timestamp, correlation_id, report_key
 | sort @timestamp desc
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_dos_data_item_updates" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_update_request_failed]
   name = "${var.project_id}/${var.blue_green_environment}/dos-data-item-updates"
 
   log_group_names = [
@@ -244,13 +224,12 @@ fields @timestamp, correlation_id
 | sort @timestamp desc
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_report_warnings" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_dos_data_item_updates]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-report-warnings"
 
   log_group_names = [
@@ -270,14 +249,13 @@ fields @timestamp, correlation_id, message
 | sort @timestamp desc
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }
 
 
 resource "aws_cloudwatch_query_definition" "search_for_quality_checker_logs_with_odscode" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_report_warnings]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-quality-checker-logs-with-odscode"
 
   log_group_names = [
@@ -290,7 +268,7 @@ fields @timestamp, level, message
 | sort @timestamp asc
 EOF
 
-  lifecycle {
-    create_before_destroy = false
+  provisioner "local-exec" {
+    command = "sleep 30"
   }
 }

infrastructure/stacks/blue-green-link/terraform.tf

@@ -7,9 +7,5 @@ terraform {
       source  = "hashicorp/aws"
       version = "~> 6.14.1"
     }
-    time = {
-      source  = "hashicorp/time"
-      version = "~> 0.13.1"
-    }
   }
 }