DS-1556 Fixing release issue for cloudwatch log insights queries (#1138)

# Task Branch Pull Request

**<https://nhsd-jira.digital.nhs.uk/browse/DS-1556>**

## Description of Changes

Fixing race condition issue of cloudwatch log queries in blue-green-link
stack.

## Type of change

Delete not appropriate

- Bug fix (Race condition issue fix for cloudwatch log queries)

## Development Checklist

- [x] I have performed a self-review of my own code
- [x] Tests have added that prove my fix is effective or that my feature
works (Integration tests)
- [x] I have updated Dependabot to include my changes (if applicable)

## Code Reviewer Checklist

- [x] I can confirm the changes have been tested or approved by a tester

Author: ajmu1

infrastructure/stacks/blue-green-link/cloudwatch-queries.tf

@@ -1,4 +1,14 @@
+// Introduce a short sleep BEFORE creating query definitions to allow
+// recently-created log groups (from other modules in the same apply)
+// to propagate. Note: This does NOT guarantee post-create consistency
+// of the query definitions themselves, but can reduce immediate read
+// failures when the provider plans many resources concurrently.
+resource "time_sleep" "wait_for_propagation" {
+  create_duration = "5s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_for_errors" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-errors"
 
   log_group_names = [
@@ -16,9 +26,14 @@ fields @timestamp, correlation_id, ods_code, function_name, message
 | filter level == 'ERROR'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id"
 
   log_group_names = [
@@ -36,9 +51,14 @@ fields @timestamp, message
 | filter correlation_id == 'REPLACE'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id_expanded" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id-expanded"
 
   log_group_names = [
@@ -56,9 +76,14 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | filter correlation_id == 'REPLACE'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_odscode" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode"
 
   log_group_names = [
@@ -76,9 +101,14 @@ fields @timestamp, message
 | filter ods_code == 'REPLACE'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_odscode_expanded" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode-expanded"
 
   log_group_names = [
@@ -96,9 +126,14 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | filter ods_code == 'REPLACE'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_invalid_postcode" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-postcode"
 
   log_group_names = [
@@ -110,9 +145,14 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | filter report_key == 'INVALID_POSTCODE'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_invalid_opening_times" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-opening-times"
 
   log_group_names = [
@@ -124,9 +164,14 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
 | filter report_key == 'INVALID_OPEN_TIMES'
 | sort @timestamp
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_email_correlation_id" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-email-correlation-id"
 
   log_group_names = [
@@ -139,9 +184,14 @@ fields correlation_id
 | filter message =="Email Correlation Id"
 | filter email_correlation_id == "ADD_EMAIL_CORRELATION_ID"
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_update_request_success" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-success"
 
   log_group_names = [
@@ -153,9 +203,14 @@ fields @timestamp, correlation_id
 | filter ServiceUpdateSuccess == 1
 | sort @timestamp desc
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_update_request_failed" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-failed"
 
   log_group_names = [
@@ -167,9 +222,14 @@ fields @timestamp, correlation_id, report_key
 | filter report_key == DOS_DB_UPDATE_DLQ_HANDLER_RECEIVED_EVENT
 | sort @timestamp desc
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_by_dos_data_item_updates" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/dos-data-item-updates"
 
   log_group_names = [
@@ -183,9 +243,14 @@ fields @timestamp, correlation_id
 | filter field == 'REPLACE'
 | sort @timestamp desc
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 resource "aws_cloudwatch_query_definition" "search_for_report_warnings" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-report-warnings"
 
   log_group_names = [
@@ -204,10 +269,15 @@ fields @timestamp, correlation_id, message
 | filter level == 'WARNING'
 | sort @timestamp desc
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 
 resource "aws_cloudwatch_query_definition" "search_for_quality_checker_logs_with_odscode" {
+  depends_on = [time_sleep.wait_for_propagation]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-quality-checker-logs-with-odscode"
 
   log_group_names = [
@@ -219,4 +289,8 @@ fields @timestamp, level, message
 | filter odscode = 'TO_ADD'
 | sort @timestamp asc
 EOF
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }

infrastructure/stacks/blue-green-link/terraform.tf

@@ -7,5 +7,9 @@ terraform {
       source  = "hashicorp/aws"
       version = "~> 6.14.1"
     }
+    time = {
+      source  = "hashicorp/time"
+      version = "~> 0.13.1"
+    }
   }
 }