Fixing race condition issue in SW queries

Author: ajmu1

infrastructure/stacks/blue-green-link/cloudwatch-queries.tf

@@ -32,8 +32,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_for_errors" {
+  depends_on      = [aws_cloudwatch_query_definition.search_for_errors]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_errors]
+  depends_on = [time_sleep.wait_after_search_for_errors]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id"
 
   log_group_names = [
@@ -57,8 +62,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_correlation_id" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_correlation_id]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_correlation_id_expanded" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id]
+  depends_on = [time_sleep.wait_after_search_by_correlation_id]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id-expanded"
 
   log_group_names = [
@@ -82,8 +92,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_correlation_id_expanded" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_correlation_id_expanded]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_odscode" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id_expanded]
+  depends_on = [time_sleep.wait_after_search_by_correlation_id_expanded]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode"
 
   log_group_names = [
@@ -107,8 +122,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_odscode" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_odscode]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_odscode_expanded" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_odscode]
+  depends_on = [time_sleep.wait_after_search_by_odscode]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode-expanded"
 
   log_group_names = [
@@ -132,8 +152,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_odscode_expanded" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_odscode_expanded]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_for_invalid_postcode" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_odscode_expanded]
+  depends_on = [time_sleep.wait_after_search_by_odscode_expanded]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-postcode"
 
   log_group_names = [
@@ -151,8 +176,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_for_invalid_postcode" {
+  depends_on      = [aws_cloudwatch_query_definition.search_for_invalid_postcode]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_for_invalid_opening_times" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_invalid_postcode]
+  depends_on = [time_sleep.wait_after_search_for_invalid_postcode]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-opening-times"
 
   log_group_names = [
@@ -170,8 +200,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_for_invalid_opening_times" {
+  depends_on      = [aws_cloudwatch_query_definition.search_for_invalid_opening_times]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_email_correlation_id" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_invalid_opening_times]
+  depends_on = [time_sleep.wait_after_search_for_invalid_opening_times]
   name = "${var.project_id}/${var.blue_green_environment}/search-by-email-correlation-id"
 
   log_group_names = [
@@ -190,8 +225,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_email_correlation_id" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_email_correlation_id]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_update_request_success" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_email_correlation_id]
+  depends_on = [time_sleep.wait_after_search_by_email_correlation_id]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-success"
 
   log_group_names = [
@@ -209,8 +249,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_update_request_success" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_update_request_success]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_update_request_failed" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_update_request_success]
+  depends_on = [time_sleep.wait_after_search_by_update_request_success]
   name = "${var.project_id}/${var.blue_green_environment}/update-request-failed"
 
   log_group_names = [
@@ -228,8 +273,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_update_request_failed" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_update_request_failed]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_by_dos_data_item_updates" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_update_request_failed]
+  depends_on = [time_sleep.wait_after_search_by_update_request_failed]
   name = "${var.project_id}/${var.blue_green_environment}/dos-data-item-updates"
 
   log_group_names = [
@@ -249,8 +299,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_by_dos_data_item_updates" {
+  depends_on      = [aws_cloudwatch_query_definition.search_by_dos_data_item_updates]
+  create_duration = "15s"
+}
+
 resource "aws_cloudwatch_query_definition" "search_for_report_warnings" {
-  depends_on = [aws_cloudwatch_query_definition.search_by_dos_data_item_updates]
+  depends_on = [time_sleep.wait_after_search_by_dos_data_item_updates]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-report-warnings"
 
   log_group_names = [
@@ -275,9 +330,13 @@ EOF
   }
 }
 
+resource "time_sleep" "wait_after_search_for_report_warnings" {
+  depends_on      = [aws_cloudwatch_query_definition.search_for_report_warnings]
+  create_duration = "15s"
+}
 
 resource "aws_cloudwatch_query_definition" "search_for_quality_checker_logs_with_odscode" {
-  depends_on = [aws_cloudwatch_query_definition.search_for_report_warnings]
+  depends_on = [time_sleep.wait_after_search_for_report_warnings]
   name = "${var.project_id}/${var.blue_green_environment}/search-for-quality-checker-logs-with-odscode"
 
   log_group_names = [