national-document-repository-infrastructure/infrastructure/sqs_alarms.tf at 6552f41073cd4fae499d199f3e73dfc3714222e8 · NHSDigital/national-document-repository-infrastructure · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
locals {
  monitored_queues = {
    # main queues
    "nrl_main"       = module.sqs-nrl-queue.sqs_name
    "stitching_main" = module.sqs-stitching-queue.sqs_name
    "lg_bulk_main"   = module.sqs-lg-bulk-upload-metadata-queue.sqs_name
    "lg_inv_main"    = module.sqs-lg-bulk-upload-invalid-queue.sqs_name
    "mns_main"       = module.sqs-mns-notification-queue[0].sqs_name
    # dead-letter queues
    "nrl_dlq"       = module.sqs-nrl-queue.dlq_name
    "stitching_dlq" = module.sqs-stitching-queue.dlq_name
    "mns_dlq"       = module.sqs-mns-notification-queue[0].dlq_name
  }


  days_until_alarm = [
    [6, "medium"],
    [10, "high"]
  ]

  flat_list = flatten([
    for queue_key in keys(local.monitored_queues) : [
      for day in local.days_until_alarm : [
        queue_key,
        local.monitored_queues[queue_key],
        day[0], #day
        day[1]  #severity
      ]
    ]
  ])

  monitored_queue_day_list = [
    for i in range(0, length(local.flat_list), 4) : [
      local.flat_list[i],     # key
      local.flat_list[i + 1], # queue name
      local.flat_list[i + 2], # day
      local.flat_list[i + 3], # severity
    ]
  ]
}


module "global_sqs_age_alarm_topic" {
  count                  = local.is_sandbox ? 0 : 1
  source                 = "./modules/sns"
  sns_encryption_key_id  = module.sns_encryption_key.id
  topic_name             = "global-sqs-age-alarm-topic"
  topic_protocol         = "email"
  is_topic_endpoint_list = true
  topic_endpoint_list    = nonsensitive(split(",", data.aws_ssm_parameter.cloud_security_notification_email_list.value))

  delivery_policy = jsonencode({
    "Version" : "2012-10-17",
    "Statement" : [
      {
        "Effect" : "Allow",
        "Principal" : {
          "Service" : "cloudwatch.amazonaws.com"
        },
        "Action" : "SNS:Publish",
        "Condition" : {
          "ArnLike" : {
            "aws:SourceArn" : "arn:aws:cloudwatch:eu-west-2:${data.aws_caller_identity.current.account_id}:alarm:*"
          }
        },
        "Resource" : "*"
      }
    ]
  })
}


resource "aws_cloudwatch_metric_alarm" "sqs_oldest_message" {
  count = local.is_sandbox ? 0 : length(local.monitored_queue_day_list)

  alarm_name          = "${terraform.workspace}_${local.monitored_queue_day_list[count.index][0]}_oldest_message_alarm_${local.monitored_queue_day_list[count.index][2]}d"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateAgeOfOldestMessage"
  namespace           = "AWS/SQS"
  period              = 86400
  statistic           = "Maximum"
  threshold           = local.monitored_queue_day_list[count.index][2] * 24 * 60 * 60
  treat_missing_data  = "notBreaching"

  dimensions = {
    QueueName = local.monitored_queue_day_list[count.index][1]
  }

  alarm_description = "Alarm when a message in queue '${local.monitored_queue_day_list[count.index][1]}' is older than '${local.monitored_queue_day_list[count.index][2]}' days."

  alarm_actions = [module.sqs_alarm_lambda_topic.arn]
  ok_actions    = [module.sqs_alarm_lambda_topic.arn]

  tags = {
    Name         = "${terraform.workspace}_${local.monitored_queue_day_list[count.index][0]}_oldest_message_alarm_${local.monitored_queue_day_list[count.index][2]}d"
    severity     = local.monitored_queue_day_list[count.index][3]
    alarm_group  = local.monitored_queue_day_list[count.index][1]
    alarm_metric = "ApproximateAgeOfOldestMessage"
    is_kpi       = "true"
  }
}

module "sqs_alarm_lambda_topic" {
  source                = "./modules/sns"
  sns_encryption_key_id = module.sns_encryption_key.id
  topic_name            = "sqs-alarms-to-lambda-topic"
  topic_protocol        = "lambda"
  topic_endpoint        = module.im-alerting-lambda.lambda_arn

  delivery_policy = jsonencode({
    "Version" : "2012-10-17",
    "Statement" : [
      {
        "Effect" : "Allow",
        "Principal" : {
          "Service" : "cloudwatch.amazonaws.com"
        },
        "Action" : ["SNS:Publish"],
        "Condition" : {
          "ArnLike" : {
            "aws:SourceArn" : "arn:aws:cloudwatch:eu-west-2:${data.aws_caller_identity.current.account_id}:alarm:*"
          }
        },
        "Resource" : "*"
      }
    ]
  })
}

resource "aws_lambda_permission" "sqs_im_alerting" {
  statement_id  = "AllowExecutionFromSQSAlarmSNS"
  action        = "lambda:InvokeFunction"
  function_name = module.im-alerting-lambda.lambda_arn
  principal     = "sns.amazonaws.com"
  source_arn    = module.sqs_alarm_lambda_topic.arn
}