Merge pull request #5430 from nhsuk/prometheus_exports_to_cloudwatch_metrics

TheOneFromNorway · web-flow · commit 998333867aea · 2026-01-15T10:28:22.000Z
MAV-2918: Prometheus exporter setup with cloudwatch
diff --git a/Gemfile b/Gemfile
@@ -52,6 +52,7 @@ gem "omniauth_openid_connect"
 gem "omniauth-rails_csrf_protection"
 gem "pagy"
 gem "phonelib"
+gem "prometheus_exporter"
 gem "pstore"
 gem "pundit"
 gem "rails_semantic_logger"
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -463,6 +463,8 @@ GEM
     prettier_print (1.2.1)
     prettyprint (0.2.0)
     prism (1.8.0)
+    prometheus_exporter (2.3.1)
+      webrick
     propshaft (1.3.1)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
@@ -852,6 +854,7 @@ DEPENDENCIES
   pg
   phonelib
   prettier_print
+  prometheus_exporter
   propshaft
   pry-rails
   pstore
diff --git a/bin/docker-start b/bin/docker-start
@@ -4,9 +4,13 @@ BIN_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 if [ "$SERVER_TYPE" == "web" ]; then
   echo "Starting web server..."
+  "$BIN_DIR"/prometheus_exporter &
+  sleep 5
   exec "$BIN_DIR"/thrust "$BIN_DIR"/rails server
 elif [ "$SERVER_TYPE" == "sidekiq" ]; then
   echo "Starting sidekiq server..."
+  "$BIN_DIR"/prometheus_exporter &
+  sleep 5
   exec "$BIN_DIR"/sidekiq
 elif [ "$SERVER_TYPE" == "none" ]; then
   echo "No server started"
diff --git a/bin/prometheus_exporter b/bin/prometheus_exporter
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+TASK_DETAILS=$(curl ${ECS_CONTAINER_METADATA_URI_V4})
+TASK_ID=$(echo "$TASK_DETAILS" | jq -r '.Labels["com.amazonaws.ecs.task-arn"] | split("/") | last')
+
+task_details() {
+  curl -sSf ${ECS_CONTAINER_METADATA_URI_V4}
+}
+
+MAX_RETRIES=12
+RETRY_DELAY=10
+TASK_DETAILS=""
+
+for ((i=1; i<=MAX_RETRIES; i++)); do
+    if TASK_DETAILS=$(task_details); then
+        break
+    fi
+    if [[ $i -eq $MAX_RETRIES ]]; then
+        echo "Failed to fetch task details after $MAX_RETRIES attempts" >&2
+        exit 1
+    fi
+    sleep $RETRY_DELAY
+done
+
+TASK_ID=$(echo "$TASK_DETAILS" | jq '.Labels["com.amazonaws.ecs.task-arn"] | split("/") | last')
+
+bundle exec prometheus_exporter \
+  --port 9394 \
+  --bind 0.0.0.0 \
+  --label "{\"TaskId\": ${TASK_ID}, \"ServiceName\": \"${SERVICE_NAME}\"}"
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
@@ -9,7 +9,25 @@
   redis_config[:timeout] = 10
 end
 
-Sidekiq.configure_server { |config| config.redis = redis_config }
+Sidekiq.configure_server do |config|
+  config.redis = redis_config
+  if ENV["EXPORT_SIDEKIQ_METRICS"] == "true"
+    require "prometheus_exporter/instrumentation"
+    config.server_middleware do |chain|
+      chain.add PrometheusExporter::Instrumentation::Sidekiq
+    end
+    config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
+    config.on :startup do
+      PrometheusExporter::Instrumentation::Process.start type: "sidekiq"
+      PrometheusExporter::Instrumentation::SidekiqProcess.start
+      PrometheusExporter::Instrumentation::SidekiqQueue.start
+      PrometheusExporter::Instrumentation::SidekiqStats.start
+    end
+    at_exit do
+      PrometheusExporter::Client.default.stop(wait_timeout_seconds: 10)
+    end
+  end
+end
 
 Sidekiq.configure_client { |config| config.redis = redis_config }
 
diff --git a/config/puma.rb b/config/puma.rb
@@ -60,3 +60,14 @@
 
 # Re-open appenders after forking the process; needed for Semantic Logger
 before_worker_boot { SemanticLogger.reopen }
+
+# Export puma metrics
+if ENV["EXPORT_PUMA_METRICS"] == "true"
+  after_worker_boot do
+    require "prometheus_exporter/instrumentation"
+    # optional check, avoids spinning up and down threads per worker
+    unless PrometheusExporter::Instrumentation::Puma.started?
+      PrometheusExporter::Instrumentation::Puma.start
+    end
+  end
+end
diff --git a/script/shell.sh b/script/shell.sh
@@ -24,9 +24,9 @@ describe_tasks() {
     aws ecs describe-tasks --region "$region" --cluster "$cluster_name" --tasks $task_arns
 }
 
-select_running_container() {
+select_running_container() { 
     local task_data="$1"
-    echo "$task_data" | jq -r '.containers | map(select(.lastStatus == "RUNNING" and .runtimeId != null))[0].name'
+    echo "$task_data" | jq -r '.containers | map(select(.lastStatus == "RUNNING" and .name == "application"))[0].name'
 }
 
 if [ "$1" = "--help" ]; then
diff --git a/terraform/account/iam.tf b/terraform/account/iam.tf
@@ -8,6 +8,12 @@ resource "aws_iam_role_policy_attachment" "ecs_task_fargate" {
   policy_arn = aws_iam_policy.session_manager_access.arn
 }
 
+resource "aws_iam_role_policy_attachment" "ecs_task_cloudwatch_agent" {
+  role       = aws_iam_role.ecs_task_role.name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
+
+
 resource "aws_iam_role_policy_attachment" "get_s3_object" {
   role       = aws_iam_role.ecs_task_role.name
   policy_arn = aws_iam_policy.get_s3_object.arn
@@ -113,4 +119,4 @@ resource "aws_iam_policy" "get_s3_object" {
   lifecycle {
     ignore_changes = [description]
   }
-}
+}
diff --git a/terraform/app/ecs.tf b/terraform/app/ecs.tf
@@ -64,7 +64,7 @@ data "aws_iam_role" "ecs_task_role" {
 module "web_service" {
   source = "./modules/ecs_service"
   task_config = {
-    environment          = local.task_envs["CORE"]
+    environment          = local.web_envs
     secrets              = local.task_secrets["CORE"]
     cpu                  = 2048
     memory               = 4096
@@ -74,6 +74,17 @@ module "web_service" {
     region               = var.region
     health_check_command = ["CMD-SHELL", "./bin/internal_healthcheck http://localhost:${local.container_ports.web}/health/database"]
   }
+  export_prometheus_metrics = local.export_prometheus_metrics
+  cloudwatch_agent_secrets = [
+    {
+      "name" : "PROMETHEUS_CONFIG_CONTENT",
+      "valueFrom" : aws_ssm_parameter.prometheus_config.arn
+    },
+    {
+      "name" : "CW_CONFIG_CONTENT",
+      "valueFrom" : aws_ssm_parameter.cloudwatch_agent_config.arn
+    }
+  ]
   network_params = {
     subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
     vpc_id  = aws_vpc.application_vpc.id
@@ -122,7 +133,7 @@ module "web_service" {
 module "sidekiq_service" {
   source = "./modules/ecs_service"
   task_config = {
-    environment          = local.task_envs["CORE"]
+    environment          = local.sidekiq_envs
     secrets              = local.task_secrets["CORE"]
     cpu                  = 1024
     memory               = 6144
@@ -132,6 +143,17 @@ module "sidekiq_service" {
     region               = var.region
     health_check_command = ["CMD-SHELL", "./bin/internal_healthcheck && grep -q '[s]idekiq' /proc/*/cmdline 2>/dev/null || exit 1"]
   }
+  export_prometheus_metrics = local.export_prometheus_metrics
+  cloudwatch_agent_secrets = [
+    {
+      "name" : "PROMETHEUS_CONFIG_CONTENT",
+      "valueFrom" : aws_ssm_parameter.prometheus_config.arn
+    },
+    {
+      "name" : "CW_CONFIG_CONTENT",
+      "valueFrom" : aws_ssm_parameter.cloudwatch_agent_config.arn
+    }
+  ]
   network_params = {
     subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
     vpc_id  = aws_vpc.application_vpc.id
diff --git a/terraform/app/iam_policy_documents.tf b/terraform/app/iam_policy_documents.tf
@@ -16,10 +16,13 @@ data "aws_iam_policy_document" "ecs_secrets_access" {
   dynamic "statement" {
     for_each = length(local.parameter_values[each.key]) == 0 ? [] : [1]
     content {
-      sid       = "ssmParameterStoreAccessSid"
-      actions   = ["ssm:GetParameters"]
-      resources = [for kv_pair in local.parameter_values[each.key] : kv_pair["valueFrom"]]
-      effect    = "Allow"
+      sid     = "ssmParameterStoreAccessSid"
+      actions = ["ssm:GetParameters"]
+      resources = concat(
+        [for kv_pair in local.parameter_values[each.key] : kv_pair["valueFrom"]],
+        [aws_ssm_parameter.prometheus_config.arn, aws_ssm_parameter.cloudwatch_agent_config.arn]
+      )
+      effect = "Allow"
     }
   }
   dynamic "statement" {
diff --git a/terraform/app/modules/ecs_service/main.tf b/terraform/app/modules/ecs_service/main.tf
@@ -104,7 +104,7 @@ resource "aws_ecs_task_definition" "this" {
   memory                   = var.task_config.memory
   execution_role_arn       = var.task_config.execution_role_arn
   task_role_arn            = var.task_config.task_role_arn
-  container_definitions = jsonencode([
+  container_definitions = jsonencode(concat([
     {
       name                   = var.container_name
       image                  = "CHANGE_ME"
@@ -118,8 +118,18 @@ resource "aws_ecs_task_definition" "this" {
           protocol      = "tcp"
         }
       ]
-      environment = concat(var.task_config.environment, [{ name = "SERVER_TYPE", value = var.server_type }])
-      secrets     = var.task_config.secrets
+      environment = concat(
+        var.task_config.environment, [
+          {
+            name  = "SERVER_TYPE",
+            value = var.server_type
+          },
+          {
+            name  = "SERVICE_NAME"
+            value = "mavis-${var.environment}-${local.server_type_name}"
+          }
+      ])
+      secrets = var.task_config.secrets
       logConfiguration = {
         logDriver = "awslogs"
         options = {
@@ -136,5 +146,7 @@ resource "aws_ecs_task_definition" "this" {
         startPeriod = 10
       }
     }
-  ])
+    ],
+    local.prometheus_metric_export_containers
+  ))
 }
diff --git a/terraform/app/modules/ecs_service/variables.tf b/terraform/app/modules/ecs_service/variables.tf
@@ -68,6 +68,16 @@ variable "task_config" {
   nullable    = false
 }
 
+variable "cloudwatch_agent_secrets" {
+  type = list(object({
+    name      = string
+    valueFrom = string
+  }))
+  default     = null
+  description = "List of secrets for CloudWatch Agent container"
+  nullable    = true
+}
+
 variable "cluster_id" {
   type        = string
   description = "The ID of the ECS cluster."
@@ -146,7 +156,31 @@ variable "readonly_file_system" {
   nullable    = false
 }
 
+variable "export_prometheus_metrics" {
+  type        = bool
+  description = "Whether to export Prometheus metrics from the ECS service."
+  default     = false
+  nullable    = false
+}
+
 locals {
   autoscaling_enabled = var.maximum_replica_count > var.minimum_replica_count
   server_type_name    = var.server_type_name != null ? var.server_type_name : var.server_type
+
+  prometheus_metric_export_containers = var.export_prometheus_metrics ? [
+    {
+      name      = "cloudwatch-agent",
+      image     = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300062.0b1304",
+      secrets   = var.cloudwatch_agent_secrets == null ? [] : var.cloudwatch_agent_secrets
+      essential = false
+      logConfiguration = {
+        logDriver = "awslogs",
+        options = {
+          awslogs-group         = var.task_config.log_group_name
+          awslogs-region        = var.task_config.region
+          awslogs-stream-prefix = "${var.environment}-${local.server_type_name}-cwagent-logs"
+        }
+      }
+    }
+  ] : []
 }
diff --git a/terraform/app/ssm_parameters.tf b/terraform/app/ssm_parameters.tf
@@ -19,3 +19,24 @@ resource "aws_ssm_parameter" "reporting_environment_overwrites" {
     ignore_changes = all
   }
 }
+
+resource "aws_ssm_parameter" "prometheus_config" {
+  name        = "${aws_ecs_cluster.cluster.name}-PrometheusConfig"
+  type        = "String"
+  tier        = "Standard"
+  description = "Prometheus Scraping SSM Parameter for ECS Cluster: ${aws_ecs_cluster.cluster.name}"
+  value = templatefile("templates/prometheus_exporter_config.yaml.tpl", {
+    scrape_interval = "30s"
+    scrape_timeout  = "10s"
+  })
+}
+
+resource "aws_ssm_parameter" "cloudwatch_agent_config" {
+  name        = "${aws_ecs_cluster.cluster.name}-CloudWatchAgentConfig"
+  type        = "String"
+  tier        = "Intelligent-Tiering"
+  description = "CWAgent SSM Parameter for ECS Cluster: ${aws_ecs_cluster.cluster.name}"
+  value = templatefile("templates/cloudwatch_agent_config.json.tpl", {
+    log_group_name = aws_cloudwatch_log_group.ecs_log_group.name
+  })
+}
diff --git a/terraform/app/templates/cloudwatch_agent_config.json.tpl b/terraform/app/templates/cloudwatch_agent_config.json.tpl
@@ -0,0 +1,21 @@
+{
+  "logs": {
+    "metrics_collected": {
+      "prometheus": {
+        "prometheus_config_path": "env:PROMETHEUS_CONFIG_CONTENT",
+        "log_group_name": "${log_group_name}",
+        "emf_processor": {
+          "metric_declaration": [
+            {
+              "source_labels": [],
+              "label_matcher": ".*",
+              "dimensions": [["ClusterName"]],
+              "metric_selectors": ["^.*$"]
+            }
+          ]
+        }
+      }
+    },
+    "force_flush_interval": 5
+  }
+}
diff --git a/terraform/app/templates/prometheus_exporter_config.yaml.tpl b/terraform/app/templates/prometheus_exporter_config.yaml.tpl
@@ -0,0 +1,8 @@
+global:
+  scrape_interval: ${scrape_interval}
+  scrape_timeout: ${scrape_timeout}
+scrape_configs:
+  - job_name: mavis-metrics
+    static_configs:
+      - targets: ['localhost:9394']
+    metrics_path: /metrics
diff --git a/terraform/app/variables.tf b/terraform/app/variables.tf
@@ -253,6 +253,30 @@ locals {
     ]
   }
 
+  export_prometheus_metrics = contains([
+    "performance", "production"
+  ], var.environment)
+
+  sidekiq_envs = concat(
+    local.task_envs["CORE"],
+    [
+      {
+        name  = "EXPORT_SIDEKIQ_METRICS"
+        value = tostring(local.export_prometheus_metrics)
+      }
+    ]
+  )
+
+  web_envs = concat(
+    local.task_envs["CORE"],
+    [
+      {
+        name  = "EXPORT_PUMA_METRICS"
+        value = tostring(local.export_prometheus_metrics)
+      }
+    ]
+  )
+
   task_secrets = {
     CORE      = concat(local.secret_values["CORE"], local.parameter_values["CORE"])
     REPORTING = concat(local.secret_values["REPORTING"], local.parameter_values["REPORTING"])

Original file line number	Diff line number	Diff line change
`@@ -24,9 +24,9 @@ describe_tasks() {`
`24`	`24`	`aws ecs describe-tasks --region "$region" --cluster "$cluster_name" --tasks $task_arns`
`25`	`25`	`}`
`26`	`26`
`27`		`-select_running_container() {`
	`27`	`+select_running_container() {`
`28`	`28`	`local task_data="$1"`
`29`		`- echo "$task_data" \| jq -r '.containers \| map(select(.lastStatus == "RUNNING" and .runtimeId != null))[0].name'`
	`29`	`+ echo "$task_data" \| jq -r '.containers \| map(select(.lastStatus == "RUNNING" and .name == "application"))[0].name'`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`if [ "$1" = "--help" ]; then`