Create the configuration for a dedicated metrics publishing service

- This allows for publishing service-independent metrics
- This is valuable for higher level stats that should not be produced
  per running container

Author: TheOneFromNorway

bin/docker-start

@@ -12,10 +12,15 @@ elif [ "$SERVER_TYPE" == "sidekiq" ]; then
   "$BIN_DIR"/prometheus_exporter &
   sleep 5
   exec "$BIN_DIR"/sidekiq
+elif [ "$SERVER_TYPE" == "metric-publisher" ]; then
+  echo "Starting metrics publisher..."
+  "$BIN_DIR"/prometheus_exporter skip-server-labels &
+  sleep 5
+  exec "$BIN_DIR"/metrics-publisher
 elif [ "$SERVER_TYPE" == "none" ]; then
   echo "No server started"
   exec tail -f /dev/null  # Keep container running
 else
-  echo "SERVER_TYPE variable: '$SERVER_TYPE' unknown. Allowed values: web, sidekiq, none"
+  echo "SERVER_TYPE variable: '$SERVER_TYPE' unknown. Allowed values: web, sidekiq, metric-publisher, none"
   exit 1
 fi

bin/metrics-publisher

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../config/environment"
+require "prometheus_exporter/instrumentation"
+
+PrometheusExporter::Instrumentation::SidekiqStats.start
+PrometheusExporter::Instrumentation::SidekiqQueue.start(all_queues: true)
+
+at_exit { PrometheusExporter::Client.default.stop(wait_timeout_seconds: 10) }
+
+Kernel.sleep

bin/prometheus_exporter

@@ -27,18 +27,31 @@ def fetch_ecs_task_id
   end
 end
 
-if ARGV.include?("local")
-  task_id = "N/A"
-  service_name = "N/A"
-else
-  task_id = fetch_ecs_task_id
-  service_name = ENV["SERVICE_NAME"]
+# An explicitly-set env var (including empty string) overrides the default.
+# An empty value omits the label entirely — useful for the metrics task,
+# whose series identity must stay stable across container replacements.
+def resolve_label(env_key, &default)
+  if ENV.key?(env_key)
+    value = ENV[env_key]
+    return value.empty? ? nil : value
+  end
+
+  default.call
 end
 
+labels = {
+  "TaskId" => resolve_label("PROMETHEUS_TASK_ID_LABEL") do
+    ARGV.include?("skip-server-labels") ? nil : fetch_ecs_task_id
+  end,
+  "ServiceName" => resolve_label("PROMETHEUS_SERVICE_NAME_LABEL") do
+    ARGV.include?("skip-server-labels") ? nil : ENV["SERVICE_NAME"]
+  end
+}.compact
+
 runner = PrometheusExporter::Server::Runner.new(
   port: 9394,
   bind: "0.0.0.0",
-  label: { "TaskId" => task_id, "ServiceName" => service_name },
+  label: labels,
   type_collectors: [PrometheusExporter::CustomActiveRecordCollector],
 )
 

config/initializers/sidekiq.rb

@@ -49,8 +49,9 @@ def call(_worker, job, _queue)
       PrometheusExporter::Instrumentation::Process.start type: "sidekiq"
       PrometheusExporter::Instrumentation::ActiveRecord.start
       PrometheusExporter::Instrumentation::SidekiqProcess.start
-      PrometheusExporter::Instrumentation::SidekiqQueue.start
-      PrometheusExporter::Instrumentation::SidekiqStats.start
+      # SidekiqStats and SidekiqQueue are global Redis-backed metrics published
+      # by the metrics task (bin/metrics-publisher) to avoid duplicate
+      # series across Sidekiq containers.
     end
 
     at_exit do