wip

Author: mrlockstar

lung_cancer_screening/questions/services/metrics.py

@@ -1,9 +1,11 @@
 import logging
 import os
 from threading import Lock
+from typing import Iterable
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter
 from opentelemetry import metrics
+from opentelemetry.metrics import Observation, CallbackOptions
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 
@@ -16,9 +18,7 @@ class Metrics:
     _initialised = False
 
     def __new__(cls, *args, **kwargs):
-
         logger.info("Creating a new instance of Metrics class.")
-
         if cls._instance is None:
             with cls._lock:
                 if cls._instance is None:
@@ -50,7 +50,11 @@ def __init__(self):
             self.meter = metrics.get_meter("lungcs.models")
 
         self.environment = environment
-        self._gauges = {}
+
+        # store latest gauge values here
+        self._gauge_values = {}
+        self._gauge_lock = Lock()
+        self._registered_observable_gauges = set()
 
         self.requests_created = self.meter.create_counter(
             name="requests.created",
@@ -66,10 +70,7 @@ def __init__(self):
         self.__class__._initialised = True
 
     def record_request_created(self, model_name: str):
-        logger.info(
-            "Metrics: record_request_created(model_name=%s)",
-            model_name
-        )
+        logger.info("Metrics: record_request_created(model_name=%s)", model_name)
         self.requests_created.add(
             1,
             {
@@ -79,11 +80,7 @@ def record_request_created(self, model_name: str):
         )
 
     def record_request_submitted(self, model_name: str):
-        logger.info(
-            "Metrics: record_request_submitted(model_name=%s)",
-            model_name
-        )
-        logger.info("record_request_submitted.")
+        logger.info("Metrics: record_request_submitted(model_name=%s)", model_name)
         self.requests_submitted.add(
             1,
             {
@@ -92,6 +89,18 @@ def record_request_submitted(self, model_name: str):
             },
         )
 
+    def _make_gauge_callback(self, metric_name: str):
+        def callback(options: CallbackOptions) -> Iterable[Observation]:
+            with self._gauge_lock:
+                value = self._gauge_values.get(metric_name, 0)
+
+            yield Observation(
+                value,
+                {"environment": self.environment},
+            )
+
+        return callback
+
     def set_gauge_value(self, metric_name, units, description, value):
         logger.debug(
             "Metrics: set_gauge_value(metric_name=%s, units=%s, description=%s, value=%s)",
@@ -101,16 +110,14 @@ def set_gauge_value(self, metric_name, units, description, value):
             value,
         )
 
-        gauge = self._gauges.get(metric_name)
-        if gauge is None:
-            gauge = self.meter.create_gauge(
-                name=metric_name,
-                unit=units,
-                description=description,
-            )
-            self._gauges[metric_name] = gauge
-
-        gauge.record(
-            value,
-            {"environment": self.environment},
-        )
+        with self._gauge_lock:
+            self._gauge_values[metric_name] = value
+
+            if metric_name not in self._registered_observable_gauges:
+                self.meter.create_observable_gauge(
+                    name=metric_name,
+                    callbacks=[self._make_gauge_callback(metric_name)],
+                    unit=units,
+                    description=description,
+                )
+                self._registered_observable_gauges.add(metric_name)