incorporate review comments

- Accept inference table name from CLI
- Merge monitoring related resources into a single file
- Parametrize the metric and validation threshold

Author: s-udhaya

databricks_template_schema.json

@@ -301,6 +301,19 @@
           }
         ]
       }
+    },
+    "input_inference_table_name": {
+      "order": 19,
+      "type": "string",
+      "description": "\nName of inference table to attach monitoring to.\nThis table must already exist and service principals must have access.",
+      "default": "predictions",
+      "skip_prompt_if": {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+      }
     }
   },
   "success_message" : "\n*** Your MLOps Stack has been created in the '{{.input_root_dir}}{{if not (eq .input_setup_cicd_and_project `CICD_Only`) }}/{{.input_project_name}}{{end}}' directory! ***\n\nPlease refer to the README.md for further instructions on getting started."

template/{{.input_root_dir}}/{{template `project_name_alphanumeric_underscore` .}}/monitoring/metric_violation_check_query.py.tmpl

@@ -4,13 +4,13 @@ import pathlib
 sys.path.append(str(pathlib.Path(__file__).parent.parent.parent.resolve()))
 
 """The SQL query is divided into three main parts. The first part selects the top 5
-values of the metric to be monitored, `root_mean_squared_error`,
+values of the metric to be monitored,
 ordered by the time window.
 ```sql
 ( SELECT
-   root_mean_squared_error
+   {metric_to_monitor}
  FROM
-   {}_profile_metrics
+   {table_name_under_monitor}_profile_metrics
  where
    column_name = ":table"
    and slice_key IS NULL
@@ -32,7 +32,7 @@ The second part of the query selects the metric values that exceed
 the defined threshold:
 ```sql
 WHERE
- root_mean_squared_error > 100
+ {metric_to_monitor} > {metric_violation_threshold}
 ```
 The final part of the query sets the `query_result` to 1 if the threshold violation
 occurred more than twice within the checking interval, and 0 otherwise:
@@ -53,9 +53,9 @@ sql_query = """SELECT
 FROM
  (
    SELECT
-     root_mean_squared_error
+     {metric_to_monitor}
    FROM
-     {}_profile_metrics
+     {table_name_under_monitor}_profile_metrics
    where
      column_name = ":table"
      and slice_key IS NULL
@@ -67,4 +67,4 @@ FROM
      5
  )
 WHERE
- root_mean_squared_error > 100"""
+ {metric_to_monitor} > {metric_violation_threshold}"""

template/{{.input_root_dir}}/{{template `project_name_alphanumeric_underscore` .}}/monitoring/notebooks/MonitoredMetricViolationCheck.py.tmpl

@@ -12,8 +12,18 @@
 #
 # Name of the table that is currently being monitored
 dbutils.widgets.text(
-    "table_name_under_monitor", " ${bundle.target}.{{ .input_project_name }}.predictions", label="Full (three-Level) table name"
+    "table_name_under_monitor", " ${bundle.target}.{{ .input_project_name }}.{{ .input_inference_table_name }}", label="Full (three-Level) table name"
 )
+# Metric to be used for threshold violation check
+dbutils.widgets.text(
+    "metric_to_monitor", "root_mean_squared_error", label="Metric to be monitored for threshold violation"
+)
+
+# Threshold value to be checked
+dbutils.widgets.text(
+    "metric_violation_threshold", "100", label="Threshold value for metric violation"
+)
+
 # COMMAND ----------
 
 import os
@@ -28,7 +38,14 @@ sys.path.append("../..")
 from metric_violation_check_query import sql_query
 
 table_name_under_monitor = dbutils.widgets.get("table_name_under_monitor")
-is_metric_violated = bool(spark.sql(sql_query.format(table_name_under_monitor)).toPandas()["query_result"][0])
+metric_to_monitor = dbutils.widgets.get("metric_to_monitor")
+metric_violation_threshold = dbutils.widgets.get("metric_violation_threshold")
+
+formatted_sql_query = sql_query.format(
+    table_name_under_monitor=table_name_under_monitor,
+    metric_to_monitor=metric_to_monitor,
+    metric_violation_threshold=metric_violation_threshold)
+is_metric_violated = bool(spark.sql(formatted_sql_query).toPandas()["query_result"][0])
 
 dbutils.jobs.taskValues.set("is_metric_violated", is_metric_violated)
 

template/{{.input_root_dir}}/{{template `project_name_alphanumeric_underscore` .}}/resources/README.md.tmpl

@@ -151,17 +151,16 @@ violation threshold defined given the ground truth labels.
 Its central purpose is to track production model performances, feature distributions and comparing different versions.
 
 Monitoring contains four components:
-* [monitoring-resource.yml](./monitoring-resource.yml) contains the resource config and input parameters for monitoring.
 * [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) defines a query that check for violation of monitored metric.
 * [notebooks/MonitoredMetricViolationCheck](../monitoring/notebooks/MonitoredMetricViolationCheck.py) acts as an entry pointexecuting the violation check query against the monitored inference table.
 It emits a boolean value based on the query result.
-* [monitoring-workflow-resource.yml](./monitoring-workflow-resource.yml) orchestrates model retraining based on monitoring. It first runs the [notebooks/MonitoredMetricViolationCheck](../monitoring/notebooks/MonitoredMetricViolationCheck.py)
+* [monitoring-workflow-resource.yml](./monitoring-workflow-resource.yml) contains the resource config, input parameters for monitoring and orchestrates model retraining based on monitoring. It first runs the [notebooks/MonitoredMetricViolationCheck](../monitoring/notebooks/MonitoredMetricViolationCheck.py)
 entry point then decides whether to execute the model retraining workflow.
 
 To set up and enable monitoring:
  * Update [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) to capture the specific metrics of interest,
  * Generate inference table, join it with ground truth labels,and update the table name in [monitoring-resource.yml](./monitoring-resource.yml).
- * Resolve the `TODOs`  in [monitoring-resource.yml](./monitoring-resource.yml)
+ * Resolve the `TODOs`  in [monitoring-workflow-resource.yml](./monitoring-workflow-resource.yml)
 
 Retraining Constraints:
 The retraining job has constraints for optimal functioning:

template/{{.input_root_dir}}/{{template `project_name_alphanumeric_underscore` .}}/resources/monitoring-resource.yml.tmpl

@@ -1,5 +1,25 @@
-# Note: Lakehouse monitring only works for unity catalog enabled workspaces. Remove this workflow if your application
+# Define lakehouse monitor
+# Note: Lakehouse monitring only works for Unity catalog enabled workspaces. Remove this resource if your application
 # is running on Non-UC workspaces
+
+resources:
+    lakehouse_monitors:
+        {{ .input_project_name }}_lakehouse_monitor:
+            # TODO: Update the table name to the inference table that already exists
+            full_name: ${bundle.target}.{{ .input_project_name }}.{{ .input_inference_table_name }}
+            output_schema_name: ${bundle.target}.{{ .input_project_name }}
+            # TODO: Update the below parameters as per your requirements
+            inference_log:
+                granularities: [1 day]
+                model_id_col: model_id
+                prediction_col: predictions
+                label_col: labels
+                problem_type: PROBLEM_TYPE_REGRESSION
+                timestamp_col: timestamp
+            schedule:
+                quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am
+                timezone_id: UTC
+
 new_cluster: &new_cluster
   new_cluster:
     num_workers: 3
@@ -24,7 +44,10 @@ resources:
             notebook_path: ../monitoring/notebooks/MonitoredMetricViolationCheck.py
             base_parameters:
               env: ${bundle.target}
-              table_name_under_monitor: ${bundle.target}.{{ .input_project_name }}.predictions
+              table_name_under_monitor: ${bundle.target}.{{ .input_project_name }}.{{ .input_inference_table_name }}
+              # TODO: Update the metric to be monitored and violation threshold
+              metric_to_monitor: root_mean_squared_error
+              metric_violation_threshold: 100
               # git source information of current ML resource deployment. It will be persisted as part of the workflow run
               git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}