fix(telemetry): derive built-in features from manifests (#2301)

Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com>

Author: Pouyanpi

docs/telemetry.mdx

@@ -41,9 +41,9 @@ In this context, a *session* is the lifetime of a single Python process running
 
 ### Possible Values for Built-In Features
 
-Each value corresponds to a directory under `nemoguardrails/library/`. This list reflects the current release; the directories under `nemoguardrails/library/` are the authoritative source.
+Each value identifies a built-in library feature. The built-in rail manifests are the authoritative source. Related manifests can share a stable feature identifier so telemetry remains comparable across releases.
 
-`activefence`, `ai_defense`, `autoalign`, `clavata`, `cleanlab`, `content_safety`, `crowdstrike_aidr`, `factchecking`, `fiddler`, `gliner`, `guardrails_ai`, `hallucination`, `injection_detection`, `jailbreak_detection`, `llama_guard`, `pangea`, `patronusai`, `policyai`, `prompt_security`, `regex`, `self_check`, `sensitive_data_detection`, `topic_safety`, `trend_micro`.
+`activefence`, `ai_defense`, `autoalign`, `clavata`, `cleanlab`, `content_safety`, `context_bloat_detection`, `crowdstrike_aidr`, `f5`, `factchecking`, `fiddler`, `gcp_moderate_text`, `gliner`, `guardrails_ai`, `hallucination`, `hf_classifier`, `injection_detection`, `jailbreak_detection`, `llama_guard`, `pangea`, `patronusai`, `policyai`, `polygraf`, `prompt_security`, `regex`, `self_check`, `sensitive_data_detection`, `topic_safety`, `trend_micro`.
 
 ## Data Not Collected by Telemetry
 

nemoguardrails/telemetry.py

@@ -387,86 +387,23 @@ def _is_usage_stats_enabled() -> bool:
     return True
 
 
-_KNOWN_BUILTIN_FLOWS = {
-    "activefence moderation on input": "activefence",
-    "activefence moderation on input detailed": "activefence",
-    "activefence moderation on output": "activefence",
-    "ai defense inspect prompt": "ai_defense",
-    "ai defense inspect response": "ai_defense",
-    "alignscore check facts": "factchecking",
-    "autoalign check input": "autoalign",
-    "autoalign check output": "autoalign",
-    "autoalign factcheck output": "autoalign",
-    "autoalign groundedness output": "autoalign",
-    "clavata check for": "clavata",
-    "clavata check input": "clavata",
-    "clavata check output": "clavata",
-    "cleanlab trustworthiness": "cleanlab",
-    "content safety check input": "content_safety",
-    "content safety check output": "content_safety",
-    "crowdstrike aidr guard input": "crowdstrike_aidr",
-    "crowdstrike aidr guard output": "crowdstrike_aidr",
-    "detect pii on input": "sensitive_data_detection",
-    "detect pii on output": "sensitive_data_detection",
-    "detect pii on retrieval": "sensitive_data_detection",
-    "detect sensitive data on input": "sensitive_data_detection",
-    "detect sensitive data on output": "sensitive_data_detection",
-    "detect sensitive data on retrieval": "sensitive_data_detection",
-    "fiddler bot faithfulness": "fiddler",
-    "fiddler bot safety": "fiddler",
-    "fiddler user safety": "fiddler",
-    "gliner detect pii on input": "gliner",
-    "gliner detect pii on output": "gliner",
-    "gliner detect pii on retrieval": "gliner",
-    "gliner mask pii on input": "gliner",
-    "gliner mask pii on output": "gliner",
-    "gliner mask pii on retrieval": "gliner",
-    "guardrailsai check input": "guardrails_ai",
-    "guardrailsai check output": "guardrails_ai",
-    "hallucination warning": "hallucination",
-    "injection detection": "injection_detection",
-    "jailbreak detection heuristics": "jailbreak_detection",
-    "jailbreak detection model": "jailbreak_detection",
-    "llama guard check input": "llama_guard",
-    "llama guard check output": "llama_guard",
-    "mask pii on input": "sensitive_data_detection",
-    "mask pii on output": "sensitive_data_detection",
-    "mask pii on retrieval": "sensitive_data_detection",
-    "mask sensitive data on input": "sensitive_data_detection",
-    "mask sensitive data on output": "sensitive_data_detection",
-    "mask sensitive data on retrieval": "sensitive_data_detection",
-    "pangea ai guard input": "pangea",
-    "pangea ai guard output": "pangea",
-    "patronus api check output": "patronusai",
-    "patronus lynx check output hallucination": "patronusai",
-    "policyai moderation on input": "policyai",
-    "policyai moderation on output": "policyai",
-    "protect prompt": "prompt_security",
-    "protect response": "prompt_security",
-    "regex check input": "regex",
-    "regex check output": "regex",
-    "regex check retrieval": "regex",
-    "self check facts": "self_check",
-    "self check hallucination": "self_check",
-    "self check input": "self_check",
-    "self check output": "self_check",
-    "topic safety check input": "topic_safety",
-    "trend ai guard input": "trend_micro",
-    "trend ai guard output": "trend_micro",
-}
-
-_CONFIG_BUILTIN_FEATURE_ALIASES = {
+_BUILTIN_FEATURE_ID_ALIASES = {
     "fact_checking": "factchecking",
     "patronus": "patronusai",
+    "privateai": "sensitive_data_detection",
     "regex_detection": "regex",
 }
 
+_BUILTIN_FLOW_FEATURE_OVERRIDES = {
+    "self check hallucination": "self_check",
+}
+
 _COLANG_V2_LIBRARY_DIR = Path(__file__).resolve().parent / "colang" / "v2_x" / "library"
 
 
-def _normalize_builtin_feature_id(field_name: str) -> str:
-    """Return the documented feature id for a RailsConfigData field."""
-    return _CONFIG_BUILTIN_FEATURE_ALIASES.get(field_name, field_name)
+def _normalize_builtin_feature_id(feature_id: str) -> str:
+    """Return the stable telemetry id for a config field or manifest."""
+    return _BUILTIN_FEATURE_ID_ALIASES.get(feature_id, feature_id.split(".", 1)[0])
 
 
 def _flow_file_name(flow: Any) -> Optional[str]:
@@ -509,7 +446,7 @@ def _detect_builtin_features(config: "RailsConfig") -> List[str]:
 
     Uses two signals: (1) fields on ``RailsConfigData`` that differ from
     their defaults (explicit config), and (2) exact-match flow names
-    against a known set of built-in library flows. Only our own feature
+    declared by the built-in rail manifest catalog. Only our own feature
     names are ever reported, never user-defined flow names.
 
     Args:
@@ -539,6 +476,17 @@ def _detect_builtin_features(config: "RailsConfig") -> List[str]:
         except Exception:
             pass
 
+    try:
+        from nemoguardrails.manifests import default_rail_catalog
+
+        catalog = default_rail_catalog()
+    except Exception:
+        log.debug("Failed to load the built-in rail catalog for usage telemetry", exc_info=True)
+        catalog = None
+
+    if catalog is None:
+        return sorted(features)
+
     all_flows = []
     for rail_group in ["input", "output", "retrieval", "tool_output", "tool_input"]:
         group = getattr(rails, rail_group, None)
@@ -547,9 +495,14 @@ def _detect_builtin_features(config: "RailsConfig") -> List[str]:
 
     for flow_name in all_flows:
         normalized = _normalize_flow_id(flow_name)
-        feature = _KNOWN_BUILTIN_FLOWS.get(normalized)
-        if feature is not None:
-            features.add(feature)
+        manifest_name = catalog.owner_for_flow(normalized)
+        if manifest_name is None:
+            continue
+        feature = _BUILTIN_FLOW_FEATURE_OVERRIDES.get(
+            normalized,
+            _normalize_builtin_feature_id(manifest_name),
+        )
+        features.add(feature)
 
     return sorted(features)
 

tests/telemetry/test_usage_reporting.py

@@ -31,6 +31,7 @@
     _detect_builtin_features,
     _get_heartbeat_interval_s,
     _is_usage_stats_enabled,
+    _normalize_builtin_feature_id,
     _rotate_audit_file,
     _send_report,
     _write_audit_file,
@@ -983,6 +984,32 @@ def test_config_feature_ids_are_normalized(self):
         assert "patronus" not in result
         assert "regex_detection" not in result
 
+    @pytest.mark.parametrize(
+        ("feature_id", "expected"),
+        [
+            ("factchecking.align_score", "factchecking"),
+            ("self_check.tool_call", "self_check"),
+        ],
+    )
+    def test_dotted_feature_ids_use_namespace(self, feature_id, expected):
+        assert _normalize_builtin_feature_id(feature_id) == expected
+
+    def test_documented_builtin_feature_ids_match_manifest_catalog(self):
+        from nemoguardrails.manifests import default_rail_catalog
+
+        docs = (Path(__file__).parents[2] / "docs" / "telemetry.mdx").read_text(encoding="utf-8")
+        section = docs.split("### Possible Values for Built-In Features", 1)[1].split(
+            "## Data Not Collected by Telemetry",
+            1,
+        )[0]
+        documented_line = next(line for line in section.splitlines() if line.startswith("`"))
+        documented = set(documented_line.removesuffix(".").replace("`", "").split(", "))
+        expected = {
+            _normalize_builtin_feature_id(manifest.name) for manifest in default_rail_catalog().manifests.values()
+        }
+
+        assert documented == expected
+
     def test_detects_features_from_exact_flow_names(self):
         from nemoguardrails.rails.llm.config import Rails
 
@@ -999,6 +1026,107 @@ def test_detects_features_from_exact_flow_names(self):
         assert "topic_safety" in result
         assert "jailbreak_detection" in result
 
+    def test_detects_features_added_to_manifest_catalog(self):
+        from nemoguardrails.rails.llm.config import Rails
+
+        config = MagicMock()
+        config.rails = Rails()
+        config.rails.input.flows = [
+            "context bloat detection on input",
+            "f5 guardrails scan input",
+            "gcpnlp moderation",
+            "hf classifier check input",
+            "polygraf detect pii on input",
+        ]
+
+        result = _detect_builtin_features(config)
+
+        assert result == [
+            "context_bloat_detection",
+            "f5",
+            "gcp_moderate_text",
+            "hf_classifier",
+            "polygraf",
+        ]
+
+    def test_manifest_catalog_failure_skips_flow_detection(self):
+        from nemoguardrails.rails.llm.config import JailbreakDetectionConfig, Rails, RailsConfigData
+
+        config = MagicMock()
+        config.rails = Rails(
+            config=RailsConfigData(
+                jailbreak_detection=JailbreakDetectionConfig(nim_base_url="https://example.com"),
+            )
+        )
+        config.rails.input.flows = ["content safety check input"]
+
+        with patch(
+            "nemoguardrails.manifests.default_rail_catalog",
+            side_effect=RuntimeError("catalog unavailable"),
+        ):
+            assert _detect_builtin_features(config) == ["jailbreak_detection"]
+
+    def test_every_manifest_flow_resolves_to_the_expected_builtin_feature(self):
+        from nemoguardrails.manifests import default_rail_catalog
+        from nemoguardrails.rails.llm.config import Rails
+
+        config = MagicMock()
+        config.rails = Rails()
+        catalog = default_rail_catalog()
+        manifest_feature_ids = {
+            "privateai": "sensitive_data_detection",
+        }
+        flow_feature_overrides = {
+            "self check hallucination": "self_check",
+        }
+
+        mismatches = []
+        for manifest in catalog.manifests.values():
+            if manifest.flows is None:
+                continue
+            for flow_name in manifest.flows.flow_names:
+                config.rails.input.flows = [flow_name]
+                expected = flow_feature_overrides.get(
+                    flow_name,
+                    manifest_feature_ids.get(manifest.name, manifest.name.split(".", 1)[0]),
+                )
+                actual = _detect_builtin_features(config)
+                if actual != [expected]:
+                    mismatches.append((flow_name, expected, actual))
+
+        assert mismatches == []
+
+    @pytest.mark.parametrize(
+        ("flow_name", "feature_id"),
+        [
+            ("alignscore check facts", "factchecking"),
+            ("detect pii on input", "sensitive_data_detection"),
+            ("self check facts", "self_check"),
+            ("self check hallucination", "self_check"),
+        ],
+    )
+    def test_manifest_flow_preserves_compatibility_feature_id(self, flow_name, feature_id):
+        from nemoguardrails.rails.llm.config import Rails
+
+        config = MagicMock()
+        config.rails = Rails()
+        config.rails.input.flows = [flow_name]
+
+        assert _detect_builtin_features(config) == [feature_id]
+
+    def test_privateai_config_and_flow_share_compatibility_feature_id(self):
+        from nemoguardrails.rails.llm.config import PrivateAIDetection, Rails, RailsConfigData
+
+        config = MagicMock()
+        config.rails = Rails(
+            config=RailsConfigData(
+                privateai=PrivateAIDetection(server_endpoint="https://private-ai.example.com"),
+            )
+        )
+        config.rails.input.flows = ["detect pii on input"]
+
+        assert _detect_builtin_features(config) == ["sensitive_data_detection"]
+
     def test_ignores_unknown_flow_names(self):
         from nemoguardrails.rails.llm.config import Rails