fix(library): unblock reasoning models in self-check and content-safety actions

Reasoning models (OpenAI o-series, gpt-5, DeepSeek-R1, Gemini 2.5,
Qwen QwQ, etc.) spend output tokens on internal reasoning before
producing visible text. The library's safety/self-check actions
defaulted max_tokens to 3 (or 10 for topic safety), which reasoning
models consume entirely on thinking, returning empty content with
finish_reason='length'. Callers saw a silent "" result and the checks
produced nonsense.

- Bump the fallback _MAX_TOKENS from 3 to 1024 in:
    self_check/input_check, self_check/output_check, self_check/facts,
    content_safety (both input and output checks).
  Classical models still stop early via stop tokens / natural completion,
  so cost impact is negligible. User-configured max_tokens (via prompts
  config) still wins.

- Add warn_if_truncated() in actions/llm/utils.py: emits a WARNING when
  response.content is empty and finish_reason == 'length'. Called from
  each patched action so any user who tunes max_tokens too low for a
  reasoning model gets an actionable log line instead of silent failure.

- Unit-test warn_if_truncated for empty-with-length, non-empty, and
  non-length finish-reason cases.

topic_safety is not affected (its max_tokens is computed but never
passed to llm_call in the current code). Not touched here.

Author: Pouyanpi

nemoguardrails/actions/llm/utils.py

@@ -396,6 +396,25 @@ def _extract_content(response: LLMResponse) -> str:
     return response.content
 
 
+def warn_if_truncated(response: LLMResponse, task: str) -> None:
+    """Emit a warning if the LLM produced no visible content because it hit the max_tokens budget.
+
+    Reasoning models (OpenAI o-series, gpt-5, DeepSeek-R1, Gemini 2.5, Qwen QwQ, etc.)
+    spend output tokens on internal reasoning before emitting visible text. A small
+    max_tokens budget can be fully consumed by the reasoning phase, leaving empty
+    content and finish_reason="length". The call succeeds silently and callers that
+    only inspect response.content see nothing.
+    """
+    if not response.content and response.finish_reason == "length":
+        logger.warning(
+            "Task %s: LLM returned empty content with finish_reason='length'. "
+            "The max_tokens budget was likely consumed before any visible output. "
+            "If using a reasoning model (o1/o3/o4-mini, gpt-5, deepseek-r1, "
+            "gemini-2.5, etc.), increase the prompt's max_tokens in your config.",
+            task,
+        )
+
+
 def get_colang_history(
     events: List[dict],
     include_texts: bool = True,

nemoguardrails/library/content_safety/actions.py

@@ -17,7 +17,7 @@
 from typing import Dict, FrozenSet, Optional
 
 from nemoguardrails.actions.actions import action
-from nemoguardrails.actions.llm.utils import llm_call
+from nemoguardrails.actions.llm.utils import llm_call, warn_if_truncated
 from nemoguardrails.context import llm_call_info_var
 from nemoguardrails.llm.cache import CacheInterface
 from nemoguardrails.llm.cache.utils import (
@@ -47,7 +47,7 @@ async def content_safety_check_input(
     model_caches: Optional[Dict[str, CacheInterface]] = None,
     **kwargs,
 ) -> dict:
-    _MAX_TOKENS = 3
+    _MAX_TOKENS = 1024
     user_input: str = ""
 
     if context is not None:
@@ -97,16 +97,14 @@ async def content_safety_check_input(
             log.debug(f"Content safety cache hit for model '{model_name}'")
             return cached_result
 
-    result = (
-        await llm_call(
-            llm,
-            check_input_prompt,
-            stop=stop,
-            llm_params={"temperature": 1e-20, "max_tokens": max_tokens},
-        )
-    ).content
-
-    result = llm_task_manager.parse_task_output(task, output=result)
+    llm_response = await llm_call(
+        llm,
+        check_input_prompt,
+        stop=stop,
+        llm_params={"temperature": 1e-20, "max_tokens": max_tokens},
+    )
+    warn_if_truncated(llm_response, task)
+    result = llm_task_manager.parse_task_output(task, output=llm_response.content)
 
     is_safe, *violated_policies = result
 
@@ -150,7 +148,7 @@ async def content_safety_check_output(
     model_caches: Optional[Dict[str, CacheInterface]] = None,
     **kwargs,
 ) -> dict:
-    _MAX_TOKENS = 3
+    _MAX_TOKENS = 1024
     user_input: str = ""
     bot_response: str = ""
 
@@ -203,16 +201,14 @@ async def content_safety_check_output(
             log.debug(f"Content safety output cache hit for model '{model_name}'")
             return cached_result
 
-    result = (
-        await llm_call(
-            llm,
-            check_output_prompt,
-            stop=stop,
-            llm_params={"temperature": 1e-20, "max_tokens": max_tokens},
-        )
-    ).content
-
-    result = llm_task_manager.parse_task_output(task, output=result)
+    llm_response = await llm_call(
+        llm,
+        check_output_prompt,
+        stop=stop,
+        llm_params={"temperature": 1e-20, "max_tokens": max_tokens},
+    )
+    warn_if_truncated(llm_response, task)
+    result = llm_task_manager.parse_task_output(task, output=llm_response.content)
 
     is_safe, *violated_policies = result
 

nemoguardrails/library/self_check/facts/actions.py

@@ -18,7 +18,7 @@
 
 from nemoguardrails import RailsConfig
 from nemoguardrails.actions import action
-from nemoguardrails.actions.llm.utils import llm_call
+from nemoguardrails.actions.llm.utils import llm_call, warn_if_truncated
 from nemoguardrails.context import llm_call_info_var
 from nemoguardrails.llm.taskmanager import LLMTaskManager
 from nemoguardrails.llm.types import Task
@@ -48,7 +48,7 @@ async def self_check_facts(
     **kwargs,
 ):
     """Checks the facts for the bot response by appropriately prompting the base llm."""
-    _MAX_TOKENS = 3
+    _MAX_TOKENS = 1024
     evidence = context.get("relevant_chunks", [])
     response = context.get("bot_message")
 
@@ -70,14 +70,14 @@ async def self_check_facts(
     # Initialize the LLMCallInfo object
     llm_call_info_var.set(LLMCallInfo(task=task.value))
 
-    response = (
-        await llm_call(
-            llm,
-            prompt,
-            stop=stop,
-            llm_params={"temperature": config.lowest_temperature, "max_tokens": max_tokens},
-        )
-    ).content
+    llm_response = await llm_call(
+        llm,
+        prompt,
+        stop=stop,
+        llm_params={"temperature": config.lowest_temperature, "max_tokens": max_tokens},
+    )
+    warn_if_truncated(llm_response, task.value)
+    response = llm_response.content
 
     if llm_task_manager.has_output_parser(task):
         result = llm_task_manager.parse_task_output(task, output=response)

nemoguardrails/library/self_check/input_check/actions.py

@@ -18,7 +18,7 @@
 
 from nemoguardrails import RailsConfig
 from nemoguardrails.actions.actions import ActionResult, action
-from nemoguardrails.actions.llm.utils import llm_call
+from nemoguardrails.actions.llm.utils import llm_call, warn_if_truncated
 from nemoguardrails.context import llm_call_info_var
 from nemoguardrails.llm.taskmanager import LLMTaskManager
 from nemoguardrails.llm.types import Task
@@ -46,7 +46,7 @@ async def self_check_input(
         True if the input should be allowed, False otherwise.
     """
 
-    _MAX_TOKENS = 3
+    _MAX_TOKENS = 1024
     user_input = context.get("user_message")
     task = Task.SELF_CHECK_INPUT
 
@@ -64,17 +64,17 @@ async def self_check_input(
         # Initialize the LLMCallInfo object
         llm_call_info_var.set(LLMCallInfo(task=task.value))
 
-        response = (
-            await llm_call(
-                llm,
-                prompt,
-                stop=stop,
-                llm_params={
-                    "temperature": config.lowest_temperature,
-                    "max_tokens": max_tokens,
-                },
-            )
-        ).content
+        llm_response = await llm_call(
+            llm,
+            prompt,
+            stop=stop,
+            llm_params={
+                "temperature": config.lowest_temperature,
+                "max_tokens": max_tokens,
+            },
+        )
+        warn_if_truncated(llm_response, task.value)
+        response = llm_response.content
 
         log.info(f"Input self-checking result is: `{response}`.")
 

nemoguardrails/library/self_check/output_check/actions.py

@@ -18,7 +18,7 @@
 
 from nemoguardrails import RailsConfig
 from nemoguardrails.actions import action
-from nemoguardrails.actions.llm.utils import llm_call
+from nemoguardrails.actions.llm.utils import llm_call, warn_if_truncated
 from nemoguardrails.context import llm_call_info_var
 from nemoguardrails.llm.taskmanager import LLMTaskManager
 from nemoguardrails.llm.types import Task
@@ -48,7 +48,7 @@ async def self_check_output(
         True if the output should be allowed, False otherwise.
     """
 
-    _MAX_TOKENS = 3
+    _MAX_TOKENS = 1024
     bot_response = context.get("bot_message")
     user_input = context.get("user_message")
     bot_thinking = context.get("bot_thinking")
@@ -71,17 +71,17 @@ async def self_check_output(
         # Initialize the LLMCallInfo object
         llm_call_info_var.set(LLMCallInfo(task=task.value))
 
-        response = (
-            await llm_call(
-                llm,
-                prompt,
-                stop=stop,
-                llm_params={
-                    "temperature": config.lowest_temperature,
-                    "max_tokens": max_tokens,
-                },
-            )
-        ).content
+        llm_response = await llm_call(
+            llm,
+            prompt,
+            stop=stop,
+            llm_params={
+                "temperature": config.lowest_temperature,
+                "max_tokens": max_tokens,
+            },
+        )
+        warn_if_truncated(llm_response, task.value)
+        response = llm_response.content
 
         log.info(f"Output self-checking result is: `{response}`.")
 

tests/integrations/langchain/test_actions_llm_utils.py

@@ -24,6 +24,7 @@
     _stream_llm_call,
     _update_token_stats_from_chunk,
     llm_call,
+    warn_if_truncated,
 )
 from nemoguardrails.context import (
     llm_call_info_var,
@@ -640,3 +641,29 @@ async def test_clears_metadata_var_when_none(self):
         await _stream_llm_call(model, "test", StreamingHandler(), stop=None)
 
         assert llm_response_metadata_var.get() is None
+
+
+class TestWarnIfTruncated:
+    def test_warns_on_empty_content_with_length_finish(self, caplog):
+        from nemoguardrails.types import LLMResponse
+
+        response = LLMResponse(content="", finish_reason="length")
+        with caplog.at_level("WARNING"):
+            warn_if_truncated(response, "self_check_input")
+        assert any("self_check_input" in rec.message and "length" in rec.message for rec in caplog.records)
+
+    def test_silent_on_non_empty_content(self, caplog):
+        from nemoguardrails.types import LLMResponse
+
+        response = LLMResponse(content="yes", finish_reason="length")
+        with caplog.at_level("WARNING"):
+            warn_if_truncated(response, "self_check_input")
+        assert not caplog.records
+
+    def test_silent_on_non_length_finish_reason(self, caplog):
+        from nemoguardrails.types import LLMResponse
+
+        response = LLMResponse(content="", finish_reason="stop")
+        with caplog.at_level("WARNING"):
+            warn_if_truncated(response, "self_check_input")
+        assert not caplog.records