Skip to content

Commit a383a69

Browse files
committed
apply review suggestions
1 parent 949a4f5 commit a383a69

3 files changed

Lines changed: 29 additions & 8 deletions

File tree

nemoguardrails/actions/llm/utils.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -396,23 +396,28 @@ def _extract_content(response: LLMResponse) -> str:
396396
return response.content
397397

398398

399-
def warn_if_truncated(response: LLMResponse, task: str) -> None:
400-
"""Emit a warning if the LLM produced no visible content because it hit the max_tokens budget.
399+
def warn_if_truncated(response: LLMResponse, task: str) -> bool:
400+
"""Return True and emit a warning if the LLM produced no visible content because it hit the max_tokens budget.
401401
402402
Reasoning models (OpenAI o-series, gpt-5, DeepSeek-R1, Gemini 2.5, Qwen QwQ, etc.)
403403
spend output tokens on internal reasoning before emitting visible text. A small
404404
max_tokens budget can be fully consumed by the reasoning phase, leaving empty
405405
content and finish_reason="length". The call succeeds silently and callers that
406-
only inspect response.content see nothing.
406+
only inspect response.content see nothing. Callers whose downstream parser
407+
does not fail safely on empty input (e.g. self_check_facts, whose parser
408+
inverts the result) should use the return value to take an explicit
409+
fail-safe branch.
407410
"""
408-
if not response.content and response.finish_reason == "length":
411+
truncated = not response.content and response.finish_reason == "length"
412+
if truncated:
409413
logger.warning(
410414
"Task %s: LLM returned empty content with finish_reason='length'. "
411415
"The max_tokens budget was likely consumed before any visible output. "
412416
"If using a reasoning model (o1/o3/o4-mini, gpt-5, deepseek-r1, "
413417
"gemini-2.5, etc.), increase the prompt's max_tokens in your config.",
414418
task,
415419
)
420+
return truncated
416421

417422

418423
def get_colang_history(

nemoguardrails/library/self_check/facts/actions.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,11 @@ async def self_check_facts(
7676
stop=stop,
7777
llm_params={"temperature": config.lowest_temperature, "max_tokens": max_tokens},
7878
)
79-
warn_if_truncated(llm_response, task.value)
79+
if warn_if_truncated(llm_response, task.value):
80+
# is_content_safe returns [False] on empty input, which this action
81+
# inverts to "facts are correct". Fail safe instead: treat truncated
82+
# content as a failed fact-check so the output gets blocked.
83+
return 0.0
8084
response = llm_response.content
8185

8286
if llm_task_manager.has_output_parser(task):

tests/integrations/langchain/test_actions_llm_utils.py

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -649,21 +649,33 @@ def test_warns_on_empty_content_with_length_finish(self, caplog):
649649

650650
response = LLMResponse(content="", finish_reason="length")
651651
with caplog.at_level("WARNING"):
652-
warn_if_truncated(response, "self_check_input")
652+
result = warn_if_truncated(response, "self_check_input")
653+
assert result is True
653654
assert any("self_check_input" in rec.message and "length" in rec.message for rec in caplog.records)
654655

655656
def test_silent_on_non_empty_content(self, caplog):
656657
from nemoguardrails.types import LLMResponse
657658

658659
response = LLMResponse(content="yes", finish_reason="length")
659660
with caplog.at_level("WARNING"):
660-
warn_if_truncated(response, "self_check_input")
661+
result = warn_if_truncated(response, "self_check_input")
662+
assert result is False
661663
assert not caplog.records
662664

663665
def test_silent_on_non_length_finish_reason(self, caplog):
664666
from nemoguardrails.types import LLMResponse
665667

666668
response = LLMResponse(content="", finish_reason="stop")
667669
with caplog.at_level("WARNING"):
668-
warn_if_truncated(response, "self_check_input")
670+
result = warn_if_truncated(response, "self_check_input")
671+
assert result is False
672+
assert not caplog.records
673+
674+
def test_silent_on_none_finish_reason(self, caplog):
675+
from nemoguardrails.types import LLMResponse
676+
677+
response = LLMResponse(content="", finish_reason=None)
678+
with caplog.at_level("WARNING"):
679+
result = warn_if_truncated(response, "self_check_input")
680+
assert result is False
669681
assert not caplog.records

0 commit comments

Comments
 (0)