Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 163 additions & 0 deletions apps/backend/qa/fixer.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,169 @@
# =============================================================================


def build_qa_fixer_prompt(
*,
base_prompt: str,
fixer_memory_context: str | None,
failure_patterns: str | None,
spec_dir: Path,
fix_session: int,
) -> str:
"""Assemble the QA fixer prompt (pure, provider-neutral).

Shared by the Claude SDK path and the runtime-adapter path. Fixes land
on disk and success is read back via ``is_fixes_applied`` /
``qa_signoff``, so the same prompt yields the same contract on any
provider. The Claude path keeps its additional per-iteration recovery
guidance; this is the common base both paths build on.
"""
prompt = base_prompt
if fixer_memory_context:
prompt += "\n\n" + fixer_memory_context
if failure_patterns:
prompt += "\n\n" + failure_patterns
prompt += f"\n\n---\n\n**Fix Session**: {fix_session}\n"
prompt += f"**Spec Directory**: {spec_dir}\n"
prompt += f"**Spec Name**: {spec_dir.name}\n"
prompt += f"\n**IMPORTANT**: All spec files are located in: `{spec_dir}/`\n"
prompt += f"The fix request file is at: `{spec_dir}/QA_FIX_REQUEST.md`\n"
return prompt


async def run_qa_fixer_via_runtime(
runtime_session: object,
project_dir: Path,
spec_dir: Path,
fix_session: int,
verbose: bool = False,
) -> tuple[str, str]:
"""Run one QA fixer pass through the provider-neutral runtime layer.

Mirror of :func:`run_qa_fixer_session` for direct-API providers. Unlike
the Claude path it does NOT reimplement the Claude-SDK recovery / model-
fallback loop: it does a single ``generic_edit`` pass (which already
provides mutation snapshots + transaction rollback) and relies on the
outer QA loop for re-validation and retries. Success is the file-based
``is_fixes_applied`` signal, so the verdict matches the Claude path.

Returns ``(status, response_text)`` with ``"fixed"`` / ``"error"``.
"""
from agents.runtime import RuntimeRequirements, run_runtime_session

from .criteria import is_fixes_applied

Check notice

Code scanning / CodeQL

Cyclic import Note

Import of module
backend.qa.criteria
begins an import cycle.

fix_request_file = spec_dir / "QA_FIX_REQUEST.md"
if not fix_request_file.exists():
return "error", "QA_FIX_REQUEST.md not found"

Check failure on line 127 in apps/backend/qa/fixer.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal "QA_FIX_REQUEST.md not found" 3 times.

See more on https://sonarcloud.io/project/issues?id=OBenner_Auto-Coding&issues=AZ59fMdpDJcDccZnBlGh&open=AZ59fMdpDJcDccZnBlGh&pullRequest=277

base_prompt = load_qa_fixer_prompt()
fixer_memory_context = await get_graphiti_context(
spec_dir,
project_dir,
{
"description": "Fixing QA issues and implementing corrections",
"id": f"qa_fixer_{fix_session}",
},
)
fix_request_content = fix_request_file.read_text(encoding="utf-8")
failure_patterns = await get_failure_patterns(
spec_dir,
project_dir,
query=fix_request_content[:500],
failure_types=["qa_rejection", "build_error", "test_failure"],
num_results=5,
min_score=0.5,
)
prompt = build_qa_fixer_prompt(
base_prompt=base_prompt,
fixer_memory_context=fixer_memory_context,
failure_patterns=failure_patterns,
spec_dir=spec_dir,
fix_session=fix_session,
)

result = await run_runtime_session(
runtime_session,
message=prompt,
spec_dir=spec_dir,
verbose=verbose,
requirements=RuntimeRequirements.generic_edit(),
)
response_text = result.response_text or ""

fixer_discoveries = {
"files_understood": {},
"patterns_found": [],
"gotchas_encountered": [],
}
if is_fixes_applied(spec_dir):
debug_success("qa_fixer", "Fixes applied (runtime path)")
fixer_discoveries["patterns_found"].append(
f"QA fixer session {fix_session}: fixes applied via runtime path"
)
await save_session_memory(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=f"qa_fixer_{fix_session}",
session_num=fix_session,
success=True,
subtasks_completed=[f"qa_fixer_{fix_session}"],
discoveries=fixer_discoveries,
)
return "fixed", response_text

debug_error("qa_fixer", "Fixer did not apply fixes (runtime path)")
return "error", "QA fixer did not apply fixes (runtime path)"


async def run_qa_fixer_runtime_session(
*,
provider_name: str,
runtime_mode: str,
model: str,
project_dir: Path,
spec_dir: Path,
fix_session: int,
verbose: bool = False,
) -> tuple[str, str]:
"""Build a direct-provider runtime session and run the QA fixer on it.

Thin orchestration shim mirroring ``run_qa_reviewer_runtime_session``;
builds the provider session + runtime adapter then delegates to
:func:`run_qa_fixer_via_runtime`.
"""
from agents.runtime import create_runtime_session

config = ProviderConfig.from_env(agent_type="qa_fixer")
if config.provider != provider_name:
raise ValueError(
"qa_fixer runtime provider mismatch: "
f"routed={provider_name}, env={config.provider}"
)
provider = create_engine_provider(config)
session = provider.create_session(
SessionConfig(
name=f"qa_fixer-runtime-{fix_session}",
model=model,
extra={"agent_type": "qa_fixer"},
)
)
runtime_session = create_runtime_session(
provider_name=provider_name,
agent_session=session,
runtime_mode=runtime_mode,
project_dir=project_dir,
agent_type="qa_fixer",
)
return await run_qa_fixer_via_runtime(
runtime_session,
project_dir,
spec_dir,
fix_session,
verbose=verbose,
)


async def run_qa_fixer_session(
client: ClaudeSDKClient,
spec_dir: Path,
Expand Down
128 changes: 128 additions & 0 deletions tests/test_qa_fixer_runtime.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""qa_fixer on the provider-neutral runtime layer.

Covers the shared prompt builder (`build_qa_fixer_prompt`) and the
runtime-adapter execution path (`run_qa_fixer_via_runtime`) that lets a
direct-API provider run the QA fixer through `run_runtime_session` instead
of the Claude SDK client. The fixer mutates source, so the runtime path
relies on generic_edit's mutation snapshots + transaction rollback; success
is the file-based `is_fixes_applied` signal, matching the Claude path.
"""

from __future__ import annotations

import contextlib
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from qa.fixer import build_qa_fixer_prompt, run_qa_fixer_via_runtime


def test_build_qa_fixer_prompt_full():
prompt = build_qa_fixer_prompt(
base_prompt="FIXER_BASE",
fixer_memory_context="MEM",
failure_patterns="FAILURES",
spec_dir=Path("/proj/.auto-claude/specs/001-x"),
fix_session=2,
)

assert "FIXER_BASE" in prompt
assert "MEM" in prompt
assert "FAILURES" in prompt
assert "**Fix Session**: 2" in prompt
assert "QA_FIX_REQUEST.md" in prompt
assert "001-x" in prompt # spec name


def test_build_qa_fixer_prompt_minimal():
prompt = build_qa_fixer_prompt(
base_prompt="B",
fixer_memory_context=None,
failure_patterns=None,
spec_dir=Path("/proj/spec"),
fix_session=1,
)

assert prompt.startswith("B")
assert "MEM" not in prompt
assert "**Fix Session**: 1" in prompt


def _patch_fixer_runtime(*, fixes_applied: bool, response_text: str = "fixed-text"):
"""Patch every external dependency of run_qa_fixer_via_runtime."""
fake_result = MagicMock()
fake_result.response_text = response_text
return [
patch("qa.fixer.load_qa_fixer_prompt", return_value="FIXER_PROMPT"),
patch("qa.fixer.get_graphiti_context", new=AsyncMock(return_value="")),
patch("qa.fixer.get_failure_patterns", new=AsyncMock(return_value="")),
patch("qa.fixer.save_session_memory", new=AsyncMock(return_value=None)),
patch(
"agents.runtime.run_runtime_session",
new=AsyncMock(return_value=fake_result),
),
patch("qa.criteria.is_fixes_applied", return_value=fixes_applied),
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
"fixes_applied,expected",
[(True, "fixed"), (False, "error")],
)
async def test_run_qa_fixer_via_runtime_verdict(tmp_path, fixes_applied, expected):
(tmp_path / "QA_FIX_REQUEST.md").write_text("fix these issues", encoding="utf-8")

with contextlib.ExitStack() as stack:
for patcher in _patch_fixer_runtime(fixes_applied=fixes_applied):
stack.enter_context(patcher)
status, response = await run_qa_fixer_via_runtime(
MagicMock(), tmp_path, tmp_path, fix_session=1
)

assert status == expected
if expected == "fixed":
assert response == "fixed-text"


@pytest.mark.asyncio
async def test_run_qa_fixer_via_runtime_missing_fix_request(tmp_path):
"""No QA_FIX_REQUEST.md => error before any runtime session is built."""
run_session = AsyncMock()
with patch("agents.runtime.run_runtime_session", new=run_session):
status, _ = await run_qa_fixer_via_runtime(
MagicMock(), tmp_path, tmp_path, fix_session=1
)

assert status == "error"
run_session.assert_not_awaited()


@pytest.mark.asyncio
async def test_run_qa_fixer_via_runtime_drives_runtime_session(tmp_path):
"""The fixer runs through run_runtime_session with the assembled prompt."""
(tmp_path / "QA_FIX_REQUEST.md").write_text("fix", encoding="utf-8")
fake_result = MagicMock()
fake_result.response_text = "ok"
run_session = AsyncMock(return_value=fake_result)

with (
patch("qa.fixer.load_qa_fixer_prompt", return_value="FIXER_PROMPT"),
patch("qa.fixer.get_graphiti_context", new=AsyncMock(return_value="")),
patch("qa.fixer.get_failure_patterns", new=AsyncMock(return_value="")),
patch("qa.fixer.save_session_memory", new=AsyncMock(return_value=None)),
patch("agents.runtime.run_runtime_session", new=run_session),
patch("qa.criteria.is_fixes_applied", return_value=True),
):
runtime_session = MagicMock()
await run_qa_fixer_via_runtime(
runtime_session, tmp_path, tmp_path, fix_session=4
)

run_session.assert_awaited_once()
args, kwargs = run_session.call_args
assert args[0] is runtime_session
assert kwargs["message"].startswith("FIXER_PROMPT")
assert "**Fix Session**: 4" in kwargs["message"]
assert kwargs["spec_dir"] == tmp_path
Loading