improvements

Author: AlexanderLanin

.github/workflows/consumer_test.yml

@@ -42,14 +42,13 @@ jobs:
           egress-policy: audit
 
       - name: Checkout PR
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@v6.0.2
 
       - name: Prepare Python
-        run: |
-          bazel run //:ide_support
+        run: bazel run //:ide_support
 
       - name: Run Consumer tests
-        run: .venv_docs/bin/python -m pytest -vv src/tests/ -k "$CONSUMER"
+        run: .venv_docs/bin/python -m pytest -vv -s src/tests/test_consumer.py -k "$CONSUMER"
         env:
           FORCE_COLOR: "1"
           TERM: xterm-256color

src/tests/test_consumer.py

@@ -13,31 +13,25 @@
 """
 Consumer tests: verify that downstream repos build successfully against this branch.
 
-Run all tests:
+Via Python (requires ide_support to have been run first):
+    bazel run //:ide_support  # once, to set up .venv_docs
     python -m pytest src/tests/test_consumer.py -s
-
-Filter by repo or override type (pytest -k):
     python -m pytest src/tests/test_consumer.py -k "score and local"
     python -m pytest src/tests/test_consumer.py -k "process_description"
-
-Use a temp dir instead of the persistent cache:
     python -m pytest src/tests/test_consumer.py --disable-cache
 
 Known non-passing tests are sometimes marked xfail and do not count as failures.
 """
 
-import atexit
-import random
 import re
 import shutil
 import subprocess
+import warnings
 from dataclasses import dataclass
 from pathlib import Path
 
 import pytest
-from _pytest.config import Config
 from pytest import TempPathFactory
-from rich.console import Console
 
 from src.helper_lib import find_git_root, get_current_git_hash, get_github_base_url
 
@@ -46,9 +40,6 @@
 # ---------------------------------------------------------------------------
 
 CACHE_DIR = Path.home() / ".cache" / "docs_as_code_consumer_tests"
-_log_fp = open("consumer_test.log", "a", encoding="utf-8")  # noqa: SIM115
-atexit.register(_log_fp.close)
-_console = Console(file=_log_fp, force_terminal=False, width=120, color_system=None)
 
 # ---------------------------------------------------------------------------
 # Data model
@@ -130,7 +121,7 @@ def _strip_score_docs_overrides(content: str) -> str:
     return "\n".join(result) + ("\n" if content.endswith("\n") else "")
 
 
-_BAZEL_DEP_PATTERN = r'bazel_dep\(name = "score_docs_as_code"(?:, version = "[^"]+")?\)'
+_BAZEL_DEP_PATTERN = r"""^bazel_dep\(name = ["']score_docs_as_code["'](?:, version = ["'][^"']+["'])?\)"""
 
 
 def _write_module_bazel(
@@ -155,20 +146,20 @@ def _write_module_bazel(
 )"""
 
     (repo_path / "MODULE.bazel").write_text(
-        re.sub(_BAZEL_DEP_PATTERN, replacement, base), encoding="utf-8"
+        re.sub(_BAZEL_DEP_PATTERN, replacement, base, flags=re.MULTILINE), encoding="utf-8"
     )
 
 
 # ---------------------------------------------------------------------------
 # Repo helpers
 # ---------------------------------------------------------------------------
 
-_cloned: set[str] = set()
+_cloned: set[Path] = set()
 
 
 def _ensure_repo(repo_path: Path, git_url: str, use_cache: bool) -> None:
-    """Clone or update a repo exactly once per session per repo name."""
-    if repo_path.name in _cloned:
+    """Clone or update a repo exactly once per session per path."""
+    if repo_path in _cloned:
         return
     if repo_path.exists():
         if use_cache:
@@ -199,23 +190,25 @@ def _ensure_repo(repo_path: Path, git_url: str, use_cache: bool) -> None:
             capture_output=True,
             cwd=repo_path.parent,
         )
-    _cloned.add(repo_path.name)
+    _cloned.add(repo_path)
 
 
 def _cleanup_before_cmd(cwd: Path, cmd: str) -> None:
+    # ubproject.toml is created by :docs
     for p in cwd.glob("*/ubproject.toml"):
         p.unlink()
+
+    # _build is created by :docs
     shutil.rmtree(cwd / "_build", ignore_errors=True)
+
+    # for ide_support, also clear the venv and bazel cache to ensure a clean slate
     if cmd == "bazel run //:ide_support":
         shutil.rmtree(cwd / ".venv_docs", ignore_errors=True)
-        subprocess.run(["bazel", "clean", "--async"], text=True, cwd=cwd)
+        subprocess.run(["bazel", "clean", "--async"], check=True, text=True, cwd=cwd)
 
 
 def _run_bazel_cmd(cmd: str, repo_name: str, cwd: Path) -> None:
-    """Stream a bazel command to the log file; fail on non-zero exit or warnings."""
-    _console.print(f"\n[cyan]{'=' * 80}[/cyan]")
-    _console.print(f"[cornflower_blue]{repo_name}: {cmd}[/cornflower_blue]")
-
+    """Stream a bazel command to stdout; fail on non-zero exit, warn on WARNING lines."""
     process = subprocess.Popen(
         cmd.split(),
         stdout=subprocess.PIPE,
@@ -226,16 +219,12 @@ def _run_bazel_cmd(cmd: str, repo_name: str, cwd: Path) -> None:
     )
 
     assert process.stdout is not None
-    for line in iter(process.stdout.readline, ""):
-        _console.print(line.rstrip())
+    for line in process.stdout:
+        print(line, end="")
         if "WARNING" in line:
-            process.terminate()
-            process.wait()
-            pytest.fail(
-                f"Unexpected warning in {repo_name} `{cmd}`: {line.strip()}",
-                pytrace=False,
-            )
+            warnings.warn(f"{repo_name} `{cmd}`: {line.strip()}", stacklevel=2)
     process.stdout.close()
+
     rc = process.wait()
     if rc != 0:
         pytest.fail(
@@ -250,20 +239,20 @@ def _run_bazel_cmd(cmd: str, repo_name: str, cwd: Path) -> None:
 
 
 @pytest.fixture(scope="module")
-def sphinx_base_dir(tmp_path_factory: TempPathFactory, pytestconfig: Config) -> Path:
+def repos_base_dir(tmp_path_factory: TempPathFactory, pytestconfig: pytest.Config) -> Path:
     if pytestconfig.getoption("--disable-cache"):
         return tmp_path_factory.mktemp("consumer_tests")
     CACHE_DIR.mkdir(parents=True, exist_ok=True)
     return CACHE_DIR
 
 
 @pytest.fixture(scope="module")
-def consumer_env(sphinx_base_dir: Path) -> tuple[str, str]:
+def consumer_env(repos_base_dir: Path) -> tuple[str, str]:
     """Resolve git metadata and set up the local symlink used by local_path_override."""
     git_root = find_git_root()
     assert git_root, "Git root not found"
 
-    dest = sphinx_base_dir / "docs_as_code"
+    dest = repos_base_dir / "docs_as_code"
     if dest.is_symlink():
         dest.unlink()
     elif dest.is_dir():
@@ -279,7 +268,7 @@ def consumer_env(sphinx_base_dir: Path) -> tuple[str, str]:
 
 
 def _check_remote_available() -> tuple[bool, str]:
-    """Return (available, skip_reason). Called once at module import."""
+    """Return (available, reason). Called once at module import."""
     git_root = find_git_root()
     if git_root is None:
         return False, "git root not found"
@@ -329,12 +318,13 @@ def _make_params() -> list[pytest.param]:  # type: ignore[type-arg]
         for override in ("local", "remote"):
             for cmd in repo.commands:
                 marks = []
-                if override == "remote" and not _REMOTE_AVAILABLE:
-                    marks.append(pytest.mark.skip(reason=_REMOTE_SKIP_REASON))
-                if repo.name == "module_template" and random.choice([True, False]):
+                if (
+                    repo.name == "module_template"
+                    and cmd == "bazel build //:needs_json"
+                ):
                     marks.append(
                         pytest.mark.xfail(
-                            reason="module_template is currently broken",
+                            reason="needs_json in module_template is currently broken",
                             strict=False,
                         )
                     )
@@ -355,14 +345,17 @@ def test_consumer_repo(
     repo: ConsumerRepo,
     override_type: str,
     cmd: str,
-    sphinx_base_dir: Path,
+    repos_base_dir: Path,
     consumer_env: tuple[str, str],
-    pytestconfig: Config,
+    pytestconfig: pytest.Config,
 ) -> None:
+    if override_type == "remote" and not _REMOTE_AVAILABLE:
+        pytest.fail(_REMOTE_SKIP_REASON, pytrace=False)
+
     gh_url, current_hash = consumer_env
     use_cache = not bool(pytestconfig.getoption("--disable-cache"))
 
-    repo_path = sphinx_base_dir / repo.name
+    repo_path = repos_base_dir / repo.name
     _ensure_repo(repo_path, repo.git_url, use_cache)
     _write_module_bazel(repo_path, override_type, current_hash, gh_url)
     _cleanup_before_cmd(repo_path, cmd)