Knowledge Freshness Check #175
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Knowledge Freshness Check | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' # Weekly Monday 6AM UTC | |
| push: | |
| paths: | |
| - 'data/**' | |
| - 'src/tools/deep_knowledge.py' | |
| - 'src/backends/*/backend.py' | |
| pull_request: | |
| # Broad path filter so the lightweight tests gate every PR that | |
| # could affect the catalog, the tooling around it, or the ingest | |
| # pipeline. Pure-docs PRs that touch only README.md / *.md still | |
| # skip CI (documented intent); anything code-touching runs. | |
| paths: | |
| - 'src/**' | |
| - 'scripts/**' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/knowledge-freshness.yml' | |
| workflow_dispatch: # Manual trigger | |
| # Both drift jobs auto-file a GitHub issue on the weekly cron when the | |
| # catalog/fingerprint goes stale. ``issues.create`` needs ``issues: | |
| # write``; the repo's default GITHUB_TOKEN permission is read-only, so | |
| # without this block the issue step 403s -- a failure that only shows | |
| # up the first time real drift is detected (verified by a one-off | |
| # dispatch run that confirmed the issue is created with this grant in | |
| # place). ``contents: read`` is all the checkout needs. | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| catalog-consistency: | |
| # Lightweight: greps backend source for the canonical names the catalog | |
| # promises (currently 4C only; extension pattern in tests/groundtruth/). | |
| # Always runs (PR + push + cron) -- network-only, no solver installs. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install minimal dependencies + introspection backends | |
| # Install the Python-importable backends that have a probe | |
| # in tests/groundtruth/ so their catalog-consistency tests | |
| # actually run in CI rather than skipping. Source-grep | |
| # backends (4C, deal.II) need no install -- they fetch from | |
| # raw.githubusercontent.com. | |
| # | |
| # When adding a new introspection probe (kratos, dune, | |
| # fenics) extend this extra list and the test runs | |
| # automatically on every PR. | |
| run: pip install -e ".[skfem,ngsolve,dev]" | |
| - name: Run lightweight gating tests | |
| # Catalog-consistency + ingest-pipeline tests are the suite of | |
| # checks that need no solver installs. Keep them together in | |
| # one fast job that gates every PR. Tee into a file so the | |
| # issue-creation step can reuse the exact output that produced | |
| # the failure -- avoiding misleading issues if a second pytest | |
| # run produces different output (e.g. cache repopulated, | |
| # network hiccup resolved). | |
| run: | | |
| set -o pipefail | |
| pytest tests/test_catalog_consistency.py tests/test_ingest_session.py tests/test_introspect.py -v -s 2>&1 | tee /tmp/catalog-test.log | |
| # Only swallow failures on the weekly cron so the auto-issue step | |
| # can still run. On PRs and pushes we WANT the workflow to fail | |
| # so the check is gating. | |
| continue-on-error: ${{ github.event_name == 'schedule' }} | |
| id: catalog | |
| - name: Create issue on catalog drift | |
| # Only auto-file an issue on scheduled runs. PR failures fail the | |
| # workflow above and surface through the PR's status check. | |
| if: steps.catalog.outcome == 'failure' && github.event_name == 'schedule' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const output = fs.existsSync('/tmp/catalog-test.log') | |
| ? fs.readFileSync('/tmp/catalog-test.log', 'utf8') | |
| : '(test output not captured)'; | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: '⚠️ Catalog/source drift detected', | |
| body: `Weekly catalog-consistency check failed:\n\n\`\`\`\n${output.slice(-6000)}\n\`\`\`\n\nA backend source change has broken the MCP catalog. Edit the catalog so its parameter keys / DYNAMICTYPE strings / etc. match what the input parser now accepts.`, | |
| labels: ['knowledge-drift'], | |
| }); | |
| fingerprint: | |
| # Only run the heavy ``.[all-solvers,dev]`` install + fingerprint scan | |
| # on push and on the weekly cron. PRs are gated on | |
| # ``catalog-consistency`` only -- adding fingerprint to every PR | |
| # would install all pip-installable solvers (hundreds of MB) on | |
| # ubuntu-latest and frequently exceed the runner timeout, turning a | |
| # heavyweight monitoring job into a noisy PR blocker. | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| pip install -e ".[all-solvers,dev]" | |
| - name: Generate fingerprints | |
| run: python scripts/fingerprint_solvers.py | |
| - name: Compare against saved fingerprints | |
| run: python scripts/fingerprint_solvers.py --compare | |
| continue-on-error: true | |
| id: compare | |
| - name: Create issue on drift | |
| if: steps.compare.outcome == 'failure' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { execSync } = require('child_process'); | |
| const output = execSync('python scripts/fingerprint_solvers.py --compare 2>&1 || true').toString(); | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: '⚠️ Solver API drift detected', | |
| body: `Weekly fingerprint check found API changes:\n\n\`\`\`\n${output}\n\`\`\`\n\nPlease review and update the knowledge base accordingly.`, | |
| labels: ['knowledge-drift'], | |
| }); | |
| - name: Upload fingerprints | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: solver-fingerprints | |
| path: data/fingerprints/ |