diff --git a/.github/workflows/docker-security.yml b/.github/workflows/docker-security.yml index 42cd075dc..84e9aef2e 100644 --- a/.github/workflows/docker-security.yml +++ b/.github/workflows/docker-security.yml @@ -12,6 +12,11 @@ on: # two, editing the cutoff or adding a suppression ships without ever # running the job it modifies. WOR-852. - '.github/workflows/docker-security.yml' + # The RELEASE gate too. publish-docker.yml only triggers on `v*` tags, + # so without this a change to it is exercised by nothing until a real + # release — WOR-871 landed its whole release-gate change on a PR with + # 36 green checks and zero scans among them. + - '.github/workflows/publish-docker.yml' - '.grype*.yaml' # NOT covered by the glob above: Actions path globs do not cross `/`. # grype reads this second location too, and it is the exact blind spot @@ -38,6 +43,11 @@ on: paths: # See the push filter above — a change to the gate must exercise it. - '.github/workflows/docker-security.yml' + # The RELEASE gate too. publish-docker.yml only triggers on `v*` tags, + # so without this a change to it is exercised by nothing until a real + # release — WOR-871 landed its whole release-gate change on a PR with + # 36 green checks and zero scans among them. + - '.github/workflows/publish-docker.yml' - '.grype*.yaml' # NOT covered by the glob above: Actions path globs do not cross `/`. # grype reads this second location too, and it is the exact blind spot diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 914cf4c7c..b2f74db53 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -18,6 +18,30 @@ on: tags: - "v*" +# NO `workflow_dispatch`. It was added in WOR-871 as break-glass for a blocked +# release, then removed in the same PR once reviewed: +# +# - GitHub runs the workflow FROM the dispatched ref, so a branch dispatch +# supplies both the Dockerfile and this file — a branch could delete the +# release gate and still publish under this repo's OIDC identity. +# - `type=semver` yields no tags on a non-tag ref, so the build, the push by +# digest and `cosign sign` all SUCCEED and only promotion fails, leaving a +# signed orphan digest whose Fulcio SAN is `@refs/heads/...` — not the +# `@refs/tags/v.*` the verify command we document pins against. +# +# IF THE GATE BLOCKS A RELEASE, the recovery is NOT "re-run the job" — a re-run +# replays the identical tree at the tag's SHA, so a lapsed expiry or a newly +# published CVE fails again, deterministically. Re-run only helps a transient +# failure (e.g. the cosign retry path below). The real path forward: +# +# 1. Fix the finding: bump the pinned base image, or add a dated, argued +# entry to .grype.yaml (see the reachability arguments already there). +# 2. Delete the tag, re-push it at the new commit. That keeps the trigger, +# and therefore the Fulcio SAN, on `refs/tags/v*` where verify expects it. +# +# Doing this under release pressure is exactly how a suppression gets +# rubber-stamped, so prefer fixing the base image over widening the waiver. + # Workflow-scoped minimum. Per-job permissions widen as needed. permissions: contents: read @@ -91,6 +115,31 @@ jobs: flavor: | latest=auto + # Both release-gate scans below read .grype.yaml and honour every + # suppression in it. Grype silently drops the unknown `expiry` key, so + # without this step a lapsed suppression keeps suppressing forever on + # the one path that reaches users. docker-security.yml has run this + # since WOR-852; the release path did not. WOR-871. + # + # Placed BEFORE the builds, on the same principle the cosign preflight + # states above: this parses ~50 lines of YAML, so a lapsed date should + # fail in seconds rather than after 25 minutes of QEMU arm64 build. + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + # Caching is on by default and this workflow PUBLISHES a signed + # artifact — a poisoned cache would reach the image we cosign. + # zizmor flags it High here and is right to. This step parses ~50 + # lines of YAML, so there is nothing worth caching anyway. + enable-cache: false + - name: Grype ignores are dated and current + # Version-pinned deliberately. This job holds `id-token: write` and + # `packages: write`, so an unpinned resolve would let a malicious + # release of this dependency run with the signing token. Every action + # here is SHA-pinned; this dependency gets the same treatment. + # --no-project: parsing one small YAML file does not justify syncing + # the whole project venv. + run: uv run --no-project --with 'pyyaml==6.0.3' python scripts/hooks/check_grype_ignore_expiry.py + # Scan-before-push gate: amd64 loads to daemon, arm64 writes tarball # (foreign-arch daemon load is unreliable across runner images). - name: Build amd64 image locally for vulnerability scan @@ -108,14 +157,24 @@ jobs: # compromise of aquasecurity/trivy-action in Feb-Mar 2026. Different # vendor, different codebase, different CI pipeline — limits blast # radius of any one scanner repo being compromised. Same archetype: - # OSS, OS-pkg + language-dep coverage, fails on fixable CRITICAL. - - name: Grype release-gate amd64 (fail on critical) + # OSS, OS-pkg + language-dep coverage. + # + # Cutoff MATCHES docker-security.yml. It sat at `critical` while the PR + # gate moved to `medium`, so a fixable High could be published to users + # while a Medium blocked a branch — the strict gate guarding the + # proposal, the loose one guarding the artifact. WOR-871. + - name: Grype release-gate amd64 (fail on MEDIUM+) uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 with: image: worthless-proxy:scan-${{ github.sha }} - severity-cutoff: critical + severity-cutoff: medium only-fixed: true fail-build: true + # The action defaults to sarif, writes it to a file, and echoes the + # report to the log ONLY for `table`. Nothing here uploads that + # file, so without this a green release reports what it checked: + # nothing at all. Same fix as docker-security.yml. WOR-871. + output-format: table # arm64-specific CVEs (different Debian slim package variants) would # otherwise ship unscanned. Build arm64 to a local OCI tarball (more @@ -133,15 +192,47 @@ jobs: tags: worthless-proxy:scan-arm64-${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} - - name: Grype release-gate arm64 (fail on critical) + - name: Grype release-gate arm64 (fail on MEDIUM+) uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 with: # docker-archive: scheme tells Grype to read the tarball directly # (via its internal libs). No daemon load needed for foreign arch. image: docker-archive:${{ env.ARM64_TAR }} - severity-cutoff: critical + severity-cutoff: medium only-fixed: true fail-build: true + output-format: table # see the amd64 step above + + # Informational, never blocking. The two gates above honour .grype.yaml, + # so a suppressed CVE is skipped by the gate AND filtered out of the + # gate's report — it would appear in NO release output at all, and a + # green release log would read "nothing found" while seven documented + # CVEs shipped. This scan reads the ignore-free config so the release + # record states what we chose to carry. Mirrors docker-security.yml. + - name: Grype full scan amd64 (including unfixed — informational) + uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 + env: + GRYPE_CONFIG: .grype-informational.yaml + with: + image: worthless-proxy:scan-${{ github.sha }} + only-fixed: false + fail-build: false + output-format: table + + # arm64 needs its own informational pass. The gate above reads the arm64 + # tarball but honours .grype.yaml and `only-fixed`, so an arm64-only + # suppressed or unfixable CVE would appear in NO release output — the + # amd64 informational scan cannot see it. Both arches ship; both arches + # get a record of what they carry. + - name: Grype full scan arm64 (including unfixed — informational) + uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 + env: + GRYPE_CONFIG: .grype-informational.yaml + with: + image: docker-archive:${{ env.ARM64_TAR }} + only-fixed: false + fail-build: false + output-format: table # Sign-before-tag invariant: we push by DIGEST ONLY (no human-readable # tags) so the image exists at sha256:... but is not yet pullable via diff --git a/docs/install-docker.md b/docs/install-docker.md index 4faced5fd..bc905fbf2 100644 --- a/docs/install-docker.md +++ b/docs/install-docker.md @@ -5,7 +5,9 @@ description: "Pull a pre-built, signed multi-arch image from GHCR." # Install -- Docker (from GHCR) -Pull a pre-built, multi-arch image from the GitHub Container Registry. No clone, no build. Every image is vulnerability-scanned with [Grype](https://github.com/anchore/grype) on both architectures and signed with cosign before publish. +Pull a pre-built, multi-arch image from the GitHub Container Registry. No clone, no build. Every image is scanned with [Grype](https://github.com/anchore/grype) on both architectures, and cosign-signed before any release tag is promoted. The release fails on any **fixable Medium-or-higher** vulnerability — the same bar a pull request has to clear. + +"Scanned" is not "zero known CVEs". The image carries a small number of CPython vulnerabilities inherited from the base image that have no stable upstream fix, and each one is listed in [`.grype.yaml`](https://github.com/shacharm2/worthless/blob/main/.grype.yaml) with a written argument for why it is not reachable from the proxy and a date by which it must be re-examined. Read that file if you want to check our reasoning rather than take our word for it. ```bash docker run -d --name worthless -p 127.0.0.1:8787:8787 \ diff --git a/tests/test_grype_ignore_expiry.py b/tests/test_grype_ignore_expiry.py index 6193f1169..ec45ad64e 100644 --- a/tests/test_grype_ignore_expiry.py +++ b/tests/test_grype_ignore_expiry.py @@ -18,6 +18,7 @@ REPO = Path(__file__).resolve().parents[1] HOOK = REPO / "scripts" / "hooks" / "check_grype_ignore_expiry.py" WORKFLOW = REPO / ".github" / "workflows" / "docker-security.yml" +PUBLISH_WORKFLOW = REPO / ".github" / "workflows" / "publish-docker.yml" INFORMATIONAL_CONFIG = REPO / ".grype-informational.yaml" @@ -247,3 +248,153 @@ def test_the_scan_reruns_when_its_own_config_changes() -> None: # the nested location — the same blind spot CONFIGS covers above. An # ignore parked there would otherwise never re-trigger the scan. assert ".grype/config.yaml" in paths, f"{event}: nested grype config not in paths filter" + # The release workflow triggers only on `v*` tags, so it is exercised + # by nothing on a PR unless the scan job watches it. WOR-871 shipped + # its entire release-gate change with 36 green checks and no scan + # among them; this makes that impossible to repeat. + assert ".github/workflows/publish-docker.yml" in paths, ( + f"{event}: release workflow changes run no scan" + ) + + +# --- the RELEASE gate (WOR-871) -------------------------------------------- +# Everything above guards the PR path. The artifact users actually `docker +# pull` is gated by a different workflow, and nothing kept the two in step. + + +# Grype severities, loosest gate first. A cutoff further right blocks more. +_STRICTNESS = ("negligible", "low", "medium", "high", "critical") + + +def _scan_step_images(workflow: Path) -> list[str]: + """The `image:` each build-failing scan points at, in file order.""" + wf = yaml.safe_load(workflow.read_text()) + return [ + str(s.get("with", {}).get("image", "")) + for j in wf["jobs"].values() + for s in j.get("steps", []) + if "anchore/scan-action" in str(s.get("uses", "")) + and s.get("with", {}).get("fail-build") is True + ] + + +def _gate_cutoffs(workflow: Path) -> list[str]: + """Every build-failing anchore/scan-action cutoff in a workflow, lowercased. + + grype's --fail-on is case-insensitive, so `Medium` is valid config; compare + normalised or a legal value blows up with a ValueError instead of a verdict. + """ + wf = yaml.safe_load(workflow.read_text()) + cutoffs = [] + for job in wf["jobs"].values(): + for step in job.get("steps", []): + with_ = step.get("with", {}) + if "anchore/scan-action" not in str(step.get("uses", "")): + continue + if with_.get("fail-build") is not True: + continue + cutoff = with_.get("severity-cutoff") + assert cutoff, f"{workflow.name}: build-failing scan step has no severity-cutoff" + cutoffs.append(str(cutoff).lower()) + return cutoffs + + +def test_the_release_gate_is_never_looser_than_the_pr_gate() -> None: + """The published image must not be held to a weaker standard than a branch. + + A PR is a proposal; the GHCR image is what users run against real keys. + Gating the proposal harder than the artifact is backwards, and it drifted + that way silently — WOR-852 tightened the PR gate and nothing flagged that + the release gate had been left two tiers behind. + """ + pr = _gate_cutoffs(WORKFLOW) + release = _gate_cutoffs(PUBLISH_WORKFLOW) + assert pr, "no build-failing scan step in the PR workflow" + # Arity, not just presence: the release path scans amd64 AND arm64, and + # deleting one of those steps would otherwise leave this test green while + # an entire architecture ships unscanned — the exact class of silent drift + # this suite exists to catch. + assert len(release) == 2, f"expected 2 release-gate scans (amd64, arm64), found {len(release)}" + # Arity alone is gameable: duplicate the amd64 step, delete arm64, and a + # count-only assertion stays green while an architecture ships unscanned. + # The two scans must point at DIFFERENT images. + images = _scan_step_images(PUBLISH_WORKFLOW) + assert len(set(images)) == 2, f"both release scans point at the same image: {images}" + archives = [i for i in images if i.startswith("docker-archive:")] + assert archives, "no release scan reads a tarball — arm64 is unscanned" + # A `docker-archive:` input proves a tarball, NOT an architecture. Two + # amd64 builds with one exported to a tarball would satisfy everything + # above while the arm64 image users pull has no gate at all. Trace the + # tarball back to the step that produced it and check what it built. + wf = yaml.safe_load(PUBLISH_WORKFLOW.read_text()) + steps = [s for j in wf["jobs"].values() for s in j.get("steps", [])] + producers = [ + s + for s in steps + if "build-push-action" in str(s.get("uses", "")) + and "ARM64_TAR" in str(s.get("with", {}).get("outputs", "")) + ] + assert producers, "nothing produces the scanned tarball" + for step in producers: + assert step["with"]["platforms"] == "linux/arm64", ( + f"the scanned tarball is built for {step['with']['platforms']}, not arm64" + ) + weakest_pr = min(_STRICTNESS.index(c) for c in pr) + for cutoff in release: + assert _STRICTNESS.index(cutoff) <= weakest_pr, ( + f"release gate `{cutoff}` is looser than the PR gate `{_STRICTNESS[weakest_pr]}`" + ) + + +def test_the_release_gate_enforces_ignore_expiry() -> None: + """Suppressions are time-boxed only where something checks the date. + + Grype drops the unknown `expiry` key silently, so the hook is the sole + enforcement. The PR gate runs it; without this the release path honours + every suppression in .grype.yaml with nothing policing whether they have + lapsed — on the one path that reaches users. + """ + # Structural, not a substring grep on the file text: `if: false`, + # `continue-on-error: true`, or commenting the step out must all fail this. + wf = yaml.safe_load(PUBLISH_WORKFLOW.read_text()) + steps = [s for j in wf["jobs"].values() for s in j.get("steps", [])] + hook = [ + i + for i, s in enumerate(steps) + if "check_grype_ignore_expiry.py" in str(s.get("run", "")) + and s.get("if") is None + and s.get("continue-on-error") is not True + ] + assert hook, "release workflow never enforces .grype.yaml expiry dates" + # Existing is not enough — it has to run BEFORE the scans it protects. + # Placed after them, a lapsed suppression is still honoured by every scan + # and the hook only reports it once the gate has already passed. + scans = [i for i, s in enumerate(steps) if "anchore/scan-action" in str(s.get("uses", ""))] + assert hook[0] < min(scans), "expiry check runs after the scans it is supposed to guard" + + +def test_the_release_workflow_cannot_be_triggered_by_hand() -> None: + """`workflow_dispatch` here is a publishing hole, not a break-glass. + + GitHub runs the workflow FROM the dispatched ref, so a branch dispatch + supplies both the Dockerfile and this workflow — a branch could delete the + release gate and still publish under the repo's OIDC identity. And because + `type=semver` yields no tags on a non-tag ref, the build, the push by + digest and `cosign sign` all SUCCEED; only promotion fails, leaving a + signed orphan digest whose Fulcio SAN is `@refs/heads/...` rather than the + `@refs/tags/v.*` our documented verify command pins. + + Added and removed inside WOR-871; this keeps it from coming back. + + An ALLOWLIST, not a `workflow_dispatch not in ...` denylist. `workflow_call` + (a callee inherits the caller's ref), `repository_dispatch`, `schedule`, or + `push: branches:` each reopen the same hole while leaving that one string + absent. Tag-push is the only way this workflow may ever start. + """ + triggers = yaml.safe_load(PUBLISH_WORKFLOW.read_text())[True] + assert set(triggers) == {"push"}, ( + f"release workflow must trigger ONLY on push; found {sorted(triggers)}" + ) + assert set(triggers["push"]) == {"tags"}, ( + f"release workflow must trigger only on TAG push; found {sorted(triggers['push'])}" + )