diff --git a/README.md b/README.md index 005e14b..a6453be 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,10 @@ To generate the tool-expansion review artifacts in one command: bash demo/run-tool-expansion-review.sh ``` +See [`docs/tutorials/tool-expansion-quality-gate.md`](docs/tutorials/tool-expansion-quality-gate.md) +for the quality-gate flow and [`docs/promo/social-hooks.md`](docs/promo/social-hooks.md) +for grounded launch copy. + ## Development ```bash diff --git a/demo/run-tool-expansion-review.sh b/demo/run-tool-expansion-review.sh index 33d326e..7d5f938 100755 --- a/demo/run-tool-expansion-review.sh +++ b/demo/run-tool-expansion-review.sh @@ -1,58 +1,41 @@ #!/usr/bin/env bash set -euo pipefail -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -OUT_DIR="${TMPDIR:-/tmp}/promptdiff-tool-expansion-demo" -MARKDOWN_REPORT="$OUT_DIR/tool-expansion-report.md" -JSON_REPORT="$OUT_DIR/tool-expansion-report.json" -RULES_REPORT="$OUT_DIR/rules-check.md" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" -rm -rf "$OUT_DIR" -mkdir -p "$OUT_DIR" +npm run build >/dev/null -npm --prefix "$ROOT_DIR" run build >/dev/null +rm -rf .tmp/promptdiff-demo +mkdir -p .tmp/promptdiff-demo -set +e -node "$ROOT_DIR/dist/cli.js" compare \ - "$ROOT_DIR/examples/prompts/tool-expansion-old.md" \ - "$ROOT_DIR/examples/prompts/tool-expansion-new.md" \ - --out "$MARKDOWN_REPORT" -markdown_status=$? - -node "$ROOT_DIR/dist/cli.js" compare \ - "$ROOT_DIR/examples/prompts/tool-expansion-old.md" \ - "$ROOT_DIR/examples/prompts/tool-expansion-new.md" \ +node dist/cli.js compare \ + examples/prompts/tool-expansion-old.md \ + examples/prompts/tool-expansion-new.md \ + --out .tmp/promptdiff-demo/tool-expansion.md + +node dist/cli.js compare \ + examples/prompts/tool-expansion-old.md \ + examples/prompts/tool-expansion-new.md \ --format json \ - --out "$JSON_REPORT" -json_status=$? + --out .tmp/promptdiff-demo/tool-expansion.json + +set +e +node dist/cli.js check examples/prompts/*.md --rules examples/rules.json --fail-on high \ + --out .tmp/promptdiff-demo/rules-check.md +gate_code=$? set -e -if [ "$markdown_status" -ne 0 ] && [ "$markdown_status" -ne 2 ]; then - echo "promptdiff compare failed with exit code $markdown_status" >&2 - exit "$markdown_status" +if [ "$gate_code" -ne 2 ]; then + printf 'expected rules quality gate to exit 2, got %s\n' "$gate_code" >&2 + exit 1 fi -if [ "$json_status" -ne 0 ] && [ "$json_status" -ne 2 ]; then - echo "promptdiff JSON compare failed with exit code $json_status" >&2 - exit "$json_status" -fi +grep -q "Tool" .tmp/promptdiff-demo/tool-expansion.md +grep -q "highestSeverity" .tmp/promptdiff-demo/tool-expansion.json +grep -q "PromptDiff" .tmp/promptdiff-demo/rules-check.md +node -e "const fs=require('node:fs'); const report=JSON.parse(fs.readFileSync('.tmp/promptdiff-demo/tool-expansion.json','utf8')); if (!report.summary || !Array.isArray(report.findings)) process.exit(1);" -node "$ROOT_DIR/dist/cli.js" check \ - "$ROOT_DIR/examples/prompts/safe.md" \ - --rules "$ROOT_DIR/examples/rules.json" \ - --out "$RULES_REPORT" - -test -s "$MARKDOWN_REPORT" -test -s "$JSON_REPORT" -test -s "$RULES_REPORT" -grep -q "Tool surface changed" "$MARKDOWN_REPORT" -grep -q '"highestSeverity": "critical"' "$JSON_REPORT" -grep -q "PromptDiff Rules Check" "$RULES_REPORT" -node -e "const fs=require('node:fs'); const report=JSON.parse(fs.readFileSync(process.argv[1],'utf8')); if (!report.summary || !Array.isArray(report.findings)) process.exit(1);" "$JSON_REPORT" - -echo "Markdown report: $MARKDOWN_REPORT" -echo "JSON report: $JSON_REPORT" -echo "Rules report: $RULES_REPORT" -echo "Expected compare Markdown exit: $markdown_status" -echo "Expected compare JSON exit: $json_status" -find "$OUT_DIR" -maxdepth 1 -type f -print | sort +printf 'Markdown report: .tmp/promptdiff-demo/tool-expansion.md\n' +printf 'JSON report: .tmp/promptdiff-demo/tool-expansion.json\n' +printf 'Rules report: .tmp/promptdiff-demo/rules-check.md\n' diff --git a/docs/promo/social-hooks.md b/docs/promo/social-hooks.md index d61dc21..28c759c 100644 --- a/docs/promo/social-hooks.md +++ b/docs/promo/social-hooks.md @@ -1,26 +1,26 @@ -# Social Hooks +# PromptDiff Promotion Hooks -These drafts are grounded in the current README, examples, and CLI behavior. +## Grounded facts -## Prompt Tool Review +- PromptDiff compares prompt revisions from local files. +- It emits Markdown or JSON reports. +- It redacts common secret-like values by default. +- It includes a `check` command backed by a JSON rules file. +- It is deterministic and heuristic, not an LLM judge. -Prompt edits can quietly change tool access, safety language, and output contracts. +## Short posts -PromptDiff gives those changes names in a local Markdown or JSON report, so reviewers can discuss the actual risk instead of eyeballing a wall of text. +1. Prompt changes can expand tool access without looking dramatic. PromptDiff + turns that revision into a local review artifact. +2. Treat prompts like code: compare revisions, name risky categories, and keep + a Markdown report with the PR. +3. Demo angle: old support prompt, new tool-expanded prompt, one report showing + what changed and one rules gate that exits non-zero. -Demo: compare `examples/prompts/tool-expansion-old.md` with `examples/prompts/tool-expansion-new.md`. +## Video outline -## CI Angle - -PromptDiff has two useful modes: - -- `compare` for prompt revision reports -- `check` for required phrases, forbidden phrases, and section rules - -It is deterministic, local-first, and built for review evidence rather than scoring prompts with another model. - -## Limitation-Aware Post - -PromptDiff is not an LLM judge and does not claim to understand every semantic change. - -That is the point: it catches concrete review signals such as risky instruction language, removed guardrails, tool references, output-contract shifts, and secret-like values. +1. Open `examples/prompts/tool-expansion-old.md`. +2. Open `examples/prompts/tool-expansion-new.md`. +3. Run `bash demo/run-tool-expansion-review.sh`. +4. Show `.tmp/promptdiff-demo/tool-expansion.md`. +5. Show the expected quality-gate exit behavior from the script. diff --git a/docs/tutorials/tool-expansion-quality-gate.md b/docs/tutorials/tool-expansion-quality-gate.md new file mode 100644 index 0000000..040de64 --- /dev/null +++ b/docs/tutorials/tool-expansion-quality-gate.md @@ -0,0 +1,54 @@ +# Tool Expansion Quality Gate Demo + +This walkthrough creates reviewer-facing artifacts for a prompt change that +expands tool language and then runs the checked-in rules gate. + +## Build the CLI + +```sh +npm run build +``` + +## Compare the prompt revision + +```sh +node dist/cli.js compare \ + examples/prompts/tool-expansion-old.md \ + examples/prompts/tool-expansion-new.md \ + --out .tmp/promptdiff-demo/tool-expansion.md +``` + +Write JSON for automation: + +```sh +node dist/cli.js compare \ + examples/prompts/tool-expansion-old.md \ + examples/prompts/tool-expansion-new.md \ + --format json \ + --out .tmp/promptdiff-demo/tool-expansion.json +``` + +## Run the rules gate + +```sh +node dist/cli.js check examples/prompts/*.md --rules examples/rules.json --fail-on high \ + --out .tmp/promptdiff-demo/rules-check.md +``` + +Exit code `2` means the configured prompt quality gate failed. In this demo, +that is expected evidence for the risky fixture set, not a runtime error. + +## One-command demo + +```sh +bash demo/run-tool-expansion-review.sh +``` + +The script writes Markdown and JSON artifacts, verifies expected report text, +and confirms the rules gate exits with code `2`. + +## Boundaries + +- PromptDiff is deterministic and heuristic; it is not an LLM judge. +- Reports are review artifacts, not final safety decisions. +- Do not claim hosted scanning, telemetry, or network behavior.