Skip to content

Commit 631ecc6

Browse files
anticomputeranticomputer
committed
Add configurable advisory state, demo script
Both pvr_triage and pvr_triage_batch now accept a 'state' global (default: triage). Pass state=draft when testing with owner-created advisories, which land in draft state rather than triage. Add scripts/demo_pvr_triage.sh for live testing against anticomputer/vulnerable-test-app. Subcommands: tools - test MCP tools against live API (no AI calls) batch - run batch scoring taskflow triage - run full single-advisory triage all - everything in sequence Uses gh auth token for GitHub API and passage for AI endpoint token.
1 parent a36d9b3 commit 631ecc6

3 files changed

Lines changed: 282 additions & 4 deletions

File tree

‎scripts/demo_pvr_triage.sh‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
#!/bin/bash
2+
# SPDX-FileCopyrightText: GitHub, Inc.
3+
# SPDX-License-Identifier: MIT
4+
#
5+
# Live demo of PVR triage taskflows against anticomputer/vulnerable-test-app.
6+
#
7+
# Exercises: advisory listing, dedup detection, security policy fetch,
8+
# code verification, report generation, and batch scoring.
9+
#
10+
# Prerequisites:
11+
# - gh CLI authenticated
12+
# - passage available for AI token
13+
# - seclab-taskflows installed in .venv
14+
#
15+
# Usage:
16+
# ./scripts/demo_pvr_triage.sh [tools|batch|triage|all]
17+
#
18+
# tools - test individual MCP tools against live API (fast, no AI calls)
19+
# batch - run the batch scoring taskflow
20+
# triage - run full single-advisory triage on the high-quality report
21+
# all - run everything in sequence
22+
23+
set -euo pipefail
24+
25+
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
26+
__root="$(cd "${__dir}/.." && pwd)"
27+
28+
REPO="anticomputer/vulnerable-test-app"
29+
# Advisory state: "draft" for owner-created test advisories,
30+
# "triage" for real PVR submissions from external reporters.
31+
ADVISORY_STATE="${ADVISORY_STATE:-draft}"
32+
33+
# --- environment ---
34+
35+
if [ -d "${__root}/.venv/bin" ]; then
36+
export PATH="${__root}/.venv/bin:${PATH}"
37+
fi
38+
39+
export GH_TOKEN="${GH_TOKEN:-$(gh auth token 2>/dev/null)}"
40+
if [ -z "${GH_TOKEN}" ]; then
41+
echo "FATAL: gh auth token failed. Run: gh auth login" >&2
42+
exit 1
43+
fi
44+
45+
export AI_API_TOKEN="${AI_API_TOKEN:-$(passage show github/capi-token 2>/dev/null)}"
46+
if [ -z "${AI_API_TOKEN}" ]; then
47+
echo "FATAL: AI_API_TOKEN not set and passage unavailable." >&2
48+
exit 1
49+
fi
50+
51+
export AI_API_ENDPOINT="${AI_API_ENDPOINT:-https://api.githubcopilot.com}"
52+
export REPORT_DIR="${REPORT_DIR:-${__root}/reports/demo}"
53+
export LOG_DIR="${LOG_DIR:-${__root}/logs}"
54+
mkdir -p "${REPORT_DIR}" "${LOG_DIR}"
55+
56+
# --- helpers ---
57+
58+
sep() { echo; echo "========== $1 =========="; echo; }
59+
ok() { echo "[OK] $1"; }
60+
fail() { echo "[FAIL] $1" >&2; FAILURES=$((FAILURES + 1)); }
61+
62+
FAILURES=0
63+
64+
run_agent() {
65+
python -m seclab_taskflow_agent "$@"
66+
}
67+
68+
# --- tools: test individual MCP tools against live API ---
69+
70+
cmd_tools() {
71+
sep "MCP Tool Tests (live API, no AI calls)"
72+
73+
echo "--- list_pvr_advisories (state=draft) ---"
74+
ADVISORIES=$(python -c "
75+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
76+
print(pvr.list_pvr_advisories.fn(owner='anticomputer', repo='vulnerable-test-app', state='draft'))
77+
")
78+
COUNT=$(echo "$ADVISORIES" | python -c "import sys,json; print(len(json.load(sys.stdin)))")
79+
if [ "$COUNT" -ge 1 ]; then
80+
ok "Found $COUNT advisories in draft state"
81+
else
82+
fail "No advisories found. Create test advisories first."
83+
return
84+
fi
85+
echo "$ADVISORIES" | python -c "
86+
import sys, json
87+
for a in json.load(sys.stdin):
88+
print(f\" {a['ghsa_id']} {a['severity']:8s} {a['summary']}\")
89+
"
90+
echo
91+
92+
echo "--- fetch_pvr_advisory (first advisory) ---"
93+
GHSA=$(echo "$ADVISORIES" | python -c "import sys,json; print(json.load(sys.stdin)[0]['ghsa_id'])")
94+
DETAIL=$(python -c "
95+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
96+
print(pvr.fetch_pvr_advisory.fn(owner='anticomputer', repo='vulnerable-test-app', ghsa_id='${GHSA}'))
97+
")
98+
if echo "$DETAIL" | python -c "import sys,json; d=json.load(sys.stdin); assert d['ghsa_id']" 2>/dev/null; then
99+
ok "Fetched ${GHSA}: $(echo "$DETAIL" | python -c "import sys,json; d=json.load(sys.stdin); print(f\"{d['severity']} - CWEs: {d['cwes']}\")")"
100+
else
101+
fail "Failed to fetch ${GHSA}"
102+
fi
103+
echo
104+
105+
echo "--- fetch_security_policy ---"
106+
POLICY=$(python -c "
107+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
108+
print(pvr.fetch_security_policy.fn(owner='anticomputer', repo='vulnerable-test-app'))
109+
")
110+
if [ -n "$POLICY" ]; then
111+
ok "Security policy found ($(echo "$POLICY" | wc -l | tr -d ' ') lines)"
112+
echo "$POLICY" | head -5 | sed 's/^/ /'
113+
echo " ..."
114+
else
115+
fail "No security policy found"
116+
fi
117+
echo
118+
119+
echo "--- compare_advisories (dedup detection) ---"
120+
DEDUP=$(python -c "
121+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
122+
print(pvr.compare_advisories.fn(owner='anticomputer', repo='vulnerable-test-app', state='draft', target_ghsa=''))
123+
")
124+
CLUSTERS=$(echo "$DEDUP" | python -c "import sys,json; print(len(json.load(sys.stdin)['clusters']))")
125+
TOTAL=$(echo "$DEDUP" | python -c "import sys,json; print(json.load(sys.stdin)['total'])")
126+
ok "Compared $TOTAL advisories, found $CLUSTERS duplicate cluster(s)"
127+
echo "$DEDUP" | python -c "
128+
import sys, json
129+
d = json.load(sys.stdin)
130+
for c in d['clusters']:
131+
print(f\" Cluster [{c['match_level']}]: {', '.join(c['advisories'])}\")
132+
for r in c['reasons']:
133+
print(f\" - {r}\")
134+
for s in d['singles']:
135+
print(f\" Single: {s}\")
136+
"
137+
echo
138+
139+
echo "--- fetch_file_at_ref (main.go lines 25-30) ---"
140+
CODE=$(python -c "
141+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
142+
print(pvr.fetch_file_at_ref.fn(owner='anticomputer', repo='vulnerable-test-app', path='main.go', ref='main', start_line=25, length=6))
143+
")
144+
if echo "$CODE" | grep -q "searchHandler"; then
145+
ok "Fetched vulnerable code at main.go:25"
146+
echo "$CODE" | sed 's/^/ /'
147+
else
148+
fail "Failed to fetch main.go"
149+
fi
150+
echo
151+
152+
echo "--- resolve_version_ref (0.0.1 -- expected to fail, no tags) ---"
153+
VER=$(python -c "
154+
import seclab_taskflows.mcp_servers.pvr_ghsa as pvr
155+
print(pvr.resolve_version_ref.fn(owner='anticomputer', repo='vulnerable-test-app', version='0.0.1'))
156+
")
157+
if echo "$VER" | grep -q "Could not resolve"; then
158+
ok "Graceful failure: no tags in repo (expected)"
159+
else
160+
ok "Resolved: $VER"
161+
fi
162+
echo
163+
164+
sep "Tool Tests Complete ($FAILURES failures)"
165+
}
166+
167+
# --- batch: run batch scoring taskflow ---
168+
169+
cmd_batch() {
170+
sep "Batch Scoring Taskflow"
171+
echo "Repo: ${REPO}"
172+
echo "Report dir: ${REPORT_DIR}"
173+
echo
174+
175+
# The test advisories are in draft state (owner-created), so patch the
176+
# taskflow call to use state=draft. The batch taskflow defaults to triage
177+
# state, but we can override via the run_agent globals.
178+
run_agent \
179+
-t seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch \
180+
-g "repo=${REPO}" \
181+
-g "state=${ADVISORY_STATE}"
182+
183+
echo
184+
BATCH_REPORT=$(ls -t "${REPORT_DIR}"/batch_queue_*.md 2>/dev/null | head -1)
185+
if [ -n "${BATCH_REPORT}" ]; then
186+
ok "Batch report: ${BATCH_REPORT}"
187+
echo
188+
cat "${BATCH_REPORT}"
189+
else
190+
fail "No batch report generated"
191+
fi
192+
}
193+
194+
# --- triage: run full single-advisory triage ---
195+
196+
cmd_triage() {
197+
local ghsa="${1:-}"
198+
199+
if [ -z "$ghsa" ]; then
200+
# Pick the high-quality SQL injection report
201+
ghsa=$(python -c "
202+
import json, seclab_taskflows.mcp_servers.pvr_ghsa as pvr
203+
advs = json.loads(pvr.list_pvr_advisories.fn(owner='anticomputer', repo='vulnerable-test-app', state='draft'))
204+
for a in advs:
205+
if 'SQL' in a['summary'] or 'sql' in a['summary'].lower():
206+
print(a['ghsa_id'])
207+
break
208+
else:
209+
print(advs[0]['ghsa_id'] if advs else '')
210+
")
211+
fi
212+
213+
if [ -z "$ghsa" ]; then
214+
fail "No advisories found to triage"
215+
return
216+
fi
217+
218+
sep "Single Advisory Triage: ${ghsa}"
219+
echo "Repo: ${REPO}"
220+
echo "GHSA: ${ghsa}"
221+
echo "Report dir: ${REPORT_DIR}"
222+
echo
223+
224+
run_agent \
225+
-t seclab_taskflows.taskflows.pvr_triage.pvr_triage \
226+
-g "repo=${REPO}" \
227+
-g "ghsa=${ghsa}" \
228+
-g "state=${ADVISORY_STATE}"
229+
230+
echo
231+
TRIAGE_REPORT="${REPORT_DIR}/${ghsa}_triage.md"
232+
RESPONSE_DRAFT="${REPORT_DIR}/${ghsa}_response_triage.md"
233+
if [ -f "${TRIAGE_REPORT}" ]; then
234+
ok "Triage report: ${TRIAGE_REPORT}"
235+
echo
236+
cat "${TRIAGE_REPORT}"
237+
else
238+
fail "No triage report generated"
239+
fi
240+
echo
241+
if [ -f "${RESPONSE_DRAFT}" ]; then
242+
sep "Response Draft"
243+
cat "${RESPONSE_DRAFT}"
244+
fi
245+
}
246+
247+
# --- all: run everything ---
248+
249+
cmd_all() {
250+
cmd_tools
251+
cmd_batch
252+
cmd_triage "${1:-}"
253+
sep "Demo Complete ($FAILURES total failures)"
254+
}
255+
256+
# --- dispatch ---
257+
258+
case "${1:-tools}" in
259+
tools) cmd_tools ;;
260+
batch) cmd_batch ;;
261+
triage) shift; cmd_triage "${1:-}" ;;
262+
all) shift; cmd_all "${1:-}" ;;
263+
-h|--help|help)
264+
echo "Usage: $0 [tools|batch|triage [GHSA]|all]"
265+
echo
266+
echo " tools - test MCP tools against live API (no AI calls)"
267+
echo " batch - run batch scoring taskflow"
268+
echo " triage - run full triage (picks SQL injection report by default)"
269+
echo " all - run everything in sequence"
270+
;;
271+
*) echo "Unknown command: $1" >&2; exit 1 ;;
272+
esac

‎src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ globals:
3030
repo:
3131
# GHSA ID of the advisory to triage
3232
ghsa:
33+
# Advisory state to filter by for dedup comparison (default: triage).
34+
# Use "draft" for testing with owner-created advisories.
35+
state: triage
3336

3437
taskflow:
3538
# -------------------------------------------------------------------------
@@ -146,7 +149,7 @@ taskflow:
146149
- affected_component: pvr_parsed.affected_component
147150
148151
Extract owner and repo from "{{ globals.repo }}" (format: owner/repo).
149-
Call compare_advisories with owner, repo, state="triage",
152+
Call compare_advisories with owner, repo, state="{{ globals.state }}",
150153
and target_ghsa="{{ globals.ghsa }}" to check if this advisory is a
151154
structural duplicate of another advisory currently in the triage inbox.
152155
Store the result as dedup_result.

‎src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ model_config: seclab_taskflows.configs.model_config_pvr_triage
2828
globals:
2929
# GitHub repository in owner/repo format
3030
repo:
31+
# Advisory state to filter by (default: triage). Use "draft" for testing
32+
# with owner-created advisories.
33+
state: triage
3134

3235
taskflow:
3336
# -------------------------------------------------------------------------
@@ -44,8 +47,8 @@ taskflow:
4447
user_prompt: |
4548
Extract owner and repo from "{{ globals.repo }}" (format: owner/repo).
4649
47-
Call list_pvr_advisories with owner, repo, and state="triage" to retrieve
48-
all advisories in triage state.
50+
Call list_pvr_advisories with owner, repo, and state="{{ globals.state }}" to retrieve
51+
all advisories in {{ globals.state }} state.
4952
5053
Store the full JSON list under memcache key "pvr_queue".
5154
@@ -69,7 +72,7 @@ taskflow:
6972
7073
Extract owner and repo from "{{ globals.repo }}" (format: owner/repo).
7174
72-
First, call compare_advisories with owner, repo, and state="triage" to
75+
First, call compare_advisories with owner, repo, and state="{{ globals.state }}" to
7376
detect duplicate or near-duplicate advisories via structural metadata
7477
comparison. Store the result under memcache key "dedup_result".
7578

0 commit comments

Comments
 (0)