diff --git a/README.md b/README.md index 1c2a230..08d40d3 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ It classifies: - removed safety/refusal/secret-handling guardrails - tool surface changes such as shell, browser, network, delete, database, or email access - output-contract changes such as JSON/schema/format shifts -- secret-like values, redacted before output by default +- secret-like values, redacted in output by default while still reporting when only the secret value changed ### `check` diff --git a/src/analyzer.ts b/src/analyzer.ts index 2199943..79d367c 100644 --- a/src/analyzer.ts +++ b/src/analyzer.ts @@ -1,4 +1,6 @@ import { diffLines } from './diff.js'; +import { comparisonLinesFor } from './parser.js'; +import { redactText } from './redact.js'; import { maxSeverity } from './severity.js'; import type { CompareOptions, CompareResult, Finding, ParsedPrompt, Severity } from './types.js'; @@ -21,7 +23,13 @@ function removedSafety(removed: string[]): string[] { } export function analyzePromptDiff(oldPrompt: ParsedPrompt, newPrompt: ParsedPrompt, options: CompareOptions): CompareResult { - const diff = diffLines(oldPrompt.lines, newPrompt.lines); + const comparisonDiff = diffLines(comparisonLinesFor(oldPrompt), comparisonLinesFor(newPrompt)); + const diff = options.redact + ? { + added: comparisonDiff.added.map(redactText), + removed: comparisonDiff.removed.map(redactText), + } + : comparisonDiff; const findings: Finding[] = []; const addedInstructions = select(diff.added, instructionWords); diff --git a/src/parser.ts b/src/parser.ts index 844e543..9a6a86e 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -3,6 +3,8 @@ import { extname } from 'node:path'; import type { ParsedPrompt } from './types.js'; import { redactText } from './redact.js'; +const comparisonLines = new WeakMap(); + function stableJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`; if (value && typeof value === 'object') { @@ -24,8 +26,7 @@ export function detectFormat(path: string, raw: string): ParsedPrompt['format'] return 'text'; } -export function normalizePrompt(path: string, rawInput: string, redact = true): ParsedPrompt { - const raw = redact ? redactText(rawInput) : rawInput; +function normalize(path: string, raw: string): Omit { const format = detectFormat(path, raw); let normalized = raw.replace(/\r\n/g, '\n').trim(); @@ -42,7 +43,17 @@ export function normalizePrompt(path: string, rawInput: string, redact = true): } const lines = normalized.split('\n').map((line) => line.trimEnd()); - return { path, format, raw, normalized, lines }; + return { format, raw, normalized, lines }; +} + +export function normalizePrompt(path: string, rawInput: string, redact = true): ParsedPrompt { + const parsed = { path, ...normalize(path, redact ? redactText(rawInput) : rawInput) }; + if (redact) comparisonLines.set(parsed, normalize(path, rawInput).lines); + return parsed; +} + +export function comparisonLinesFor(prompt: ParsedPrompt): string[] { + return comparisonLines.get(prompt) ?? prompt.lines; } export async function readPrompt(path: string, redact = true): Promise { diff --git a/tests/analyzer.test.mjs b/tests/analyzer.test.mjs index 76cfa01..88bf4cc 100644 --- a/tests/analyzer.test.mjs +++ b/tests/analyzer.test.mjs @@ -20,4 +20,31 @@ test('redacts secret-shaped changed lines', () => { assert.ok(result.findings.some((finding) => finding.id === 'secret-redacted')); assert.ok(JSON.stringify(result).includes('')); assert.ok(!JSON.stringify(result).includes('supersecretvalue')); + assert.ok(!JSON.stringify(result).includes('evenmoresecret')); +}); + +test('reports value-only secret changes without exposing either value', () => { + const before = normalizePrompt('old.md', 'token=aaaaaaaa', true); + const after = normalizePrompt('new.md', 'token=bbbbbbbb', true); + const result = analyzePromptDiff(before, after, { redact: true }); + const rendered = JSON.stringify(result); + + assert.equal(result.summary.changedLineCount, 2); + assert.ok(result.findings.some((finding) => finding.id === 'secret-redacted')); + assert.deepEqual(result.diff, { + added: ['token='], + removed: ['token='], + }); + assert.ok(!rendered.includes('aaaaaaaa')); + assert.ok(!rendered.includes('bbbbbbbb')); +}); + +test('does not report unchanged identical secrets as changed', () => { + const before = normalizePrompt('old.md', 'token=aaaaaaaa', true); + const after = normalizePrompt('new.md', 'token=aaaaaaaa', true); + const result = analyzePromptDiff(before, after, { redact: true }); + + assert.equal(result.summary.changedLineCount, 0); + assert.equal(result.summary.findingCount, 0); + assert.deepEqual(result.diff, { added: [], removed: [] }); }); diff --git a/tests/cli.test.mjs b/tests/cli.test.mjs index 066e080..972a87c 100644 --- a/tests/cli.test.mjs +++ b/tests/cli.test.mjs @@ -11,6 +11,28 @@ test('cli compare returns gate failure when fail-on threshold is met', () => { assert.match(run.stdout, /Dangerous instruction added/); }); +test('cli reports secret-only changes in redacted Markdown and JSON', () => { + const directory = mkdtempSync(join(tmpdir(), 'promptdiff-cli-secrets-')); + const oldPath = join(directory, 'old.md'); + const newPath = join(directory, 'new.md'); + writeFileSync(oldPath, 'token=aaaaaaaa\n'); + writeFileSync(newPath, 'token=bbbbbbbb\n'); + + try { + for (const format of ['markdown', 'json']) { + const run = spawnSync(process.execPath, [ + 'dist/cli.js', 'compare', oldPath, newPath, '--format', format, + ], { encoding: 'utf8' }); + assert.equal(run.status, 0); + assert.match(run.stdout, //); + assert.doesNotMatch(run.stdout, /aaaaaaaa|bbbbbbbb/); + assert.equal(run.stderr, ''); + } + } finally { + rmSync(directory, { recursive: true }); + } +}); + test('cli check passes safe fixture rules', () => { const run = spawnSync(process.execPath, ['dist/cli.js', 'check', 'examples/prompts/safe.md', '--rules', 'examples/rules.json'], { encoding: 'utf8' }); assert.equal(run.status, 0);