fix: detect secret-only changes in redacted diffs

README.md

@@ -54,7 +54,7 @@ It classifies:
 - removed safety/refusal/secret-handling guardrails
 - tool surface changes such as shell, browser, network, delete, database, or email access
 - output-contract changes such as JSON/schema/format shifts
-- secret-like values, redacted before output by default
+- secret-like values, redacted in output by default while still reporting when only the secret value changed
 
 ### `check`
 

src/analyzer.ts

@@ -1,4 +1,6 @@
 import { diffLines } from './diff.js';
+import { comparisonLinesFor } from './parser.js';
+import { redactText } from './redact.js';
 import { maxSeverity } from './severity.js';
 import type { CompareOptions, CompareResult, Finding, ParsedPrompt, Severity } from './types.js';
 
@@ -21,7 +23,13 @@ function removedSafety(removed: string[]): string[] {
 }
 
 export function analyzePromptDiff(oldPrompt: ParsedPrompt, newPrompt: ParsedPrompt, options: CompareOptions): CompareResult {
-  const diff = diffLines(oldPrompt.lines, newPrompt.lines);
+  const comparisonDiff = diffLines(comparisonLinesFor(oldPrompt), comparisonLinesFor(newPrompt));
+  const diff = options.redact
+    ? {
+        added: comparisonDiff.added.map(redactText),
+        removed: comparisonDiff.removed.map(redactText),
+      }
+    : comparisonDiff;
   const findings: Finding[] = [];
 
   const addedInstructions = select(diff.added, instructionWords);

src/parser.ts

@@ -3,6 +3,8 @@ import { extname } from 'node:path';
 import type { ParsedPrompt } from './types.js';
 import { redactText } from './redact.js';
 
+const comparisonLines = new WeakMap<ParsedPrompt, string[]>();
+
 function stableJson(value: unknown): string {
   if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
   if (value && typeof value === 'object') {
@@ -24,8 +26,7 @@ export function detectFormat(path: string, raw: string): ParsedPrompt['format']
   return 'text';
 }
 
-export function normalizePrompt(path: string, rawInput: string, redact = true): ParsedPrompt {
-  const raw = redact ? redactText(rawInput) : rawInput;
+function normalize(path: string, raw: string): Omit<ParsedPrompt, 'path'> {
   const format = detectFormat(path, raw);
   let normalized = raw.replace(/\r\n/g, '\n').trim();
 
@@ -42,7 +43,17 @@ export function normalizePrompt(path: string, rawInput: string, redact = true):
   }
 
   const lines = normalized.split('\n').map((line) => line.trimEnd());
-  return { path, format, raw, normalized, lines };
+  return { format, raw, normalized, lines };
+}
+
+export function normalizePrompt(path: string, rawInput: string, redact = true): ParsedPrompt {
+  const parsed = { path, ...normalize(path, redact ? redactText(rawInput) : rawInput) };
+  if (redact) comparisonLines.set(parsed, normalize(path, rawInput).lines);
+  return parsed;
+}
+
+export function comparisonLinesFor(prompt: ParsedPrompt): string[] {
+  return comparisonLines.get(prompt) ?? prompt.lines;
 }
 
 export async function readPrompt(path: string, redact = true): Promise<ParsedPrompt> {

tests/analyzer.test.mjs

@@ -20,4 +20,31 @@ test('redacts secret-shaped changed lines', () => {
   assert.ok(result.findings.some((finding) => finding.id === 'secret-redacted'));
   assert.ok(JSON.stringify(result).includes('<redacted>'));
   assert.ok(!JSON.stringify(result).includes('supersecretvalue'));
+  assert.ok(!JSON.stringify(result).includes('evenmoresecret'));
+});
+
+test('reports value-only secret changes without exposing either value', () => {
+  const before = normalizePrompt('old.md', 'token=aaaaaaaa', true);
+  const after = normalizePrompt('new.md', 'token=bbbbbbbb', true);
+  const result = analyzePromptDiff(before, after, { redact: true });
+  const rendered = JSON.stringify(result);
+
+  assert.equal(result.summary.changedLineCount, 2);
+  assert.ok(result.findings.some((finding) => finding.id === 'secret-redacted'));
+  assert.deepEqual(result.diff, {
+    added: ['token=<redacted>'],
+    removed: ['token=<redacted>'],
+  });
+  assert.ok(!rendered.includes('aaaaaaaa'));
+  assert.ok(!rendered.includes('bbbbbbbb'));
+});
+
+test('does not report unchanged identical secrets as changed', () => {
+  const before = normalizePrompt('old.md', 'token=aaaaaaaa', true);
+  const after = normalizePrompt('new.md', 'token=aaaaaaaa', true);
+  const result = analyzePromptDiff(before, after, { redact: true });
+
+  assert.equal(result.summary.changedLineCount, 0);
+  assert.equal(result.summary.findingCount, 0);
+  assert.deepEqual(result.diff, { added: [], removed: [] });
 });

tests/cli.test.mjs

@@ -11,6 +11,28 @@ test('cli compare returns gate failure when fail-on threshold is met', () => {
   assert.match(run.stdout, /Dangerous instruction added/);
 });
 
+test('cli reports secret-only changes in redacted Markdown and JSON', () => {
+  const directory = mkdtempSync(join(tmpdir(), 'promptdiff-cli-secrets-'));
+  const oldPath = join(directory, 'old.md');
+  const newPath = join(directory, 'new.md');
+  writeFileSync(oldPath, 'token=aaaaaaaa\n');
+  writeFileSync(newPath, 'token=bbbbbbbb\n');
+
+  try {
+    for (const format of ['markdown', 'json']) {
+      const run = spawnSync(process.execPath, [
+        'dist/cli.js', 'compare', oldPath, newPath, '--format', format,
+      ], { encoding: 'utf8' });
+      assert.equal(run.status, 0);
+      assert.match(run.stdout, /<redacted>/);
+      assert.doesNotMatch(run.stdout, /aaaaaaaa|bbbbbbbb/);
+      assert.equal(run.stderr, '');
+    }
+  } finally {
+    rmSync(directory, { recursive: true });
+  }
+});
+
 test('cli check passes safe fixture rules', () => {
   const run = spawnSync(process.execPath, ['dist/cli.js', 'check', 'examples/prompts/safe.md', '--rules', 'examples/rules.json'], { encoding: 'utf8' });
   assert.equal(run.status, 0);