Skip to content

Commit c002206

Browse files
committed
feat(provider): detect and warn on history rewrite breaking KV cache
Add detection for when messages sent to the LLM differ between turns, which breaks KV cache prefix matching. Logs a warning with a formatted diff showing what changed and why. Catches issues like empty reasoning_content being added to messages that previously had none (PR anomalyco#28352), as well as any other message mutations during transformation.
1 parent 0880f50 commit c002206

4 files changed

Lines changed: 187 additions & 1 deletion

File tree

packages/opencode/src/effect/runtime-flags.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ export class Service extends ConfigService.Service<Service>()("@opencode/Runtime
4848
experimentalEventSystem: enabledByExperimental("OPENCODE_EXPERIMENTAL_EVENT_SYSTEM"),
4949
experimentalWorkspaces: enabledByExperimental("OPENCODE_EXPERIMENTAL_WORKSPACES"),
5050
experimentalIconDiscovery: enabledByExperimental("OPENCODE_EXPERIMENTAL_ICON_DISCOVERY"),
51+
experimentalHistoryRewriteDetection: enabledByExperimental("OPENCODE_EXPERIMENTAL_HISTORY_REWRITE_DETECTION"),
52+
historyRewriteMaxSessions: positiveInteger("OPENCODE_EXPERIMENTAL_HISTORY_REWRITE_MAX_SESSIONS"),
5153
outputTokenMax: positiveInteger("OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX"),
5254
bashDefaultTimeoutMs: positiveInteger("OPENCODE_EXPERIMENTAL_BASH_DEFAULT_TIMEOUT_MS"),
5355
experimentalNativeLlm: enabledByExperimental("OPENCODE_EXPERIMENTAL_NATIVE_LLM"),
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import type { ModelMessage } from "ai"
2+
import { diffLines } from "diff"
3+
import * as Log from "@opencode-ai/core/util/log"
4+
5+
const log = Log.create({ service: "provider.history-rewrite" })
6+
7+
// In-memory store of previous messages per session (deep cloned to avoid mutation issues)
8+
const previousMessages = new Map<string, ModelMessage[]>()
9+
10+
export const DEFAULT_MAX_SESSIONS = 5
11+
12+
/**
13+
* Creates a stable string representation of a message for comparison.
14+
*/
15+
function messageFingerprint(msg: ModelMessage): string {
16+
const content = contentToStr(msg.content)
17+
return JSON.stringify({
18+
role: msg.role,
19+
content,
20+
})
21+
}
22+
23+
function contentToStr(content: ModelMessage["content"]): string {
24+
return typeof content === "string" ? content : JSON.stringify(content)
25+
}
26+
27+
/**
28+
* Truncates long strings for display in logs.
29+
*/
30+
function truncate(str: string, maxLen = 200): string {
31+
if (str.length <= maxLen) return str
32+
return str.slice(0, maxLen) + `... (+${str.length - maxLen} chars)`
33+
}
34+
35+
/**
36+
* Detects when historical messages have been rewritten between requests.
37+
* Compares the current messages against the previous request for the same session.
38+
* Returns a report of changes if any were detected.
39+
*/
40+
export function detectHistoryRewrite(sessionID: string, messages: ModelMessage[], maxSessions = DEFAULT_MAX_SESSIONS): string | undefined {
41+
const prev = previousMessages.get(sessionID)
42+
if (!prev) {
43+
previousMessages.set(sessionID, structuredClone(messages))
44+
return undefined
45+
}
46+
47+
// Compare message counts first
48+
if (prev.length === messages.length) {
49+
let hasChanges = false
50+
for (let i = 0; i < prev.length; i++) {
51+
if (messageFingerprint(prev[i]) !== messageFingerprint(messages[i])) {
52+
hasChanges = true
53+
break
54+
}
55+
}
56+
if (!hasChanges) {
57+
previousMessages.set(sessionID, structuredClone(messages))
58+
return undefined
59+
}
60+
}
61+
62+
// Find which messages changed
63+
const changes: {
64+
index: number
65+
role: string
66+
diff: string
67+
oldPreview: string
68+
newPreview: string
69+
}[] = []
70+
71+
const minLen = Math.min(prev.length, messages.length)
72+
for (let i = 0; i < minLen; i++) {
73+
const oldFp = messageFingerprint(prev[i])
74+
const newFp = messageFingerprint(messages[i])
75+
if (oldFp !== newFp) {
76+
const oldContent = contentToStr(prev[i].content)
77+
const newContent = contentToStr(messages[i].content)
78+
changes.push({
79+
index: i,
80+
role: prev[i].role,
81+
diff: diffLines(oldContent, newContent).map((h) => {
82+
const line = h.value.replace(/\n$/, "")
83+
if (h.added) return `+ ${line}`
84+
if (h.removed) return `- ${line}`
85+
return line
86+
}).join("\n"),
87+
oldPreview: truncate(oldContent, 150),
88+
newPreview: truncate(newContent, 150),
89+
})
90+
}
91+
}
92+
93+
// Detect added/removed messages
94+
if (messages.length > prev.length) {
95+
for (let i = prev.length; i < messages.length; i++) {
96+
const content = contentToStr(messages[i].content)
97+
changes.push({
98+
index: i,
99+
role: messages[i].role,
100+
diff: "(message added)",
101+
oldPreview: "(none)",
102+
newPreview: truncate(content, 150),
103+
})
104+
}
105+
} else if (prev.length > messages.length) {
106+
for (let i = messages.length; i < prev.length; i++) {
107+
const content = contentToStr(prev[i].content)
108+
changes.push({
109+
index: i,
110+
role: prev[i].role,
111+
diff: "(message removed)",
112+
oldPreview: truncate(content, 150),
113+
newPreview: "(none)",
114+
})
115+
}
116+
}
117+
118+
// Store current messages (bounded)
119+
if (previousMessages.size >= maxSessions) {
120+
const keys = [...previousMessages.keys()]
121+
for (const key of keys.slice(0, keys.length - maxSessions + 1)) {
122+
previousMessages.delete(key)
123+
}
124+
}
125+
previousMessages.set(sessionID, structuredClone(messages))
126+
127+
if (changes.length === 0) return undefined
128+
129+
// Build report
130+
const lines: string[] = [
131+
`History rewrite detected (session: ${sessionID})`,
132+
`${changes.length} message(s) changed:`,
133+
"",
134+
]
135+
136+
for (const change of changes) {
137+
lines.push(`--- Message [${change.index}] (${change.role}) ---`)
138+
lines.push(`Old: ${change.oldPreview}`)
139+
lines.push(`New: ${change.newPreview}`)
140+
if (change.diff) {
141+
lines.push("Diff:")
142+
lines.push(change.diff)
143+
}
144+
lines.push("")
145+
}
146+
147+
return lines.join("\n")
148+
}
149+
150+
/**
151+
* Clears the history for a session (e.g., on session close or revert).
152+
*/
153+
export function clearHistory(sessionID: string): void {
154+
previousMessages.delete(sessionID)
155+
}
156+
157+
/**
158+
* Clears all stored history (e.g., on shutdown).
159+
*/
160+
export function clearAllHistory(): void {
161+
previousMessages.clear()
162+
}

packages/opencode/src/session/llm.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import type { LLMClientService } from "@opencode-ai/llm/route"
99
import { mergeDeep } from "remeda"
1010
import { GitLabWorkflowLanguageModel } from "gitlab-ai-provider"
1111
import { ProviderTransform } from "@/provider/transform"
12+
import { detectHistoryRewrite, DEFAULT_MAX_SESSIONS } from "@/provider/history-rewrite"
1213
import { Config } from "@/config/config"
1314
import { InstanceState } from "@/effect/instance-state"
1415
import type { Agent } from "@/agent/agent"
@@ -355,6 +356,9 @@ const live: Layer.Layer<
355356
provider: item,
356357
auth: info,
357358
llmClient,
359+
sessionID: input.sessionID,
360+
experimentalHistoryRewriteDetection: flags.experimentalHistoryRewriteDetection,
361+
historyRewriteMaxSessions: flags.historyRewriteMaxSessions,
358362
isOpenaiOauth,
359363
system,
360364
messages,
@@ -449,6 +453,10 @@ const live: Layer.Layer<
449453
if (args.type === "stream") {
450454
// @ts-expect-error
451455
args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options)
456+
if (flags.experimentalHistoryRewriteDetection) {
457+
const report = detectHistoryRewrite(input.sessionID, args.params.prompt, flags.historyRewriteMaxSessions ?? DEFAULT_MAX_SESSIONS)
458+
if (report) l.warn("history rewrite detected", { report })
459+
}
452460
}
453461
return args.params
454462
},

packages/opencode/src/session/llm/native-runtime.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import type { Auth } from "@/auth"
22
import type { Provider } from "@/provider/provider"
33
import { ProviderTransform } from "@/provider/transform"
4+
import { detectHistoryRewrite, DEFAULT_MAX_SESSIONS } from "@/provider/history-rewrite"
45
import { errorMessage } from "@/util/error"
6+
import * as Log from "@opencode-ai/core/util/log"
57
import { isRecord } from "@/util/record"
68
import { asSchema, type ModelMessage, type Tool } from "ai"
79
import { Effect } from "effect"
@@ -10,6 +12,8 @@ import { tool as nativeTool, ToolFailure, type JsonSchema, type LLMEvent } from
1012
import type { LLMClientShape } from "@opencode-ai/llm/route"
1113
import { LLMNative } from "./native-request"
1214

15+
const log = Log.create({ service: "session.llm.native-runtime" })
16+
1317
export type RuntimeStatus =
1418
| { readonly type: "supported"; readonly apiKey: string; readonly baseURL?: string }
1519
| { readonly type: "unsupported"; readonly reason: string }
@@ -22,6 +26,9 @@ type StreamInput = {
2226
readonly provider: Provider.Info
2327
readonly auth: Auth.Info | undefined
2428
readonly llmClient: LLMClientShape
29+
readonly sessionID: string
30+
readonly experimentalHistoryRewriteDetection: boolean
31+
readonly historyRewriteMaxSessions?: number
2532
readonly isOpenaiOauth: boolean
2633
readonly system: string[]
2734
readonly messages: ModelMessage[]
@@ -69,7 +76,14 @@ export function stream(input: StreamInput): StreamResult {
6976
apiKey: current.apiKey,
7077
baseURL: current.baseURL,
7178
system: input.isOpenaiOauth ? input.system : [],
72-
messages: ProviderTransform.message(input.messages, input.model, input.providerOptions ?? {}),
79+
messages: (() => {
80+
const transformed = ProviderTransform.message(input.messages, input.model, input.providerOptions ?? {})
81+
if (input.experimentalHistoryRewriteDetection) {
82+
const report = detectHistoryRewrite(input.sessionID, transformed, input.historyRewriteMaxSessions ?? DEFAULT_MAX_SESSIONS)
83+
if (report) log.warn("history rewrite detected", { report })
84+
}
85+
return transformed
86+
})(),
7387
toolChoice: input.toolChoice,
7488
temperature: input.temperature,
7589
topP: input.topP,

0 commit comments

Comments
 (0)