Skip to content

Commit 8bb20e4

Browse files
alexsmirnovalexsmirnov
committed
feat: prompt cache improvement from anomalyco#14743 
1 parent a546256 commit 8bb20e4

15 files changed

Lines changed: 230 additions & 56 deletions

File tree

‎packages/opencode/src/cli/cmd/tui/routes/session/sidebar.tsx‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,17 @@ export function Sidebar(props: { sessionID: string; overlay?: boolean }) {
5353
if (!last) return
5454
const total =
5555
last.tokens.input + last.tokens.output + last.tokens.reasoning + last.tokens.cache.read + last.tokens.cache.write
56+
const totalInput = last.tokens.input + last.tokens.cache.read + last.tokens.cache.write
5657
const model = sync.data.provider.find((x) => x.id === last.providerID)?.models[last.modelID]
5758
return {
5859
tokens: total.toLocaleString(),
5960
percentage: model?.limit.context ? Math.round((total / model.limit.context) * 100) : null,
61+
cacheHitPercent: totalInput > 0 ? ((last.tokens.cache.read / totalInput) * 100).toFixed(3) : null,
62+
cacheRead: last.tokens.cache.read,
63+
cacheWrite: last.tokens.cache.write,
64+
cacheNew: last.tokens.input,
65+
cacheInput: totalInput,
66+
cacheOutput: last.tokens.output,
6067
}
6168
})
6269

@@ -106,6 +113,19 @@ export function Sidebar(props: { sessionID: string; overlay?: boolean }) {
106113
<text fg={theme.textMuted}>{context()?.percentage ?? 0}% used</text>
107114
<text fg={theme.textMuted}>{cost()} spent</text>
108115
</box>
116+
<Show when={process.env["OPENCODE_CACHE_AUDIT"] && context()?.cacheHitPercent != null}>
117+
<box>
118+
<text fg={theme.text}>
119+
<b>Cache Audit</b>
120+
</text>
121+
<text fg={theme.textMuted}>{context()!.cacheInput.toLocaleString()} input tokens</text>
122+
<text fg={theme.textMuted}> {context()!.cacheNew.toLocaleString()} new</text>
123+
<text fg={theme.textMuted}> {context()!.cacheRead.toLocaleString()} cache read</text>
124+
<text fg={theme.textMuted}> {context()!.cacheWrite.toLocaleString()} cache write</text>
125+
<text fg={theme.textMuted}>{context()!.cacheHitPercent}% hit rate</text>
126+
<text fg={theme.textMuted}>{context()!.cacheOutput.toLocaleString()} output tokens</text>
127+
</box>
128+
</Show>
109129
<Show when={mcpEntries().length > 0}>
110130
<box>
111131
<box

‎packages/opencode/src/flag/flag.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ export namespace Flag {
5858
export const OPENCODE_DISABLE_FILETIME_CHECK = truthy("OPENCODE_DISABLE_FILETIME_CHECK")
5959
export const OPENCODE_EXPERIMENTAL_PLAN_MODE = OPENCODE_EXPERIMENTAL || truthy("OPENCODE_EXPERIMENTAL_PLAN_MODE")
6060
export const OPENCODE_EXPERIMENTAL_MARKDOWN = !falsy("OPENCODE_EXPERIMENTAL_MARKDOWN")
61+
export const OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION = truthy("OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION")
62+
export const OPENCODE_EXPERIMENTAL_CACHE_1H_TTL = truthy("OPENCODE_EXPERIMENTAL_CACHE_1H_TTL")
6163
export const OPENCODE_MODELS_URL = process.env["OPENCODE_MODELS_URL"]
6264
export const OPENCODE_MODELS_PATH = process.env["OPENCODE_MODELS_PATH"]
6365

‎packages/opencode/src/provider/transform.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -171,10 +171,12 @@ export namespace ProviderTransform {
171171
return msgs
172172
}
173173

174-
function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage[] {
174+
function applyCaching(msgs: ModelMessage[], model: Provider.Model, extendedTTL?: boolean): ModelMessage[] {
175175
const system = msgs.filter((msg) => msg.role === "system").slice(0, 2)
176176
const final = msgs.filter((msg) => msg.role !== "system").slice(-2)
177177

178+
// Use 1h cache TTL on first system block (2x write cost vs 1.25x for default 5-min)
179+
const anthropicCache = extendedTTL ? { type: "ephemeral", ttl: "1h" } : { type: "ephemeral" }
178180
const providerOptions = {
179181
anthropic: {
180182
cacheControl: { type: "ephemeral" },
@@ -194,18 +196,21 @@ export namespace ProviderTransform {
194196
}
195197

196198
for (const msg of unique([...system, ...final])) {
199+
const options = msg === system[0]
200+
? { ...providerOptions, anthropic: { cacheControl: anthropicCache } }
201+
: providerOptions
197202
const useMessageLevelOptions = model.providerID === "anthropic" || model.providerID.includes("bedrock")
198203
const shouldUseContentOptions = !useMessageLevelOptions && Array.isArray(msg.content) && msg.content.length > 0
199204

200205
if (shouldUseContentOptions) {
201206
const lastContent = msg.content[msg.content.length - 1]
202207
if (lastContent && typeof lastContent === "object") {
203-
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, providerOptions)
208+
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, options)
204209
continue
205210
}
206211
}
207212

208-
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions)
213+
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, options)
209214
}
210215

211216
return msgs
@@ -261,7 +266,7 @@ export namespace ProviderTransform {
261266
model.api.npm === "@ai-sdk/anthropic") &&
262267
model.api.npm !== "@ai-sdk/gateway"
263268
) {
264-
msgs = applyCaching(msgs, model)
269+
msgs = applyCaching(msgs, model, (options.extendedTTL as boolean) ?? Flag.OPENCODE_EXPERIMENTAL_CACHE_1H_TTL)
265270
}
266271

267272
// Remap providerOptions keys from stored providerID to expected SDK key

‎packages/opencode/src/session/index.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,6 +839,15 @@ export namespace Session {
839839
},
840840
}
841841

842+
// OPENCODE_CACHE_AUDIT=1 enables per-call cache token accounting in the log
843+
if (process.env["OPENCODE_CACHE_AUDIT"]) {
844+
const totalInputTokens = tokens.input + tokens.cache.read + tokens.cache.write
845+
const cacheHitPercent = totalInputTokens > 0 ? ((tokens.cache.read / totalInputTokens) * 100).toFixed(1) : "0.0"
846+
log.info(
847+
`[CACHE] ${input.model.id} input=${totalInputTokens} (cache_read=${tokens.cache.read} cache_write=${tokens.cache.write} new=${tokens.input}) hit=${cacheHitPercent}% output=${tokens.output} total=${tokens.total ?? 0}`,
848+
)
849+
}
850+
842851
const costInfo =
843852
input.model.cost?.experimentalOver200K && tokens.input + tokens.cache.read > 200_000
844853
? input.model.cost.experimentalOver200K

‎packages/opencode/src/session/instruction.ts‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,15 @@ export namespace InstructionPrompt {
7171

7272
export async function systemPaths() {
7373
const config = await Config.get()
74-
const paths = new Set<string>()
74+
const global = new Set<string>()
75+
const project = new Set<string>()
7576

7677
if (!Flag.OPENCODE_DISABLE_PROJECT_CONFIG) {
7778
for (const file of FILES) {
7879
const matches = await Filesystem.findUp(file, Instance.directory, Instance.worktree)
7980
if (matches.length > 0) {
8081
matches.forEach((p) => {
81-
paths.add(path.resolve(p))
82+
project.add(path.resolve(p))
8283
})
8384
break
8485
}
@@ -87,7 +88,7 @@ export namespace InstructionPrompt {
8788

8889
for (const file of globalFiles()) {
8990
if (await Filesystem.exists(file)) {
90-
paths.add(path.resolve(file))
91+
global.add(path.resolve(file))
9192
break
9293
}
9394
}
@@ -106,22 +107,29 @@ export namespace InstructionPrompt {
106107
}).catch(() => [])
107108
: await resolveRelative(instruction)
108109
matches.forEach((p) => {
109-
paths.add(path.resolve(p))
110+
project.add(path.resolve(p))
110111
})
111112
}
112113
}
113114

114-
return paths
115+
return { global, project }
115116
}
116117

117-
export async function system() {
118-
const config = await Config.get()
118+
export type SystemInstructions = { global: string[]; project: string[] }
119+
120+
let cached: SystemInstructions | undefined
121+
122+
export async function system(): Promise<SystemInstructions> {
123+
if (Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION && cached) return cached
124+
119125
const paths = await systemPaths()
126+
const config = await Config.get()
120127

121-
const files = Array.from(paths).map(async (p) => {
122-
const content = await Filesystem.readText(p).catch(() => "")
123-
return content ? "Instructions from: " + p + "\n" + content : ""
124-
})
128+
const readPaths = (set: Set<string>) =>
129+
Array.from(set).map(async (p) => {
130+
const content = await Filesystem.readText(p).catch(() => "")
131+
return content ? "Instructions from: " + p + "\n" + content : ""
132+
})
125133

126134
const urls: string[] = []
127135
if (config.instructions) {
@@ -138,7 +146,14 @@ export namespace InstructionPrompt {
138146
.then((x) => (x ? "Instructions from: " + url + "\n" + x : "")),
139147
)
140148

141-
return Promise.all([...files, ...fetches]).then((result) => result.filter(Boolean))
149+
const [global, project] = await Promise.all([
150+
Promise.all(readPaths(paths.global)).then((r) => r.filter(Boolean)),
151+
Promise.all([...readPaths(paths.project), ...fetches]).then((r) => r.filter(Boolean)),
152+
])
153+
154+
const result = { global, project }
155+
if (Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION) cached = result
156+
return result
142157
}
143158

144159
export function loaded(messages: MessageV2.WithParts[]) {
@@ -166,7 +181,8 @@ export namespace InstructionPrompt {
166181
}
167182

168183
export async function resolve(messages: MessageV2.WithParts[], filepath: string, messageID: string) {
169-
const system = await systemPaths()
184+
const paths = await systemPaths()
185+
const system = new Set([...paths.global, ...paths.project])
170186
const already = loaded(messages)
171187
const results: { filepath: string; content: string }[] = []
172188

‎packages/opencode/src/session/llm.ts‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ export namespace LLM {
3333
model: Provider.Model
3434
agent: Agent.Info
3535
system: string[]
36+
systemSplit?: number
3637
abort: AbortSignal
3738
messages: ModelMessage[]
3839
small?: boolean
@@ -64,20 +65,16 @@ export namespace LLM {
6465
])
6566
const isCodex = provider.id === "openai" && auth?.type === "oauth"
6667

67-
const system = []
68-
system.push(
69-
[
70-
// use agent prompt otherwise provider prompt
71-
// For Codex sessions, skip SystemPrompt.provider() since it's sent via options.instructions
72-
...(input.agent.prompt ? [input.agent.prompt] : isCodex ? [] : SystemPrompt.provider(input.model)),
73-
// any custom prompt passed into this call
74-
...input.system,
75-
// any custom prompt from last user message
76-
...(input.user.system ? [input.user.system] : []),
77-
]
78-
.filter((x) => x)
79-
.join("\n"),
80-
)
68+
// use agent prompt otherwise provider prompt
69+
// For Codex sessions, skip SystemPrompt.provider() since it's sent via options.instructions
70+
const prompt = input.agent.prompt ? [input.agent.prompt] : isCodex ? [] : SystemPrompt.provider(input.model)
71+
const split = input.systemSplit ?? input.system.length
72+
const system = [
73+
// block 1: provider/agent prompt + global instructions (stable across repos)
74+
[...prompt, ...input.system.slice(0, split)].filter(Boolean).join("\n"),
75+
// block 2: env + project instructions + any custom prompt from last user message (dynamic)
76+
[...input.system.slice(split), ...(input.user.system ? [input.user.system] : [])].filter(Boolean).join("\n"),
77+
].filter(Boolean)
8178

8279
const header = system[0]
8380
await Plugin.trigger(

‎packages/opencode/src/session/prompt.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -655,8 +655,10 @@ export namespace SessionPrompt {
655655

656656
await Plugin.trigger("experimental.chat.messages.transform", {}, { messages: msgs })
657657

658-
// Build system prompt, adding structured output instruction if needed
659-
const system = [...(await SystemPrompt.environment(model)), ...(await InstructionPrompt.system())]
658+
// Build system prompt: global instructions first (stable), then env + project (dynamic)
659+
const instructions = await InstructionPrompt.system()
660+
const system = [...instructions.global, ...(await SystemPrompt.environment(model)), ...instructions.project]
661+
const systemSplit = instructions.global.length
660662
const format = lastUser.format ?? { type: "text" }
661663
if (format.type === "json_schema") {
662664
system.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT)
@@ -668,6 +670,7 @@ export namespace SessionPrompt {
668670
abort,
669671
sessionID,
670672
system,
673+
systemSplit,
671674
messages: [
672675
...MessageV2.toModelMessages(msgs, model),
673676
...(isLastStep

‎packages/opencode/src/session/system.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import PROMPT_GEMINI from "./prompt/gemini.txt"
1010
import PROMPT_CODEX from "./prompt/codex_header.txt"
1111
import PROMPT_TRINITY from "./prompt/trinity.txt"
1212
import type { Provider } from "@/provider/provider"
13+
import { Flag } from "@/flag/flag"
1314

1415
export namespace SystemPrompt {
1516
export function instructions() {
@@ -26,8 +27,13 @@ export namespace SystemPrompt {
2627
return [PROMPT_ANTHROPIC_WITHOUT_TODO]
2728
}
2829

30+
let cachedDate: Date | undefined
31+
2932
export async function environment(model: Provider.Model) {
3033
const project = Instance.project
34+
const date = Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION
35+
? (cachedDate ??= new Date())
36+
: new Date()
3137
return [
3238
[
3339
`You are powered by the model named ${model.api.id}. The exact model ID is ${model.providerID}/${model.api.id}`,
@@ -36,7 +42,7 @@ export namespace SystemPrompt {
3642
` Working directory: ${Instance.directory}`,
3743
` Is directory a git repo: ${project.vcs === "git" ? "yes" : "no"}`,
3844
` Platform: ${process.platform}`,
39-
` Today's date: ${new Date().toDateString()}`,
45+
` Today's date: ${date.toDateString()}`,
4046
`</env>`,
4147
`<directories>`,
4248
` ${

‎packages/opencode/src/tool/bash.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,17 @@ export const BashTool = Tool.define("bash", async () => {
5757
log.info("bash tool using shell", { shell })
5858

5959
return {
60-
description: DESCRIPTION.replaceAll("${directory}", Instance.directory)
61-
.replaceAll("${maxLines}", String(Truncate.MAX_LINES))
62-
.replaceAll("${maxBytes}", String(Truncate.MAX_BYTES)),
60+
description: DESCRIPTION.replaceAll("${maxLines}", String(Truncate.MAX_LINES)).replaceAll(
61+
"${maxBytes}",
62+
String(Truncate.MAX_BYTES),
63+
),
6364
parameters: z.object({
6465
command: z.string().describe("The command to execute"),
6566
timeout: z.number().describe("Optional timeout in milliseconds").optional(),
6667
workdir: z
6768
.string()
6869
.describe(
69-
`The working directory to run the command in. Defaults to ${Instance.directory}. Use this instead of 'cd' commands.`,
70+
`The working directory to run the command in. Defaults to the current working directory. Use this instead of 'cd' commands.`,
7071
)
7172
.optional(),
7273
description: z

‎packages/opencode/src/tool/bash.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.
22

3-
All commands run in ${directory} by default. Use the `workdir` parameter if you need to run a command in a different directory. AVOID using `cd <directory> && <command>` patterns - use `workdir` instead.
3+
All commands run in the current working directory by default. Use the `workdir` parameter if you need to run a command in a different directory. AVOID using `cd <directory> && <command>` patterns - use `workdir` instead.
44

55
IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO NOT use it for file operations (reading, writing, editing, searching, finding files) - use the specialized tools for this instead.
66

0 commit comments

Comments
 (0)