Skip to content

Commit af158e8

Browse files
henry701henry701
committed
feat: merge PR anomalyco#14743 (prompt cache stability) + PR anomalyco#14973 (agent loop fix)
PR anomalyco#14743 — fix(cache): improve Anthropic prompt cache hit rate - Split system prompt into stable (global) + dynamic (project) blocks - Remove cwd from bash tool schema (was busting cache per-repo) - Freeze date under OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION flag - Add optional 1h TTL on first system block (OPENCODE_EXPERIMENTAL_CACHE_1H_TTL) - Add OPENCODE_CACHE_AUDIT logging for per-call cache accounting - Track global vs project skill scope for stable cache prefix - Add splitSystemPrompt provider option to opt out PR anomalyco#14973 — fix(core): prevent agent loop stopping after tool calls - Check lastAssistantMsg.parts for tool type before exiting loop - Fixes OpenAI-compatible providers (Gemini, LiteLLM) returning finish_reason 'stop' instead of 'tool_calls' when tools were called ci: add FORCE_JAVASCRIPT_ACTIONS_TO_NODE24 to upstream-sync workflow build: relax bun version check to minor-level for local builds
1 parent c1cf9fa commit af158e8

15 files changed

Lines changed: 182 additions & 46 deletions

File tree

‎.github/workflows/upstream-sync.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ jobs:
1616
permissions:
1717
contents: write
1818
issues: write
19+
env:
20+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
1921

2022
steps:
2123
- name: Checkout fork

‎packages/opencode/src/flag/flag.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ export namespace Flag {
7171
export const OPENCODE_EXPERIMENTAL_PLAN_MODE = OPENCODE_EXPERIMENTAL || truthy("OPENCODE_EXPERIMENTAL_PLAN_MODE")
7272
export const OPENCODE_EXPERIMENTAL_WORKSPACES = OPENCODE_EXPERIMENTAL || truthy("OPENCODE_EXPERIMENTAL_WORKSPACES")
7373
export const OPENCODE_EXPERIMENTAL_MARKDOWN = !falsy("OPENCODE_EXPERIMENTAL_MARKDOWN")
74+
export const OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION = truthy("OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION")
75+
export const OPENCODE_EXPERIMENTAL_CACHE_1H_TTL = truthy("OPENCODE_EXPERIMENTAL_CACHE_1H_TTL")
7476
export const OPENCODE_MODELS_URL = process.env["OPENCODE_MODELS_URL"]
7577
export const OPENCODE_MODELS_PATH = process.env["OPENCODE_MODELS_PATH"]
7678
export const OPENCODE_DISABLE_EMBEDDED_WEB_UI = truthy("OPENCODE_DISABLE_EMBEDDED_WEB_UI")

‎packages/opencode/src/provider/transform.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -189,10 +189,12 @@ export namespace ProviderTransform {
189189
return msgs
190190
}
191191

192-
function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage[] {
192+
function applyCaching(msgs: ModelMessage[], model: Provider.Model, extendedTTL?: boolean): ModelMessage[] {
193193
const system = msgs.filter((msg) => msg.role === "system").slice(0, 2)
194194
const final = msgs.filter((msg) => msg.role !== "system").slice(-2)
195195

196+
// Use 1h cache TTL on first system block (2x write cost vs 1.25x for default 5-min)
197+
const anthropicCache = extendedTTL ? { type: "ephemeral", ttl: "1h" } : { type: "ephemeral" }
196198
const providerOptions = {
197199
anthropic: {
198200
cacheControl: { type: "ephemeral" },
@@ -212,6 +214,9 @@ export namespace ProviderTransform {
212214
}
213215

214216
for (const msg of unique([...system, ...final])) {
217+
const options = msg === system[0]
218+
? { ...providerOptions, anthropic: { cacheControl: anthropicCache } }
219+
: providerOptions
215220
const useMessageLevelOptions =
216221
model.providerID === "anthropic" ||
217222
model.providerID.includes("bedrock") ||
@@ -226,12 +231,12 @@ export namespace ProviderTransform {
226231
lastContent.type !== "tool-approval-request" &&
227232
lastContent.type !== "tool-approval-response"
228233
) {
229-
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, providerOptions)
234+
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, options)
230235
continue
231236
}
232237
}
233238

234-
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions)
239+
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, options)
235240
}
236241

237242
return msgs
@@ -288,7 +293,7 @@ export namespace ProviderTransform {
288293
model.api.npm === "@ai-sdk/anthropic") &&
289294
model.api.npm !== "@ai-sdk/gateway"
290295
) {
291-
msgs = applyCaching(msgs, model)
296+
msgs = applyCaching(msgs, model, (options.extendedTTL as boolean) ?? Flag.OPENCODE_EXPERIMENTAL_CACHE_1H_TTL)
292297
}
293298

294299
// Remap providerOptions keys from stored providerID to expected SDK key

‎packages/opencode/src/session/index.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,15 @@ export namespace Session {
286286
},
287287
}
288288

289+
// OPENCODE_CACHE_AUDIT=1 enables per-call cache token accounting in the log
290+
if (process.env["OPENCODE_CACHE_AUDIT"]) {
291+
const totalInputTokens = tokens.input + tokens.cache.read + tokens.cache.write
292+
const cacheHitPercent = totalInputTokens > 0 ? ((tokens.cache.read / totalInputTokens) * 100).toFixed(1) : "0.0"
293+
log.info(
294+
`[CACHE] ${input.model.id} input=${totalInputTokens} (cache_read=${tokens.cache.read} cache_write=${tokens.cache.write} new=${tokens.input}) hit=${cacheHitPercent}% output=${tokens.output} total=${tokens.total ?? 0}`,
295+
)
296+
}
297+
289298
const costInfo =
290299
input.model.cost?.experimentalOver200K && tokens.input + tokens.cache.read > 200_000
291300
? input.model.cost.experimentalOver200K

‎packages/opencode/src/session/instruction.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,10 @@ export namespace Instruction {
161161
return paths
162162
})
163163

164+
let cachedSystem: string[] | undefined
164165
const system = Effect.fn("Instruction.system")(function* () {
166+
if (Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION && cachedSystem) return cachedSystem
167+
165168
const config = yield* cfg.get()
166169
const paths = yield* systemPaths()
167170
const urls = (config.instructions ?? []).filter(
@@ -171,10 +174,12 @@ export namespace Instruction {
171174
const files = yield* Effect.forEach(Array.from(paths), read, { concurrency: 8 })
172175
const remote = yield* Effect.forEach(urls, fetch, { concurrency: 4 })
173176

174-
return [
177+
const result = [
175178
...Array.from(paths).flatMap((item, i) => (files[i] ? [`Instructions from: ${item}\n${files[i]}`] : [])),
176179
...urls.flatMap((item, i) => (remote[i] ? [`Instructions from: ${item}\n${remote[i]}`] : [])),
177180
]
181+
if (Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION) cachedSystem = result
182+
return result
178183
})
179184

180185
const find = Effect.fn("Instruction.find")(function* (dir: string) {

‎packages/opencode/src/session/llm.ts‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export namespace LLM {
2929
agent: Agent.Info
3030
permission?: Permission.Ruleset
3131
system: string[]
32+
systemSplit?: number
3233
messages: ModelMessage[]
3334
small?: boolean
3435
tools: Record<string, Tool>
@@ -98,19 +99,17 @@ export namespace LLM {
9899
// TODO: move this to a proper hook
99100
const isOpenaiOauth = provider.id === "openai" && auth?.type === "oauth"
100101

101-
const system: string[] = []
102-
system.push(
103-
[
104-
// use agent prompt otherwise provider prompt
105-
...(input.agent.prompt ? [input.agent.prompt] : SystemPrompt.provider(input.model)),
106-
// any custom prompt passed into this call
107-
...input.system,
108-
// any custom prompt from last user message
109-
...(input.user.system ? [input.user.system] : []),
110-
]
111-
.filter((x) => x)
112-
.join("\n"),
113-
)
102+
const prompt = input.agent.prompt ? [input.agent.prompt] : SystemPrompt.provider(input.model)
103+
const split = input.systemSplit ?? input.system.length
104+
const shouldSplit = provider.options?.["splitSystemPrompt"] !== false
105+
const system = shouldSplit
106+
? [
107+
[...prompt, ...input.system.slice(0, split)].filter(Boolean).join("\n"),
108+
[...input.system.slice(split), ...(input.user.system ? [input.user.system] : [])].filter(Boolean).join("\n"),
109+
].filter(Boolean)
110+
: [
111+
[...prompt, ...input.system, ...(input.user.system ? [input.user.system] : [])].filter(Boolean).join("\n"),
112+
].filter(Boolean)
114113

115114
const header = system[0]
116115
await Plugin.trigger(
@@ -119,7 +118,7 @@ export namespace LLM {
119118
{ system },
120119
)
121120
// rejoin to maintain 2-part structure for caching if header unchanged
122-
if (system.length > 2 && system[0] === header) {
121+
if (shouldSplit && system.length > 2 && system[0] === header) {
123122
const rest = system.slice(1)
124123
system.length = 0
125124
system.push(header, rest.join("\n"))

‎packages/opencode/src/session/prompt.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1504,7 +1504,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the
15041504
instruction.system().pipe(Effect.orDie),
15051505
Effect.promise(() => MessageV2.toModelMessages(msgs, model)),
15061506
])
1507-
const system = [...env, ...(skills ? [skills] : []), ...instructions]
1507+
const system = [
1508+
...(skills.global ? [skills.global] : []),
1509+
...env,
1510+
...(skills.project ? [skills.project] : []),
1511+
...instructions,
1512+
]
1513+
const systemSplit = skills.global ? 1 : 0
15081514
const format = lastUser.format ?? { type: "text" as const }
15091515
if (format.type === "json_schema") system.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT)
15101516
const result = yield* handle.process({
@@ -1513,6 +1519,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
15131519
permission: session.permission,
15141520
sessionID,
15151521
system,
1522+
systemSplit,
15161523
messages: [...modelMsgs, ...(isLastStep ? [{ role: "assistant" as const, content: MAX_STEPS }] : [])],
15171524
tools,
15181525
model,

‎packages/opencode/src/session/system.ts‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import type { Provider } from "@/provider/provider"
1515
import type { Agent } from "@/agent/agent"
1616
import { Permission } from "@/permission"
1717
import { Skill } from "@/skill"
18+
import { Flag } from "@/flag/flag"
1819

1920
export namespace SystemPrompt {
2021
export function provider(model: Provider.Model) {
@@ -33,8 +34,13 @@ export namespace SystemPrompt {
3334
return [PROMPT_DEFAULT]
3435
}
3536

37+
let cachedDate: Date | undefined
38+
3639
export async function environment(model: Provider.Model) {
3740
const project = Instance.project
41+
const date = Flag.OPENCODE_EXPERIMENTAL_CACHE_STABILIZATION
42+
? (cachedDate ??= new Date())
43+
: new Date()
3844
return [
3945
[
4046
`You are powered by the model named ${model.api.id}. The exact model ID is ${model.providerID}/${model.api.id}`,
@@ -44,7 +50,7 @@ export namespace SystemPrompt {
4450
` Workspace root folder: ${Instance.worktree}`,
4551
` Is directory a git repo: ${project.vcs === "git" ? "yes" : "no"}`,
4652
` Platform: ${process.platform}`,
47-
` Today's date: ${new Date().toDateString()}`,
53+
` Today's date: ${date.toDateString()}`,
4854
`</env>`,
4955
`<directories>`,
5056
` ${
@@ -60,17 +66,31 @@ export namespace SystemPrompt {
6066
]
6167
}
6268

63-
export async function skills(agent: Agent.Info) {
64-
if (Permission.disabled(["skill"], agent.permission).has("skill")) return
69+
export async function skills(agent: Agent.Info): Promise<{ global?: string; project?: string }> {
70+
if (Permission.disabled(["skill"], agent.permission).has("skill")) return {}
6571

6672
const list = await Skill.available(agent)
73+
const globalSkills = list.filter((s) => s.scope === "global")
74+
const projectSkills = list.filter((s) => s.scope === "project")
6775

68-
return [
76+
// the agents seem to ingest the information about skills a bit better if we present a more verbose
77+
// version of them here and a less verbose version in tool description, rather than vice versa.
78+
const preamble = [
6979
"Skills provide specialized instructions and workflows for specific tasks.",
7080
"Use the skill tool to load a skill when a task matches its description.",
71-
// the agents seem to ingest the information about skills a bit better if we present a more verbose
72-
// version of them here and a less verbose version in tool description, rather than vice versa.
73-
Skill.fmt(list, { verbose: true }),
7481
].join("\n")
82+
83+
const global = globalSkills.length > 0
84+
? [preamble, Skill.fmt(globalSkills, { verbose: true })].join("\n")
85+
: undefined
86+
87+
const project = projectSkills.length > 0
88+
? [
89+
...(globalSkills.length === 0 ? [preamble] : []),
90+
Skill.fmt(projectSkills, { verbose: true }),
91+
].join("\n")
92+
: undefined
93+
94+
return { global, project }
7595
}
7696
}

‎packages/opencode/src/skill/index.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export namespace Skill {
3030
description: z.string(),
3131
location: z.string(),
3232
content: z.string(),
33+
scope: z.enum(["global", "project"]).default("project"),
3334
})
3435
export type Info = z.infer<typeof Info>
3536

@@ -63,7 +64,7 @@ export namespace Skill {
6364
readonly available: (agent?: Agent.Info) => Effect.Effect<Info[]>
6465
}
6566

66-
const add = Effect.fnUntraced(function* (state: State, match: string, bus: Bus.Interface) {
67+
const add = Effect.fnUntraced(function* (state: State, match: string, bus: Bus.Interface, scope: "global" | "project" = "project") {
6768
const md = yield* Effect.tryPromise({
6869
try: () => ConfigMarkdown.parse(match),
6970
catch: (err) => err,
@@ -100,6 +101,7 @@ export namespace Skill {
100101
description: parsed.data.description,
101102
location: match,
102103
content: md.content,
104+
scope,
103105
}
104106
})
105107

@@ -108,7 +110,7 @@ export namespace Skill {
108110
bus: Bus.Interface,
109111
root: string,
110112
pattern: string,
111-
opts?: { dot?: boolean; scope?: string },
113+
opts?: { dot?: boolean; scope?: "global" | "project" },
112114
) {
113115
const matches = yield* Effect.tryPromise({
114116
try: () =>
@@ -128,7 +130,7 @@ export namespace Skill {
128130
}),
129131
)
130132

131-
yield* Effect.forEach(matches, (match) => add(state, match, bus), {
133+
yield* Effect.forEach(matches, (match) => add(state, match, bus, opts?.scope ?? "project"), {
132134
concurrency: "unbounded",
133135
discard: true,
134136
})
@@ -161,7 +163,7 @@ export namespace Skill {
161163

162164
const configDirs = yield* config.directories()
163165
for (const dir of configDirs) {
164-
yield* scan(state, bus, dir, OPENCODE_SKILL_PATTERN)
166+
yield* scan(state, bus, dir, OPENCODE_SKILL_PATTERN, { scope: "global" })
165167
}
166168

167169
const cfg = yield* config.get()

‎packages/opencode/src/tool/bash.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -446,10 +446,7 @@ export const BashTool = Tool.define("bash", async () => {
446446
log.info("bash tool using shell", { shell })
447447

448448
return {
449-
description: DESCRIPTION.replaceAll("${directory}", Instance.directory)
450-
.replaceAll("${os}", process.platform)
451-
.replaceAll("${shell}", name)
452-
.replaceAll("${chaining}", chain)
449+
description: DESCRIPTION.replaceAll("${chaining}", chain)
453450
.replaceAll("${maxLines}", String(Truncate.MAX_LINES))
454451
.replaceAll("${maxBytes}", String(Truncate.MAX_BYTES)),
455452
parameters: z.object({
@@ -458,7 +455,7 @@ export const BashTool = Tool.define("bash", async () => {
458455
workdir: z
459456
.string()
460457
.describe(
461-
`The working directory to run the command in. Defaults to ${Instance.directory}. Use this instead of 'cd' commands.`,
458+
`The working directory to run the command in. Defaults to the current working directory. Use this instead of 'cd' commands.`,
462459
)
463460
.optional(),
464461
description: z

0 commit comments

Comments
 (0)