diff --git a/CHANGELOG.md b/CHANGELOG.md index 873698d8..f6b893d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Fixed +- implicit-resume 区分"真 missing tool call"与"client 自包含 full replay"：`evaluateImplicitResume` 现在新增 `inlineFunctionCallIds` 入参，由 `buildProxySessionContext` 通过 `getInlineFunctionCallIds(codexRequest.input)` 在调用前收集 input 里所有 `function_call` 项的 call_id。当 input 里的 function_call_output 全部能在 input 内找到对应 function_call（典型 Codex CLI `/compact` 或客户端 fallback 自包含 replay 场景），返回 `reason: "self_contained_replay"` 而不是 `missing_tool_calls`，proxy 走正常透传不再触发 payload guard 413。混合场景（部分 inline 部分 storage 都找不到）仍判 `missing_tool_calls` 防真 runaway。新增 4 个 `evaluateImplicitResume`/`getInlineFunctionCallIds`/`isSelfContainedReplay` 单测覆盖纯 inline、混合、空 output、incremental turn 四类（`src/routes/shared/proxy-session-helpers.ts`、`src/routes/shared/proxy-session-context.ts`、`tests/unit/routes/shared/proxy-handler-implicit-resume.test.ts`、`tests/unit/routes/shared/proxy-session-context.test.ts`）。 - 放宽 `/v1/responses` 的 missing-tool-call full-history replay guard 阈值：从 250KB / 80 items 提到 2MB / 1000 items。原阈值会把 Codex CLI `/compact` 之后正常的 client-driven full replay（实测 300-800KB / 100-800 items）误判成 runaway 风暴，连续 413 卡死对话。新阈值仍能挡真正失控的 multi-MB 重试循环，但不再误伤合法 fallback；测试侧把 padding 从 80 提到 1010 items 让 guard 仍能在新阈值下触发（`src/routes/shared/proxy-handler.ts`、`tests/integration/proxy-handler.test.ts`）。 - 修复个别账号被 Cloudflare Bot Management `__cf_bm` cookie 反噬导致 `/codex/responses` 全 404、`/codex/usage` 仍正常的"配额没限却用不了"假死状态：根因是 proxy 此前在 warmup `GET /codex/usage` 时通过 `captureCookies` 把 CF 偶发下发的 `__cf_bm` 收进 jar，而 `__cf_bm` 是绑死 (IP + UA + TLS fingerprint + 时序) 的 30 分钟会话指纹 cookie——一旦 fingerprint 漂（proxy pool 切出口 IP / cookie 过期 / 时序变化），CF 在重保护路径就用空 body 404（CF "stealth deny" 模式）拒绝该 cookie 持有者，而轻保护路径继续放行，造成 `cachedQuota.rate_limit.limit_reached=false / used 78%` 但 `/codex/responses` 14 连 404 的诊断矛盾；24 个号里只这一个号偶然命中过 CF 下发，其它 cookie jar 全空的号反而都正常。修复两层：(1) `src/proxy/cookie-jar.ts` 加 `CAPTURABLE_COOKIE_NAMES = {cf_clearance}` 白名单，`captureRaw` 主动丢弃 `__cf_bm` 等非白名单 cookie，从源头不让毒 cookie 入 jar；admin API 的手动 `set()` 不受白名单约束方便调试。(2) `src/proxy/error-classification.ts` 新增 `isCfPathBlockError`（404 + trimmed body 为空）；`src/auth/cf-path-block-tracker.ts` 用 1 小时滑动窗口计数器追踪每个 entryId 的连续 CF block 次数；`src/routes/shared/proxy-error-handler.ts` 在 generic respond 之前加新分支——命中 CF block 时清这个号的 cookie jar、记录计数、`releaseBeforeRetry: true` 让请求 fail over 到不同号，同号 1h 内累计 ≥ 3 次自动 `markStatus("disabled")` 并 `appendErrorLog({ name: "CfPathBlockAutoDisable" })` 进 Errors tab；`src/services/account-mutation.ts` 在 dashboard re-enable 时 `resetCfPathBlock` 清计数避免历史欠账。新增 `tests/unit/auth/cf-path-block-tracker.test.ts`（4 个，计数 / 窗口过期 / reset / peek）、`tests/unit/proxy/error-classification.test.ts` `isCfPathBlockError` 一节（4 个分支）、`tests/unit/routes/shared/proxy-error-handler.test.ts` CF block retry/disable 路径（2 个，含非空 body 不误判），`tests/unit/proxy/cookie-jar.test.ts` 改写为白名单语义（+2，旧 `session_id` / `expired` 用例改用 `cf_clearance` 测通用 Max-Age 解析）。Full suite 2258 全绿（`src/proxy/cookie-jar.ts`、`src/proxy/error-classification.ts`、`src/auth/cf-path-block-tracker.ts`、`src/routes/shared/proxy-error-handler.ts`、`src/routes/shared/proxy-handler.ts`、`src/services/account-mutation.ts`、`tests/unit/proxy/cookie-jar.test.ts`、`tests/unit/proxy/error-classification.test.ts`、`tests/unit/auth/cf-path-block-tracker.test.ts`、`tests/unit/routes/shared/proxy-error-handler.test.ts`） - `/v1/responses` passthrough streaming / non-streaming paths now collect `function_call.call_id` from `response.output_item.done` and forward it through response metadata so implicit resume can validate following `function_call_output` turns instead of falling back to full-history replay. Oversized missing-tool-call replays are guarded with 413, and regression coverage now proves the issue red/green across the Responses format adapter (`src/routes/responses.ts`, `src/routes/shared/proxy-handler.ts`, `tests/unit/routes/responses-passthrough-metadata.test.ts`, `tests/integration/proxy-handler.test.ts`). diff --git a/src/routes/shared/proxy-session-context.ts b/src/routes/shared/proxy-session-context.ts index a0c08237..bb7c4f58 100644 --- a/src/routes/shared/proxy-session-context.ts +++ b/src/routes/shared/proxy-session-context.ts @@ -5,6 +5,7 @@ import { buildVariantIdentity, getContinuationInputStartIndex, getFunctionCallOutputIds, + getInlineFunctionCallIds, IMPLICIT_RESUME_MAX_AGE_MS, normalizeInstructions, resolvePromptCacheIdentity, @@ -74,6 +75,12 @@ export function buildProxySessionContext( const implicitStoredFunctionCallIds = implicitPrevRespId ? affinityMap.lookupFunctionCallIds(implicitPrevRespId) : []; + // Function_call entries inlined in the full request input — used by + // evaluateImplicitResume to detect self-contained replays where matching + // pairs already exist in the payload and resume is not applicable. + const inlineFunctionCallIds = implicitPrevRespId + ? getInlineFunctionCallIds(codexRequest.input) + : []; const preferredEntryId = explicitPrevRespId ? affinityMap.lookup(explicitPrevRespId) @@ -108,6 +115,7 @@ export function buildProxySessionContext( storedInstructions: implicitStoredInstructions, requiredFunctionCallOutputIds, storedFunctionCallIds: implicitStoredFunctionCallIds, + inlineFunctionCallIds, }, }; } diff --git a/src/routes/shared/proxy-session-helpers.ts b/src/routes/shared/proxy-session-helpers.ts index e1a74607..dc1cd6b6 100644 --- a/src/routes/shared/proxy-session-helpers.ts +++ b/src/routes/shared/proxy-session-helpers.ts @@ -78,6 +78,11 @@ export interface ImplicitResumeOpts { storedInstructions: string | null; requiredFunctionCallOutputIds?: string[]; storedFunctionCallIds?: string[]; + /** call_ids of `function_call` items inlined in the request input itself. + * When a function_call_output references a call_id present here, the + * client is doing a self-contained full-history replay and we should NOT + * treat the absence of that id in session-affinity as "missing tool calls". */ + inlineFunctionCallIds?: string[]; } /** Reason why implicit resume was rejected, or null if it would activate. @@ -102,7 +107,21 @@ export function evaluateImplicitResume(opts: ImplicitResumeOpts): return { active: false, reason: "instr_diff" }; } const storedFunctionCallIds = new Set(opts.storedFunctionCallIds ?? []); + const inlineFunctionCallIds = new Set(opts.inlineFunctionCallIds ?? []); const requiredFunctionCallOutputIds = opts.requiredFunctionCallOutputIds ?? []; + + // Self-contained replay: every function_call_output in the input is paired + // with a function_call also inlined in the input (typical of Codex CLI + // /compact or error-recovery fallback). Implicit resume is not applicable — + // upstream will satisfy the outputs from the inlined calls directly. Bail + // before the missing_tool_calls check so we don't 413 a legitimate replay. + if ( + requiredFunctionCallOutputIds.length > 0 && + requiredFunctionCallOutputIds.every((id) => inlineFunctionCallIds.has(id)) + ) { + return { active: false, reason: "self_contained_replay" }; + } + const missingCallIds = requiredFunctionCallOutputIds.filter((id) => !storedFunctionCallIds.has(id)); if (missingCallIds.length > 0) { return { active: false, reason: "missing_tool_calls", missingCallIds }; @@ -149,3 +168,27 @@ export function getFunctionCallOutputIds(input: CodexResponsesRequest["input"]): !("role" in item) && item.type === "function_call_output") .map((item) => item.call_id); } + +/** Collect call_ids of `function_call` items inlined in the request input. + * Codex CLI emits these when doing a client-side full-history replay (e.g. + * after /compact or error recovery): the input carries the historical + * function_call entries paired with their function_call_output entries, so + * the proxy must not try to validate those outputs against session-affinity's + * stored ids — they reference function_calls that live in the input itself, + * not in any prior upstream response we tracked. */ +export function getInlineFunctionCallIds(input: CodexResponsesRequest["input"]): string[] { + return input + .filter((item): item is { type: "function_call"; call_id: string; name: string; arguments: string } => + !("role" in item) && item.type === "function_call" && typeof item.call_id === "string") + .map((item) => item.call_id); +} + +/** True when every function_call_output in the input is paired with a + * function_call also inlined in the input (i.e. the client is sending a + * self-contained full-history replay, not an incremental continuation). */ +export function isSelfContainedReplay(input: CodexResponsesRequest["input"]): boolean { + const outputs = getFunctionCallOutputIds(input); + if (outputs.length === 0) return false; + const inlineCalls = new Set(getInlineFunctionCallIds(input)); + return outputs.every((id) => inlineCalls.has(id)); +} diff --git a/tests/unit/routes/shared/proxy-handler-implicit-resume.test.ts b/tests/unit/routes/shared/proxy-handler-implicit-resume.test.ts index 9c6160a5..4c24a537 100644 --- a/tests/unit/routes/shared/proxy-handler-implicit-resume.test.ts +++ b/tests/unit/routes/shared/proxy-handler-implicit-resume.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest"; import { PreviousResponseWebSocketError } from "@src/proxy/codex-api.js"; import type { CodexResponsesRequest } from "@src/proxy/codex-api.js"; import { + evaluateImplicitResume, resolvePromptCacheIdentity, shouldActivateImplicitResume, shouldReplayFullInputAfterImplicitResumeError, @@ -150,4 +151,96 @@ describe("shouldActivateImplicitResume", () => { expect(shouldReplayFullInputAfterImplicitResumeError(err, true)).toBe(true); expect(shouldReplayFullInputAfterImplicitResumeError(err, false)).toBe(false); }); + + it("client 主动发自包含 full replay（function_call 与 function_call_output 都在 input 内）时返回 self_contained_replay，不报 missing_tool_calls", () => { + const result = evaluateImplicitResume({ + implicitPrevRespId: "resp_prev_stale", + continuationInputStart: 2, + inputLength: 100, + preferredEntryId: "entry_1", + acquiredEntryId: "entry_1", + currentInstructions: "system-a", + storedInstructions: "system-a", + // tool_outputs in input reference call_ids that don't exist in storage + // (proxy was restarted / session-affinity lost them), but they ARE + // present inline in the same input → self-contained replay. + requiredFunctionCallOutputIds: ["call_inlined_a", "call_inlined_b"], + storedFunctionCallIds: [], + inlineFunctionCallIds: ["call_inlined_a", "call_inlined_b"], + }); + expect(result.active).toBe(false); + expect(result.reason).toBe("self_contained_replay"); + }); + + it("混合场景：部分 call_id 在 input 内 inline、部分既不在 input 也不在 storage → 仍判 missing_tool_calls", () => { + const result = evaluateImplicitResume({ + implicitPrevRespId: "resp_prev", + continuationInputStart: 2, + inputLength: 50, + preferredEntryId: "entry_1", + acquiredEntryId: "entry_1", + currentInstructions: "system-a", + storedInstructions: "system-a", + requiredFunctionCallOutputIds: ["call_inlined", "call_truly_missing"], + storedFunctionCallIds: [], + inlineFunctionCallIds: ["call_inlined"], + }); + expect(result.active).toBe(false); + expect(result.reason).toBe("missing_tool_calls"); + }); + + it("self_contained_replay 优先于 missing_tool_calls：所有 tool_output 都能在 input 找到对应 function_call 时不应误报 missing", () => { + const result = evaluateImplicitResume({ + implicitPrevRespId: "resp_prev", + continuationInputStart: 50, + inputLength: 102, + preferredEntryId: "entry_1", + acquiredEntryId: "entry_1", + currentInstructions: "system-a", + storedInstructions: "system-a", + requiredFunctionCallOutputIds: ["call_x"], + storedFunctionCallIds: ["call_unrelated_stored"], + inlineFunctionCallIds: ["call_x", "call_y"], + }); + expect(result.reason).toBe("self_contained_replay"); + }); +}); + +describe("getInlineFunctionCallIds / isSelfContainedReplay", () => { + it("getInlineFunctionCallIds 只挑 function_call 项的 call_id，跳过 user/assistant/function_call_output", async () => { + const { getInlineFunctionCallIds } = await import("@src/routes/shared/proxy-session-helpers.js"); + const ids = getInlineFunctionCallIds([ + { role: "user", content: "hi" }, + { type: "function_call", call_id: "call_a", name: "read", arguments: "{}" }, + { type: "function_call_output", call_id: "call_a", output: "{}" }, + { role: "assistant", content: "ok" }, + { type: "function_call", call_id: "call_b", name: "write", arguments: "{}" }, + ]); + expect(ids).toEqual(["call_a", "call_b"]); + }); + + it("isSelfContainedReplay 在所有 function_call_output 都能在 input 找到 function_call 配对时返回 true", async () => { + const { isSelfContainedReplay } = await import("@src/routes/shared/proxy-session-helpers.js"); + expect(isSelfContainedReplay([ + { type: "function_call", call_id: "c1", name: "x", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "{}" }, + { type: "function_call", call_id: "c2", name: "y", arguments: "{}" }, + { type: "function_call_output", call_id: "c2", output: "{}" }, + ])).toBe(true); + }); + + it("isSelfContainedReplay 在 function_call_output 找不到 inline function_call 时返回 false", async () => { + const { isSelfContainedReplay } = await import("@src/routes/shared/proxy-session-helpers.js"); + expect(isSelfContainedReplay([ + { type: "function_call_output", call_id: "c_orphan", output: "{}" }, + ])).toBe(false); + }); + + it("isSelfContainedReplay 在没有 function_call_output 时返回 false（incremental turn 不是 replay）", async () => { + const { isSelfContainedReplay } = await import("@src/routes/shared/proxy-session-helpers.js"); + expect(isSelfContainedReplay([ + { role: "user", content: "hi" }, + { type: "function_call", call_id: "c1", name: "x", arguments: "{}" }, + ])).toBe(false); + }); }); diff --git a/tests/unit/routes/shared/proxy-session-context.test.ts b/tests/unit/routes/shared/proxy-session-context.test.ts index 3bd6e552..53349a44 100644 --- a/tests/unit/routes/shared/proxy-session-context.test.ts +++ b/tests/unit/routes/shared/proxy-session-context.test.ts @@ -161,6 +161,7 @@ describe("buildProxySessionContext", () => { storedInstructions: "system", requiredFunctionCallOutputIds: ["call_a"], storedFunctionCallIds: ["call_a"], + inlineFunctionCallIds: ["call_a"], }); }); });