From 7c4221c00bde2550ea35e2618ef920a53fbbb8d5 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 12 Jul 2026 07:50:09 -0700 Subject: [PATCH] fix(miner): wire real dollar-cost tracking into the loop's budgetSpent Advances #5135 loop-cli.js's GovernorCapUsage.budgetSpent was never incremented -- only turnsTaken/elapsedMs used real signals -- so a capLimits.budget dollar-cost dimension could never trip from the loop's own spend. Investigation found a real, already-flowing cost signal was being silently dropped: the Agent SDK's own result message declares `total_cost_usd: number` unconditionally (SDKResultSuccess and SDKResultError both carry it -- a session is billed whether or not it succeeds), but agent-sdk-driver.ts only ever read `num_turns`. Wires it through the whole real pipeline: - CodingAgentDriverResult gains an optional costUsd field. - agent-sdk-driver.ts extracts the SDK's real total_cost_usd whenever a result message arrived, on both the success and the "did not complete successfully" paths (never fabricated on a thrown/no-result stream, where no billing signal exists). - iterate-loop.ts sums it into a new totalCostUsd on IterateLoopResult, mirroring the existing totalTurnsUsed accumulator exactly. - attempt-cli.js surfaces it through options.onResult alongside totalTurnsUsed/iterationsUsed. - loop-cli.js increments usage.budgetSpent with the real value before calling governor-state.js's saveCapUsage. Documented, deliberate gap: the CLI-subprocess providers (claude-cli/ codex-cli) report no cost signal today -- their production argv never passes --output-format json, so there is nothing to parse cost from without a separate, larger change to how those providers invoke the real CLI binaries. budgetSpent is real for agent-sdk and an honest 0 (not fabricated) for the CLI-subprocess providers; a capLimits.budget dimension only ever meaningfully trips against agent-sdk spend. --- .../src/miner/agent-sdk-driver.ts | 7 +++++++ .../src/miner/coding-agent-driver.ts | 3 +++ .../src/miner/iterate-loop.ts | 13 +++++++++--- .../gittensory-miner/lib/attempt-cli.d.ts | 1 + packages/gittensory-miner/lib/attempt-cli.js | 10 +++++++--- packages/gittensory-miner/lib/loop-cli.js | 6 +++++- .../coding-agent-driver-parity.test.ts | 15 +++++++++++++- test/unit/miner-attempt-cli.test.ts | 3 ++- test/unit/miner-attempt-runner.test.ts | 20 +++++++++++++++++-- test/unit/miner-loop-cli.test.ts | 7 ++++++- 10 files changed, 73 insertions(+), 12 deletions(-) diff --git a/packages/gittensory-engine/src/miner/agent-sdk-driver.ts b/packages/gittensory-engine/src/miner/agent-sdk-driver.ts index 8a29441c9..5d21ff518 100644 --- a/packages/gittensory-engine/src/miner/agent-sdk-driver.ts +++ b/packages/gittensory-engine/src/miner/agent-sdk-driver.ts @@ -144,6 +144,11 @@ export function createAgentSdkCodingAgentDriver( const turnsUsed = typeof resultMessage?.num_turns === "number" ? resultMessage.num_turns : undefined; + // Real dollar cost: the SDK's own SDKResultSuccess/SDKResultError message types both declare + // `total_cost_usd: number` unconditionally -- present whenever a result message arrived at all, success + // or not (the session was billed either way), absent only when the stream produced no result message. + const costUsd = + typeof resultMessage?.total_cost_usd === "number" ? resultMessage.total_cost_usd : undefined; const resultText = typeof resultMessage?.result === "string" ? redactSecrets(resultMessage.result) : ""; const transcript = redactSecrets( @@ -169,6 +174,7 @@ export function createAgentSdkCodingAgentDriver( summary: "agent sdk session did not complete successfully", transcript, turnsUsed, + costUsd, error: `agent_sdk_${subtype === "success" ? "errored" : subtype}`, }; } @@ -179,6 +185,7 @@ export function createAgentSdkCodingAgentDriver( summary: resultText.slice(0, MAX_REDACTED_TEXT_LENGTH) || `coding agent completed with ${changedFiles.size} changed file(s)`, transcript, turnsUsed, + costUsd, }; }, }; diff --git a/packages/gittensory-engine/src/miner/coding-agent-driver.ts b/packages/gittensory-engine/src/miner/coding-agent-driver.ts index 3b6151662..4d7993fad 100644 --- a/packages/gittensory-engine/src/miner/coding-agent-driver.ts +++ b/packages/gittensory-engine/src/miner/coding-agent-driver.ts @@ -20,6 +20,9 @@ export type CodingAgentDriverResult = { /** Opaque provider transcript for operator inspection; absent when the driver did not run. */ transcript?: string | undefined; turnsUsed?: number | undefined; + /** Real dollar cost of this driver run, when the provider reports one. Absent (not zero) when the provider + * never got far enough to have a cost, or reports no cost signal at all -- never fabricated. */ + costUsd?: number | undefined; error?: string | undefined; }; diff --git a/packages/gittensory-engine/src/miner/iterate-loop.ts b/packages/gittensory-engine/src/miner/iterate-loop.ts index 29ef021dc..6b9ec75d7 100644 --- a/packages/gittensory-engine/src/miner/iterate-loop.ts +++ b/packages/gittensory-engine/src/miner/iterate-loop.ts @@ -102,6 +102,10 @@ export type IterateLoopResult = { iterationsUsed: number; /** Cumulative `turnsUsed` summed across every iteration that ran. */ totalTurnsUsed: number; + /** Cumulative real dollar cost summed across every iteration that ran, from each iteration's + * `CodingAgentDriverResult.costUsd`. Only the `agent-sdk` provider reports this today (the CLI-subprocess + * providers report no cost signal) -- always `0` for a provider that never reports one, never fabricated. */ + totalCostUsd: number; iterations: readonly IterateLoopIterationRecord[]; /** Populated only when `outcome === "handoff"`. */ handoffPacket?: HandoffPacket | undefined; @@ -228,7 +232,7 @@ function immediateAbandonNoIterationsPermitted(input: IterateLoopInput, deps: It reason: decision.reason, payload: { iterationNumber: 0, action: decision.action, abandonReason: decision.abandonReason }, }); - return { outcome: "abandon", finalDecision: decision, iterationsUsed: 0, totalTurnsUsed: 0, iterations: [] }; + return { outcome: "abandon", finalDecision: decision, iterationsUsed: 0, totalTurnsUsed: 0, totalCostUsd: 0, iterations: [] }; } /** @@ -262,6 +266,7 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD const iterations: IterateLoopIterationRecord[] = []; let previousBlockerCodes: readonly string[] | null = null; let totalTurnsUsed = 0; + let totalCostUsd = 0; for (let iterationNumber = 1; iterationNumber <= maxIterations; iterationNumber += 1) { const driverResult = await runDriverSafely(input, deps, { @@ -272,6 +277,7 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD maxTurns: input.maxTurnsPerIteration, }); totalTurnsUsed += driverResult.turnsUsed ?? 0; + totalCostUsd += driverResult.costUsd ?? 0; const { outcome: selfReview, verdict } = evaluateSelfReviewOutcome(input, driverResult, deps); @@ -296,12 +302,13 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD finalDecision: decision, iterationsUsed: iterationNumber, totalTurnsUsed, + totalCostUsd, iterations, handoffPacket: buildHandoffPacket(input, verdict as SelfReviewVerdict, driverResult), }; } if (decision.action === "abandon") { - return { outcome: "abandon", finalDecision: decision, iterationsUsed: iterationNumber, totalTurnsUsed, iterations }; + return { outcome: "abandon", finalDecision: decision, iterationsUsed: iterationNumber, totalTurnsUsed, totalCostUsd, iterations }; } previousBlockerCodes = blockerCodesFromContinuingOutcome(selfReview); } @@ -313,5 +320,5 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD * this package's fail-closed discipline, in case a future edit to the precedence ladder ever removes that * guarantee. */ const fallbackDecision: IterateLoopDecision = { action: "abandon", abandonReason: "max_iterations_reached", reason: "Iterate loop exhausted its iteration budget." }; - return { outcome: "abandon", finalDecision: fallbackDecision, iterationsUsed: maxIterations, totalTurnsUsed, iterations }; + return { outcome: "abandon", finalDecision: fallbackDecision, iterationsUsed: maxIterations, totalTurnsUsed, totalCostUsd, iterations }; } diff --git a/packages/gittensory-miner/lib/attempt-cli.d.ts b/packages/gittensory-miner/lib/attempt-cli.d.ts index f047a08c6..5e0002aa5 100644 --- a/packages/gittensory-miner/lib/attempt-cli.d.ts +++ b/packages/gittensory-miner/lib/attempt-cli.d.ts @@ -38,6 +38,7 @@ export type AttemptCliResult = outcome: `attempt_${RunMinerAttemptResult["outcome"]}`; submissionMode: "observe" | "enforce"; totalTurnsUsed: number; + totalCostUsd: number; iterationsUsed: number; reason?: string; decision?: unknown; diff --git a/packages/gittensory-miner/lib/attempt-cli.js b/packages/gittensory-miner/lib/attempt-cli.js index c33783987..2dfabd76c 100644 --- a/packages/gittensory-miner/lib/attempt-cli.js +++ b/packages/gittensory-miner/lib/attempt-cli.js @@ -374,10 +374,14 @@ export async function runAttempt(args, options = {}) { mode, attemptId, submissionMode: amsPolicy.spec.submissionMode, - // Every runMinerAttempt outcome carries a real loopResult (#5135's loop needs its genuine turn-usage to - // save real GovernorCapUsage via governor-state.js's saveCapUsage -- nothing else in the codebase calls - // it yet). Surfaced flat rather than the whole loopResult object, matching this result's own shallow shape. + // Every runMinerAttempt outcome carries a real loopResult (#5135's loop needs its genuine turn-usage and + // cost to save real GovernorCapUsage via governor-state.js's saveCapUsage -- nothing else in the codebase + // calls it yet). Surfaced flat rather than the whole loopResult object, matching this result's own + // shallow shape. costUsd is real only for the agent-sdk provider (its own SDK result message reports + // total_cost_usd); CLI-subprocess providers (claude-cli/codex-cli) report no cost signal today, so this + // is 0 for those -- an honest absence, not a fabricated number. totalTurnsUsed: result.loopResult.totalTurnsUsed, + totalCostUsd: result.loopResult.totalCostUsd, iterationsUsed: result.loopResult.iterationsUsed, ...("reason" in result ? { reason: result.reason } : {}), ...("decision" in result ? { decision: result.decision } : {}), diff --git a/packages/gittensory-miner/lib/loop-cli.js b/packages/gittensory-miner/lib/loop-cli.js index 4186dccaa..80b258c57 100644 --- a/packages/gittensory-miner/lib/loop-cli.js +++ b/packages/gittensory-miner/lib/loop-cli.js @@ -349,7 +349,11 @@ export async function runLoop(args, options = {}) { const cycleElapsedMs = nowMsFn() - cycleStartMs; usage = { - budgetSpent: usage.budgetSpent, + // Real for the agent-sdk provider (its own SDK result message reports total_cost_usd, wired through + // runMinerAttempt's real loopResult.totalCostUsd); the CLI-subprocess providers (claude-cli/codex-cli) + // report no cost signal today, so this contributes 0 for those runs -- an honest absence, not a + // fabricated number. A capLimits.budget dimension only ever meaningfully trips against agent-sdk spend. + budgetSpent: usage.budgetSpent + (lastResult?.totalCostUsd ?? 0), turnsTaken: usage.turnsTaken + (lastResult?.totalTurnsUsed ?? 0), elapsedMs: usage.elapsedMs + cycleElapsedMs, }; diff --git a/test/contract/coding-agent-driver-parity.test.ts b/test/contract/coding-agent-driver-parity.test.ts index 4d4a9d217..d365d9aab 100644 --- a/test/contract/coding-agent-driver-parity.test.ts +++ b/test/contract/coding-agent-driver-parity.test.ts @@ -112,7 +112,7 @@ const DRIVER_HARNESSES: DriverHarness[] = [ return; } yield { type: "assistant", message: { content: [{ type: "tool_use", name: "Edit", input: { file_path: "src/a.ts" } }] } }; - yield { type: "result", subtype: "success", is_error: false, num_turns: 3, result: "edited src/a.ts" }; + yield { type: "result", subtype: "success", is_error: false, num_turns: 3, total_cost_usd: 0.15, result: "edited src/a.ts" }; })(); }, }); @@ -130,6 +130,7 @@ function expectDriverResultShape(result: CodingAgentDriverResult): void { expect(result.summary.length).toBeGreaterThan(0); if (result.transcript !== undefined) expect(typeof result.transcript).toBe("string"); if (result.turnsUsed !== undefined) expect(typeof result.turnsUsed).toBe("number"); + if (result.costUsd !== undefined) expect(typeof result.costUsd).toBe("number"); if (result.error !== undefined) { expect(typeof result.error).toBe("string"); expect(result.error.length).toBeGreaterThan(0); @@ -229,4 +230,16 @@ describe("documented divergences, locked in explicitly", () => { const result = await driver.run(task); expect(result.turnsUsed).toBe(3); }); + + it("CLI driver reports no cost signal (a subprocess exposes none without --output-format json) — costUsd stays undefined", async () => { + const { driver } = DRIVER_HARNESSES[0]!.make("success"); + const result = await driver.run(task); + expect(result.costUsd).toBeUndefined(); + }); + + it("Agent-SDK driver reports the result frame's real total_cost_usd", async () => { + const { driver } = DRIVER_HARNESSES[1]!.make("success"); + const result = await driver.run(task); + expect(result.costUsd).toBe(0.15); + }); }); diff --git a/test/unit/miner-attempt-cli.test.ts b/test/unit/miner-attempt-cli.test.ts index 88a2b25c7..0df8a00dd 100644 --- a/test/unit/miner-attempt-cli.test.ts +++ b/test/unit/miner-attempt-cli.test.ts @@ -243,7 +243,7 @@ describe("runAttempt (#5132)", () => { outcome: "submitted", spec: { command: "gh pr create", cwd: worktreeResult.worktreePath, timeoutMs: 1000 }, execResult: { code: 0 }, - loopResult: { outcome: "handoff", totalTurnsUsed: 3, iterationsUsed: 2 }, + loopResult: { outcome: "handoff", totalTurnsUsed: 3, totalCostUsd: 0.42, iterationsUsed: 2 }, }); const exitCode = await runAttempt(["acme/widgets", "7", "--miner-login", "alice", "--json"], { @@ -270,6 +270,7 @@ describe("runAttempt (#5132)", () => { attemptId: "fixed-attempt-id", submissionMode: "observe", totalTurnsUsed: 3, + totalCostUsd: 0.42, iterationsUsed: 2, spec: { command: "gh pr create", cwd: worktreeResult.worktreePath, timeoutMs: 1000 }, execResult: { code: 0 }, diff --git a/test/unit/miner-attempt-runner.test.ts b/test/unit/miner-attempt-runner.test.ts index 9318f4e7b..141d75d2a 100644 --- a/test/unit/miner-attempt-runner.test.ts +++ b/test/unit/miner-attempt-runner.test.ts @@ -92,8 +92,8 @@ function driverReturning(result: CodingAgentDriverResult): CodingAgentDriver { return { async run() { return result; } }; } -function okDriverResult(changedFiles: string[] = ["src/upload.ts"], turnsUsed = 5): CodingAgentDriverResult { - return { ok: true, changedFiles, summary: "added retry logic", turnsUsed }; +function okDriverResult(changedFiles: string[] = ["src/upload.ts"], turnsUsed = 5, costUsd = 0.42): CodingAgentDriverResult { + return { ok: true, changedFiles, summary: "added retry logic", turnsUsed, costUsd }; } // ── Governor "everything allows" fixture, mirroring test/unit/miner-governor-chokepoint.test.ts's own ──────── @@ -160,6 +160,8 @@ describe("runMinerAttempt (#2337) — the real create->review->gate->submit pipe expect(result.spec.command).toContain("'miner/attempt-1'"); expect(result.execResult).toEqual({ ranAt: 10_000 }); expect(result.loopResult.outcome).toBe("handoff"); + // Real per-iteration driver costUsd summed into the loop result (#5135's loop needs this for budgetSpent). + expect(result.loopResult.totalCostUsd).toBe(0.42); }); it("defaults the open_pr body to an empty string when the loop input never set one", async () => { @@ -180,6 +182,20 @@ describe("runMinerAttempt (#2337) — the real create->review->gate->submit pipe expect(claimLedgerListClaims).not.toHaveBeenCalled(); }); + it("abandon mid-loop: a real iteration ran (and was billed) before the ceiling forced abandon", async () => { + // maxIterations: 1 with a driver that never produces a passing self-review runs ONE real iteration, then + // decideNextActionWithReason's own iterationNumber>=maxIterations check abandons -- a genuinely different + // code path from the maxIterations:0 case above (that one never invokes the driver at all). + const failingDriverResult: CodingAgentDriverResult = { ok: false, changedFiles: [], summary: "driver failed", turnsUsed: 2, costUsd: 0.11, error: "driver_error" }; + const deps = baseDeps({ driver: driverReturning(failingDriverResult) }); + const result = await runMinerAttempt(baseAttemptInput({ loopInput: passingLoopInput({ maxIterations: 1 }) }), deps); + + expect(result.outcome).toBe("abandon"); + // The failed iteration's real turnsUsed/costUsd still counted -- an abandoned attempt was still billed. + expect(result.loopResult.totalTurnsUsed).toBe(2); + expect(result.loopResult.totalCostUsd).toBe(0.11); + }); + it("stale: a superseded claim aborts before the submission-gate or governor ever run", async () => { const deps = baseDeps({ claimLedger: { listClaims: () => [{ repoFullName: "acme/widgets", issueNumber: 7, status: "released" }] } }); const result = await runMinerAttempt(baseAttemptInput(), deps); diff --git a/test/unit/miner-loop-cli.test.ts b/test/unit/miner-loop-cli.test.ts index 6d71a345c..29ce61a51 100644 --- a/test/unit/miner-loop-cli.test.ts +++ b/test/unit/miner-loop-cli.test.ts @@ -209,6 +209,7 @@ describe("runLoop (#5135)", () => { attemptId: "loop-attempt-1", submissionMode: "observe", totalTurnsUsed: 4, + totalCostUsd: 0.37, iterationsUsed: 1, execResult: { action: "open_pr", stdout: "https://github.com/acme/widgets/pull/123\n", stderr: "", code: 0, timedOut: false }, }); @@ -249,8 +250,9 @@ describe("runLoop (#5135)", () => { // The claimed item resolved to done (real success), not left in_progress or requeued. expect(after.portfolioQueue.listQueue()).toEqual([expect.objectContaining({ identifier: "issue:7", status: "done" })]); - // Real governor cap usage was saved using runAttempt's own real totalTurnsUsed, not fabricated. + // Real governor cap usage was saved using runAttempt's own real totalTurnsUsed/totalCostUsd, not fabricated. expect(after.governorState.loadCapUsage().turnsTaken).toBe(4); + expect(after.governorState.loadCapUsage().budgetSpent).toBe(0.37); // Re-entry actually fired (a real loop_reentry_decision event, reentered on a merged outcome). const reentryEvents = after.eventLedger.readEvents({}).filter((e) => e.type === "loop_reentry_decision"); @@ -304,6 +306,9 @@ describe("runLoop (#5135)", () => { // The run-loop boundary gate released the in-flight item back to 'queued' on the fresh halt. expect(after.portfolioQueue.listQueue()).toHaveLength(1); expect(after.portfolioQueue.listQueue()[0]).toMatchObject({ status: "queued" }); + // No totalCostUsd on any of these results (mirrors a CLI-subprocess provider, which reports no cost signal + // today) -- budgetSpent stays honestly at 0 across all 3 real attempts, never fabricated. + expect(after.governorState.loadCapUsage().budgetSpent).toBe(0); }); it("REGRESSION: a permanent (AI-usage-policy) block marks the item done instead of re-queuing it forever", async () => {