Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions packages/gittensory-engine/src/miner/agent-sdk-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ export function createAgentSdkCodingAgentDriver(

const turnsUsed =
typeof resultMessage?.num_turns === "number" ? resultMessage.num_turns : undefined;
// Real dollar cost: the SDK's own SDKResultSuccess/SDKResultError message types both declare
// `total_cost_usd: number` unconditionally -- present whenever a result message arrived at all, success
// or not (the session was billed either way), absent only when the stream produced no result message.
const costUsd =
typeof resultMessage?.total_cost_usd === "number" ? resultMessage.total_cost_usd : undefined;
const resultText =
typeof resultMessage?.result === "string" ? redactSecrets(resultMessage.result) : "";
const transcript = redactSecrets(
Expand All @@ -169,6 +174,7 @@ export function createAgentSdkCodingAgentDriver(
summary: "agent sdk session did not complete successfully",
transcript,
turnsUsed,
costUsd,
error: `agent_sdk_${subtype === "success" ? "errored" : subtype}`,
};
}
Expand All @@ -179,6 +185,7 @@ export function createAgentSdkCodingAgentDriver(
summary: resultText.slice(0, MAX_REDACTED_TEXT_LENGTH) || `coding agent completed with ${changedFiles.size} changed file(s)`,
transcript,
turnsUsed,
costUsd,
};
},
};
Expand Down
3 changes: 3 additions & 0 deletions packages/gittensory-engine/src/miner/coding-agent-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ export type CodingAgentDriverResult = {
/** Opaque provider transcript for operator inspection; absent when the driver did not run. */
transcript?: string | undefined;
turnsUsed?: number | undefined;
/** Real dollar cost of this driver run, when the provider reports one. Absent (not zero) when the provider
* never got far enough to have a cost, or reports no cost signal at all -- never fabricated. */
costUsd?: number | undefined;
error?: string | undefined;
};

Expand Down
13 changes: 10 additions & 3 deletions packages/gittensory-engine/src/miner/iterate-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ export type IterateLoopResult = {
iterationsUsed: number;
/** Cumulative `turnsUsed` summed across every iteration that ran. */
totalTurnsUsed: number;
/** Cumulative real dollar cost summed across every iteration that ran, from each iteration's
* `CodingAgentDriverResult.costUsd`. Only the `agent-sdk` provider reports this today (the CLI-subprocess
* providers report no cost signal) -- always `0` for a provider that never reports one, never fabricated. */
totalCostUsd: number;
iterations: readonly IterateLoopIterationRecord[];
/** Populated only when `outcome === "handoff"`. */
handoffPacket?: HandoffPacket | undefined;
Expand Down Expand Up @@ -228,7 +232,7 @@ function immediateAbandonNoIterationsPermitted(input: IterateLoopInput, deps: It
reason: decision.reason,
payload: { iterationNumber: 0, action: decision.action, abandonReason: decision.abandonReason },
});
return { outcome: "abandon", finalDecision: decision, iterationsUsed: 0, totalTurnsUsed: 0, iterations: [] };
return { outcome: "abandon", finalDecision: decision, iterationsUsed: 0, totalTurnsUsed: 0, totalCostUsd: 0, iterations: [] };
}

/**
Expand Down Expand Up @@ -262,6 +266,7 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD
const iterations: IterateLoopIterationRecord[] = [];
let previousBlockerCodes: readonly string[] | null = null;
let totalTurnsUsed = 0;
let totalCostUsd = 0;

for (let iterationNumber = 1; iterationNumber <= maxIterations; iterationNumber += 1) {
const driverResult = await runDriverSafely(input, deps, {
Expand All @@ -272,6 +277,7 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD
maxTurns: input.maxTurnsPerIteration,
});
totalTurnsUsed += driverResult.turnsUsed ?? 0;
totalCostUsd += driverResult.costUsd ?? 0;

const { outcome: selfReview, verdict } = evaluateSelfReviewOutcome(input, driverResult, deps);

Expand All @@ -296,12 +302,13 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD
finalDecision: decision,
iterationsUsed: iterationNumber,
totalTurnsUsed,
totalCostUsd,
iterations,
handoffPacket: buildHandoffPacket(input, verdict as SelfReviewVerdict, driverResult),
};
}
if (decision.action === "abandon") {
return { outcome: "abandon", finalDecision: decision, iterationsUsed: iterationNumber, totalTurnsUsed, iterations };
return { outcome: "abandon", finalDecision: decision, iterationsUsed: iterationNumber, totalTurnsUsed, totalCostUsd, iterations };
}
previousBlockerCodes = blockerCodesFromContinuingOutcome(selfReview);
}
Expand All @@ -313,5 +320,5 @@ export async function runIterateLoop(input: IterateLoopInput, deps: IterateLoopD
* this package's fail-closed discipline, in case a future edit to the precedence ladder ever removes that
* guarantee. */
const fallbackDecision: IterateLoopDecision = { action: "abandon", abandonReason: "max_iterations_reached", reason: "Iterate loop exhausted its iteration budget." };
return { outcome: "abandon", finalDecision: fallbackDecision, iterationsUsed: maxIterations, totalTurnsUsed, iterations };
return { outcome: "abandon", finalDecision: fallbackDecision, iterationsUsed: maxIterations, totalTurnsUsed, totalCostUsd, iterations };
}
1 change: 1 addition & 0 deletions packages/gittensory-miner/lib/attempt-cli.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export type AttemptCliResult =
outcome: `attempt_${RunMinerAttemptResult["outcome"]}`;
submissionMode: "observe" | "enforce";
totalTurnsUsed: number;
totalCostUsd: number;
iterationsUsed: number;
reason?: string;
decision?: unknown;
Expand Down
10 changes: 7 additions & 3 deletions packages/gittensory-miner/lib/attempt-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -374,10 +374,14 @@ export async function runAttempt(args, options = {}) {
mode,
attemptId,
submissionMode: amsPolicy.spec.submissionMode,
// Every runMinerAttempt outcome carries a real loopResult (#5135's loop needs its genuine turn-usage to
// save real GovernorCapUsage via governor-state.js's saveCapUsage -- nothing else in the codebase calls
// it yet). Surfaced flat rather than the whole loopResult object, matching this result's own shallow shape.
// Every runMinerAttempt outcome carries a real loopResult (#5135's loop needs its genuine turn-usage and
// cost to save real GovernorCapUsage via governor-state.js's saveCapUsage -- nothing else in the codebase
// calls it yet). Surfaced flat rather than the whole loopResult object, matching this result's own
// shallow shape. costUsd is real only for the agent-sdk provider (its own SDK result message reports
// total_cost_usd); CLI-subprocess providers (claude-cli/codex-cli) report no cost signal today, so this
// is 0 for those -- an honest absence, not a fabricated number.
totalTurnsUsed: result.loopResult.totalTurnsUsed,
totalCostUsd: result.loopResult.totalCostUsd,
iterationsUsed: result.loopResult.iterationsUsed,
...("reason" in result ? { reason: result.reason } : {}),
...("decision" in result ? { decision: result.decision } : {}),
Expand Down
6 changes: 5 additions & 1 deletion packages/gittensory-miner/lib/loop-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,11 @@ export async function runLoop(args, options = {}) {
const cycleElapsedMs = nowMsFn() - cycleStartMs;

usage = {
budgetSpent: usage.budgetSpent,
// Real for the agent-sdk provider (its own SDK result message reports total_cost_usd, wired through
// runMinerAttempt's real loopResult.totalCostUsd); the CLI-subprocess providers (claude-cli/codex-cli)
// report no cost signal today, so this contributes 0 for those runs -- an honest absence, not a
// fabricated number. A capLimits.budget dimension only ever meaningfully trips against agent-sdk spend.
budgetSpent: usage.budgetSpent + (lastResult?.totalCostUsd ?? 0),
turnsTaken: usage.turnsTaken + (lastResult?.totalTurnsUsed ?? 0),
elapsedMs: usage.elapsedMs + cycleElapsedMs,
};
Expand Down
15 changes: 14 additions & 1 deletion test/contract/coding-agent-driver-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ const DRIVER_HARNESSES: DriverHarness[] = [
return;
}
yield { type: "assistant", message: { content: [{ type: "tool_use", name: "Edit", input: { file_path: "src/a.ts" } }] } };
yield { type: "result", subtype: "success", is_error: false, num_turns: 3, result: "edited src/a.ts" };
yield { type: "result", subtype: "success", is_error: false, num_turns: 3, total_cost_usd: 0.15, result: "edited src/a.ts" };
})();
},
});
Expand All @@ -130,6 +130,7 @@ function expectDriverResultShape(result: CodingAgentDriverResult): void {
expect(result.summary.length).toBeGreaterThan(0);
if (result.transcript !== undefined) expect(typeof result.transcript).toBe("string");
if (result.turnsUsed !== undefined) expect(typeof result.turnsUsed).toBe("number");
if (result.costUsd !== undefined) expect(typeof result.costUsd).toBe("number");
if (result.error !== undefined) {
expect(typeof result.error).toBe("string");
expect(result.error.length).toBeGreaterThan(0);
Expand Down Expand Up @@ -229,4 +230,16 @@ describe("documented divergences, locked in explicitly", () => {
const result = await driver.run(task);
expect(result.turnsUsed).toBe(3);
});

it("CLI driver reports no cost signal (a subprocess exposes none without --output-format json) — costUsd stays undefined", async () => {
const { driver } = DRIVER_HARNESSES[0]!.make("success");
const result = await driver.run(task);
expect(result.costUsd).toBeUndefined();
});

it("Agent-SDK driver reports the result frame's real total_cost_usd", async () => {
const { driver } = DRIVER_HARNESSES[1]!.make("success");
const result = await driver.run(task);
expect(result.costUsd).toBe(0.15);
});
});
3 changes: 2 additions & 1 deletion test/unit/miner-attempt-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ describe("runAttempt (#5132)", () => {
outcome: "submitted",
spec: { command: "gh pr create", cwd: worktreeResult.worktreePath, timeoutMs: 1000 },
execResult: { code: 0 },
loopResult: { outcome: "handoff", totalTurnsUsed: 3, iterationsUsed: 2 },
loopResult: { outcome: "handoff", totalTurnsUsed: 3, totalCostUsd: 0.42, iterationsUsed: 2 },
});

const exitCode = await runAttempt(["acme/widgets", "7", "--miner-login", "alice", "--json"], {
Expand All @@ -270,6 +270,7 @@ describe("runAttempt (#5132)", () => {
attemptId: "fixed-attempt-id",
submissionMode: "observe",
totalTurnsUsed: 3,
totalCostUsd: 0.42,
iterationsUsed: 2,
spec: { command: "gh pr create", cwd: worktreeResult.worktreePath, timeoutMs: 1000 },
execResult: { code: 0 },
Expand Down
20 changes: 18 additions & 2 deletions test/unit/miner-attempt-runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ function driverReturning(result: CodingAgentDriverResult): CodingAgentDriver {
return { async run() { return result; } };
}

function okDriverResult(changedFiles: string[] = ["src/upload.ts"], turnsUsed = 5): CodingAgentDriverResult {
return { ok: true, changedFiles, summary: "added retry logic", turnsUsed };
function okDriverResult(changedFiles: string[] = ["src/upload.ts"], turnsUsed = 5, costUsd = 0.42): CodingAgentDriverResult {
return { ok: true, changedFiles, summary: "added retry logic", turnsUsed, costUsd };
}

// ── Governor "everything allows" fixture, mirroring test/unit/miner-governor-chokepoint.test.ts's own ────────
Expand Down Expand Up @@ -160,6 +160,8 @@ describe("runMinerAttempt (#2337) — the real create->review->gate->submit pipe
expect(result.spec.command).toContain("'miner/attempt-1'");
expect(result.execResult).toEqual({ ranAt: 10_000 });
expect(result.loopResult.outcome).toBe("handoff");
// Real per-iteration driver costUsd summed into the loop result (#5135's loop needs this for budgetSpent).
expect(result.loopResult.totalCostUsd).toBe(0.42);
});

it("defaults the open_pr body to an empty string when the loop input never set one", async () => {
Expand All @@ -180,6 +182,20 @@ describe("runMinerAttempt (#2337) — the real create->review->gate->submit pipe
expect(claimLedgerListClaims).not.toHaveBeenCalled();
});

it("abandon mid-loop: a real iteration ran (and was billed) before the ceiling forced abandon", async () => {
// maxIterations: 1 with a driver that never produces a passing self-review runs ONE real iteration, then
// decideNextActionWithReason's own iterationNumber>=maxIterations check abandons -- a genuinely different
// code path from the maxIterations:0 case above (that one never invokes the driver at all).
const failingDriverResult: CodingAgentDriverResult = { ok: false, changedFiles: [], summary: "driver failed", turnsUsed: 2, costUsd: 0.11, error: "driver_error" };
const deps = baseDeps({ driver: driverReturning(failingDriverResult) });
const result = await runMinerAttempt(baseAttemptInput({ loopInput: passingLoopInput({ maxIterations: 1 }) }), deps);

expect(result.outcome).toBe("abandon");
// The failed iteration's real turnsUsed/costUsd still counted -- an abandoned attempt was still billed.
expect(result.loopResult.totalTurnsUsed).toBe(2);
expect(result.loopResult.totalCostUsd).toBe(0.11);
});

it("stale: a superseded claim aborts before the submission-gate or governor ever run", async () => {
const deps = baseDeps({ claimLedger: { listClaims: () => [{ repoFullName: "acme/widgets", issueNumber: 7, status: "released" }] } });
const result = await runMinerAttempt(baseAttemptInput(), deps);
Expand Down
7 changes: 6 additions & 1 deletion test/unit/miner-loop-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ describe("runLoop (#5135)", () => {
attemptId: "loop-attempt-1",
submissionMode: "observe",
totalTurnsUsed: 4,
totalCostUsd: 0.37,
iterationsUsed: 1,
execResult: { action: "open_pr", stdout: "https://github.com/acme/widgets/pull/123\n", stderr: "", code: 0, timedOut: false },
});
Expand Down Expand Up @@ -249,8 +250,9 @@ describe("runLoop (#5135)", () => {
// The claimed item resolved to done (real success), not left in_progress or requeued.
expect(after.portfolioQueue.listQueue()).toEqual([expect.objectContaining({ identifier: "issue:7", status: "done" })]);

// Real governor cap usage was saved using runAttempt's own real totalTurnsUsed, not fabricated.
// Real governor cap usage was saved using runAttempt's own real totalTurnsUsed/totalCostUsd, not fabricated.
expect(after.governorState.loadCapUsage().turnsTaken).toBe(4);
expect(after.governorState.loadCapUsage().budgetSpent).toBe(0.37);

// Re-entry actually fired (a real loop_reentry_decision event, reentered on a merged outcome).
const reentryEvents = after.eventLedger.readEvents({}).filter((e) => e.type === "loop_reentry_decision");
Expand Down Expand Up @@ -304,6 +306,9 @@ describe("runLoop (#5135)", () => {
// The run-loop boundary gate released the in-flight item back to 'queued' on the fresh halt.
expect(after.portfolioQueue.listQueue()).toHaveLength(1);
expect(after.portfolioQueue.listQueue()[0]).toMatchObject({ status: "queued" });
// No totalCostUsd on any of these results (mirrors a CLI-subprocess provider, which reports no cost signal
// today) -- budgetSpent stays honestly at 0 across all 3 real attempts, never fabricated.
expect(after.governorState.loadCapUsage().budgetSpent).toBe(0);
});

it("REGRESSION: a permanent (AI-usage-policy) block marks the item done instead of re-queuing it forever", async () => {
Expand Down