Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/missing-verdict-fail-closed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@gemstack/framework': patch
---

Fail closed when a checklist reply omits the required `{ blockers }` verdict. A verdict-less reply was scored as empty-blockers (production-grade) and stopped the loop; it now surfaces a blocker so the loop re-prompts instead of declaring the app done off an unverifiable reply.
5 changes: 3 additions & 2 deletions packages/framework/src/steps.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
driverImprove,
extendPrompt,
isWorkspaceEmpty,
MISSING_VERDICT_BLOCKER,
parseArchitectPlan,
} from './steps.js'

Expand Down Expand Up @@ -107,10 +108,10 @@ test('driverChecklist parses the { blockers } verdict', async () => {
assert.deepEqual(verdict.blockers, ['no auth'])
})

test('driverChecklist treats a verdict-less reply as passing', async () => {
test('driverChecklist fails closed on a verdict-less reply (not passing)', async () => {
const session = await new FakeDriver({ turns: [{ text: 'looks fine to me' }] }).start({ cwd: '/ws' })
const verdict = await driverChecklist(session)({ pass: 1, plan: PLAN, intent: 'x', blockers: [] })
assert.deepEqual(verdict.blockers, [])
assert.deepEqual(verdict.blockers, [MISSING_VERDICT_BLOCKER])
})

test('deployWith runs the target against the decided plan and uses its name', async () => {
Expand Down
11 changes: 8 additions & 3 deletions packages/framework/src/steps.ts
Original file line number Diff line number Diff line change
Expand Up @@ -267,12 +267,17 @@ export function driverChecklist(
...(opts.system ? { system: opts.system } : {}),
...(ctx.signal ? { signal: ctx.signal } : {}),
})
// A missing verdict is treated as "reported nothing concrete": passing, so a
// terse agent does not wedge the loop. parseVerdict returns undefined then.
return parseVerdict(turn.text) ?? { blockers: [] }
// Fail closed: a reply with no parseable { blockers } verdict is not a pass.
// Surface it as a blocker so the loop re-prompts (and, at maxPasses, stops with
// it) rather than declaring the app production-grade off an unverifiable reply.
return parseVerdict(turn.text) ?? { blockers: [MISSING_VERDICT_BLOCKER] }
}
}

/** The blocker surfaced when a checklist reply omits the required `{ blockers }` verdict. */
export const MISSING_VERDICT_BLOCKER =
'End your reply with the required fenced ```json { "blockers": [...] } verdict; it was missing.'

/** The improve step: a fresh invocation that fixes the current blockers. */
export function driverImprove(
session: DriverSession,
Expand Down
Loading