From b662592b2ab9f9c52e925ec0e030321146c0f279 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 20:04:25 +0300 Subject: [PATCH 1/2] feat(ai-autopilot): generalize surface stream to be event-type generic (#120) --- .changeset/ai-autopilot-generic-surface.md | 5 ++++ .../ai-autopilot/src/surface/events.test.ts | 22 ++++++++++++++ packages/ai-autopilot/src/surface/events.ts | 30 +++++++++++-------- .../ai-autopilot/src/surface/launch.test.ts | 30 +++++++++++++++++++ packages/ai-autopilot/src/surface/launch.ts | 20 ++++++++----- 5 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 .changeset/ai-autopilot-generic-surface.md diff --git a/.changeset/ai-autopilot-generic-surface.md b/.changeset/ai-autopilot-generic-surface.md new file mode 100644 index 0000000..f0138e0 --- /dev/null +++ b/.changeset/ai-autopilot-generic-surface.md @@ -0,0 +1,5 @@ +--- +"@gemstack/ai-autopilot": minor +--- + +Generalize the surface stream to be event-type generic. `EventStream`, `AutopilotHandle`, and `launchAutopilot` now take the event (and result) type as parameters, defaulting to the supervisor's `SupervisorEvent` / `SupervisorRun` so every existing supervisor surface is unchanged. This lets bootstrap (and any future surface) stream its own narration events and return its own result over the same replayable, detached transport. Closes #120. diff --git a/packages/ai-autopilot/src/surface/events.test.ts b/packages/ai-autopilot/src/surface/events.test.ts index 30399ea..b45afa1 100644 --- a/packages/ai-autopilot/src/surface/events.test.ts +++ b/packages/ai-autopilot/src/surface/events.test.ts @@ -83,6 +83,28 @@ describe('formatEvent', () => { } }) +describe('EventStream generic element type', () => { + interface BootEvent { + type: 'narrate' | 'decision' + message: string + } + + it('carries a custom event type through push, history, and iteration', async () => { + const s = new EventStream() + s.push({ type: 'narrate', message: 'picking the stack' }) + s.push({ type: 'decision', message: 'Vike + universal-orm' }) + s.close() + + assert.deepEqual( + s.history().map(e => e.message), + ['picking the stack', 'Vike + universal-orm'], + ) + const seen: BootEvent[] = [] + for await (const e of s) seen.push(e) + assert.deepEqual(seen.map(e => e.type), ['narrate', 'decision']) + }) +}) + describe('terminalSink', () => { it('writes a formatted line per event to the provided writer', () => { const lines: string[] = [] diff --git a/packages/ai-autopilot/src/surface/events.ts b/packages/ai-autopilot/src/surface/events.ts index 0bd86a9..ac0188a 100644 --- a/packages/ai-autopilot/src/surface/events.ts +++ b/packages/ai-autopilot/src/surface/events.ts @@ -1,31 +1,35 @@ import type { SupervisorEvent } from '../types.js' /** - * A replayable, multi-consumer stream of {@link SupervisorEvent}s. It is the - * transport shared by the in-page and background surfaces: buffer every event, - * hand out live async iterators, and replay history from an offset (borrowing - * Flue's Durable-Streams `tail=N`). The terminal surface uses {@link terminalSink} - * directly and does not need this. + * A replayable, multi-consumer stream of events. It is the transport shared by + * the in-page and background surfaces: buffer every event, hand out live async + * iterators, and replay history from an offset (borrowing Flue's Durable-Streams + * `tail=N`). The terminal surface uses {@link terminalSink} directly and does not + * need this. + * + * The element type `E` defaults to {@link SupervisorEvent} so existing supervisor + * surfaces are unchanged; bootstrap and any future surface pass their own event + * type (`new EventStream()`). */ -export class EventStream { - private readonly buffer: SupervisorEvent[] = [] +export class EventStream { + private readonly buffer: E[] = [] private readonly waiters: Array<() => void> = [] private closed = false - /** Append an event. Wire this in as a Supervisor `onEvent`. Ignored once closed. */ - readonly push = (event: SupervisorEvent): void => { + /** Append an event. Wire this in as an `onEvent` sink. Ignored once closed. */ + readonly push = (event: E): void => { if (this.closed) return this.buffer.push(event) for (const wake of this.waiters.splice(0)) wake() } /** Alias for {@link push}, reads well at the `onEvent:` call site. */ - get sink(): (event: SupervisorEvent) => void { + get sink(): (event: E) => void { return this.push } /** Events buffered so far, from `fromOffset` (default 0) — Flue-style tail replay. */ - history(fromOffset = 0): SupervisorEvent[] { + history(fromOffset = 0): E[] { return this.buffer.slice(fromOffset) } @@ -52,14 +56,14 @@ export class EventStream { * Independent iterators each keep their own cursor, so late consumers still * see the full history. */ - [Symbol.asyncIterator](): AsyncIterableIterator { + [Symbol.asyncIterator](): AsyncIterableIterator { let index = 0 const stream = this return { [Symbol.asyncIterator]() { return this }, - next(): Promise> { + next(): Promise> { if (index < stream.buffer.length) { return Promise.resolve({ value: stream.buffer[index++]!, done: false }) } diff --git a/packages/ai-autopilot/src/surface/launch.test.ts b/packages/ai-autopilot/src/surface/launch.test.ts index e9825cb..f456970 100644 --- a/packages/ai-autopilot/src/surface/launch.test.ts +++ b/packages/ai-autopilot/src/surface/launch.test.ts @@ -69,6 +69,36 @@ describe('launchAutopilot', () => { assert.deepEqual(seen, []) }) + it('carries a custom event and result type (bootstrap-shaped surface)', async () => { + interface BootEvent { + type: string + message: string + } + interface BootResult { + app: string + blockers: string[] + } + + let emit!: (e: BootEvent) => void + let finish!: (r: BootResult) => void + const handle = launchAutopilot(onEvent => { + emit = onEvent + return new Promise(resolve => { + finish = resolve + }) + }) + + emit({ type: 'narrate', message: 'scaffolding' }) + assert.deepEqual( + handle.events().map(e => e.message), + ['scaffolding'], + ) + finish({ app: 'shop', blockers: [] }) + const result = await handle.result() + assert.equal(result.app, 'shop') + assert.deepEqual(result.blockers, []) + }) + it('honors an explicit id and generates unique ids otherwise', async () => { const a = launchAutopilot(async () => fakeRun(), { id: 'my-run' }) const b = launchAutopilot(async () => fakeRun()) diff --git a/packages/ai-autopilot/src/surface/launch.ts b/packages/ai-autopilot/src/surface/launch.ts index 43cabf0..d27e330 100644 --- a/packages/ai-autopilot/src/surface/launch.ts +++ b/packages/ai-autopilot/src/surface/launch.ts @@ -11,18 +11,22 @@ export type AutopilotStatus = 'running' | 'done' | 'error' * {@link result}. The same handle backs an in-page surface (iterate `stream()` * and push each event over SSE) and a background process (persist the handle, * let a client poll `status()` + `events(offset)`). + * + * The type params default to the supervisor's `E`/`R` so supervisor surfaces are + * unchanged; bootstrap emits its own narration events and returns its own result, + * so it launches with `AutopilotHandle`. */ -export interface AutopilotHandle { +export interface AutopilotHandle { /** Stable id for this run. */ readonly id: string /** Current lifecycle state. */ status(): AutopilotStatus /** Events emitted so far, from `fromOffset` (default 0). */ - events(fromOffset?: number): SupervisorEvent[] + events(fromOffset?: number): E[] /** A fresh async iterator over events: replays history, then live, then ends. */ - stream(): AsyncIterableIterator + stream(): AsyncIterableIterator /** Resolves with the run's result, or rejects if the run threw. */ - result(): Promise + result(): Promise } /** Options for {@link launchAutopilot}. */ @@ -40,11 +44,11 @@ let counter = 0 * Keeping `start` caller-provided means this surface knows nothing about how the * Supervisor is built; it only owns the event stream and lifecycle. */ -export function launchAutopilot( - start: (onEvent: (event: SupervisorEvent) => void) => Promise, +export function launchAutopilot( + start: (onEvent: (event: E) => void) => Promise, opts: LaunchOptions = {}, -): AutopilotHandle { - const stream = new EventStream() +): AutopilotHandle { + const stream = new EventStream() const id = opts.id ?? `autopilot-${++counter}` let state: AutopilotStatus = 'running' From 62faf16506b71d8ed734e0bd4291bc180762f5d8 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 20:09:57 +0300 Subject: [PATCH 2/2] feat(ai-autopilot): production-grade checklist prompt + { blockers } verdict convention (#121) --- ...ai-autopilot-production-grade-checklist.md | 5 ++ .../ai-autopilot/prompts/production-grade.md | 45 ++++++++++++ packages/ai-autopilot/src/index.ts | 9 ++- packages/ai-autopilot/src/loop/index.ts | 1 + packages/ai-autopilot/src/loop/loop.test.ts | 47 ++++++++++++ packages/ai-autopilot/src/loop/loop.ts | 35 ++++++--- packages/ai-autopilot/src/loop/policy.ts | 2 + packages/ai-autopilot/src/loop/types.ts | 16 ++++- .../ai-autopilot/src/loop/verdict.test.ts | 43 +++++++++++ packages/ai-autopilot/src/loop/verdict.ts | 72 +++++++++++++++++++ .../ai-autopilot/src/prompts/library.test.ts | 7 ++ 11 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 .changeset/ai-autopilot-production-grade-checklist.md create mode 100644 packages/ai-autopilot/prompts/production-grade.md create mode 100644 packages/ai-autopilot/src/loop/verdict.test.ts create mode 100644 packages/ai-autopilot/src/loop/verdict.ts diff --git a/.changeset/ai-autopilot-production-grade-checklist.md b/.changeset/ai-autopilot-production-grade-checklist.md new file mode 100644 index 0000000..b5b55ba --- /dev/null +++ b/.changeset/ai-autopilot-production-grade-checklist.md @@ -0,0 +1,5 @@ +--- +"@gemstack/ai-autopilot": minor +--- + +Add the `production-grade` checklist prompt and a `{ blockers }` verdict convention. The new built-in prompt judges an app against a production-grade checklist (auth, data layer, error handling, instrumentation, emailing, validation, tests, build/config) and ends with a machine-readable `{ "blockers": [...] }` verdict — an empty list means production-grade. `parseVerdict()` / `isPassing()` read it back, and the loop now gates on that outcome: a `PromptOutcome` carries `verdict` and a `passing` flag (executed *and* no blockers), and `continueOnError: false` stops the chain on `!passing`. Backward compatible — with no verdict reported, `passing === ok`; pass `verdict: null` for an execution-only gate. This is what bootstrap's full-fledged loop repeats against until blockers is empty. Closes #121. diff --git a/packages/ai-autopilot/prompts/production-grade.md b/packages/ai-autopilot/prompts/production-grade.md new file mode 100644 index 0000000..46cf648 --- /dev/null +++ b/packages/ai-autopilot/prompts/production-grade.md @@ -0,0 +1,45 @@ +--- +name: production-grade +description: Checklist gate — is this app production-grade? Returns a { blockers } verdict. +appliesTo: ["**/*"] +metadata: + title: Production-grade checklist + loopId: production-grade + passes: 1 +--- + +You are the production-grade **gate** for a Vike + universal-orm app. Judge the +app *as it stands now* against the checklist below. You are not reviewing a diff; +you are deciding whether this is something you would put in front of real users. + +Go through the app and check each area. For every item, either it is genuinely +handled or it is a **blocker** — do not give credit for a stub, a TODO, or a +console.log standing in for the real thing. + +1. **Auth** — real sign-up / sign-in / sign-out, sessions scoped per user, routes + and data access gated. No hardcoded or bypassable auth. +2. **Data layer** — schema + migrations exist and run; every query is scoped to + its owner (no IDOR); writes return the data callers rely on. +3. **Error handling** — no unhandled rejections or naked `throw` reaching the + user; failures surface as proper responses, not a white screen or a 500 stack. +4. **Instrumentation** — logging and error reporting are wired so a failure in + production is observable, not silent. +5. **Emailing** — transactional mail (verification, password reset, receipts) is + sent through a real transport, not a stub, where the app's flows need it. +6. **Validation** — untrusted input is validated at the edge before it reaches + the ORM or filesystem. +7. **Tests** — the core flows have automated tests that actually run and pass. +8. **Build & config** — the app builds clean, secrets come from env (none + committed), and it starts with a documented command. + +Weigh each blocker by whether it would actually bite a real user or operator; +skip theoretical nits. When you are unsure an item is truly done, treat it as a +blocker and say why. + +End your response with **exactly one** fenced JSON block giving the verdict — the +list of concrete, still-required work. An empty list means the app is +production-grade. + +```json +{ "blockers": ["short, actionable description of each remaining gap"] } +``` diff --git a/packages/ai-autopilot/src/index.ts b/packages/ai-autopilot/src/index.ts index e230baa..4d84997 100644 --- a/packages/ai-autopilot/src/index.ts +++ b/packages/ai-autopilot/src/index.ts @@ -50,9 +50,13 @@ * - {@link Loop} — match an event to a prompt chain and run it (N fresh passes) * - {@link definePrompt} / {@link defineRule} — author prompts and policy rules * - {@link defaultLoopRules} — the built-in web-app policy + * - {@link parseVerdict} / {@link isPassing} — the `{ blockers }` verdict the loop + * gates on, so it stops on a review's *outcome*, not just execution * * The prompts library supplies the loop's prompt *bodies* as data (stack-aware - * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base. + * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base, + * and the `production-grade` checklist bootstrap's full-fledged loop repeats + * against. * * - {@link builtinLibrary} — load the shipped, stack-aware prompt bodies * - {@link loopPromptsFor} — materialize a library into loop prompts by id @@ -140,6 +144,9 @@ export { defaultLoopRules, LOOP_EVENTS, LOOP_PROMPTS, + parseVerdict, + isPassing, + type Verdict, type LoopOptions, type LoopEvent, type LoopContext, diff --git a/packages/ai-autopilot/src/loop/index.ts b/packages/ai-autopilot/src/loop/index.ts index bd78bab..846d4c5 100644 --- a/packages/ai-autopilot/src/loop/index.ts +++ b/packages/ai-autopilot/src/loop/index.ts @@ -9,6 +9,7 @@ export { definePrompt, defineRule, LoopError } from './define.js' export { Loop, createLoop, type LoopOptions } from './loop.js' export { defaultLoopRules, LOOP_EVENTS, LOOP_PROMPTS } from './policy.js' +export { parseVerdict, isPassing, type Verdict } from './verdict.js' export type { LoopEvent, LoopContext, diff --git a/packages/ai-autopilot/src/loop/loop.test.ts b/packages/ai-autopilot/src/loop/loop.test.ts index b7664eb..2be42b6 100644 --- a/packages/ai-autopilot/src/loop/loop.test.ts +++ b/packages/ai-autopilot/src/loop/loop.test.ts @@ -111,6 +111,53 @@ describe('Loop — failure policy', () => { }) }) +describe('Loop — verdict gating', () => { + const rules = [defineRule({ on: 'check', run: ['production-grade', 'after'] })] + const after = definePrompt({ id: 'after', run: () => 'ran' }) + + it('parses a verdict onto the outcome and marks blockers not-passing', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no auth"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + const outcome = result.outcomes[0]! + assert.equal(outcome.ok, true) // it executed + assert.equal(outcome.passing, false) // but it reported blockers + assert.deepEqual(outcome.verdict?.blockers, ['no auth']) + }) + + it('an empty blockers verdict is passing', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": [] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) + }) + + it('gates the chain on blockers when continueOnError is false', async () => { + const events: LoopProgress[] = [] + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no tests"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after], continueOnError: false, onEvent: e => events.push(e) }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes.length, 1) // `after` was gated out on the verdict, not an error + assert.ok(events.some(e => e.type === 'gate-stop' && e.promptId === 'production-grade')) + }) + + it('a prompt with no verdict still passes when it executes (backward compatible)', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => 'no json here' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) + assert.equal(result.outcomes[0]?.verdict, undefined) + }) + + it('verdict: null disables parsing (execution-only gate)', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["x"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after], verdict: null }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) // blockers ignored + assert.equal(result.outcomes[0]?.verdict, undefined) + }) +}) + describe('Loop — decisions + watch', () => { it('exposes the ledger to prompts via context', async () => { const ledger = new DecisionLedger() diff --git a/packages/ai-autopilot/src/loop/loop.ts b/packages/ai-autopilot/src/loop/loop.ts index c11c62a..beb380d 100644 --- a/packages/ai-autopilot/src/loop/loop.ts +++ b/packages/ai-autopilot/src/loop/loop.ts @@ -8,6 +8,8 @@ import type { PassResult, PromptOutcome, } from './types.js' +import type { Verdict } from './verdict.js' +import { parseVerdict } from './verdict.js' /** Options for {@link Loop}. */ export interface LoopOptions { @@ -21,12 +23,20 @@ export interface LoopOptions { * Failure policy, the sync-vs-async knob: * - `true` (default) — fire-and-report: every matched prompt runs regardless * of a prior one failing. - * - `false` — blocking gate: if a prompt's final pass fails, stop the chain - * (a `gate-stop` event); the remaining prompts do not run. The v1 - * approximation of "block until the review passes" (gates on execution - * failure; gating on a review's *verdict* waits for prompts to return one). + * - `false` — blocking gate: if a prompt does not *pass*, stop the chain (a + * `gate-stop` event); the remaining prompts do not run. "Pass" means the + * final pass executed and, when a {@link LoopOptions.verdict} parser is set, + * returned no blockers. */ continueOnError?: boolean + /** + * Parse a prompt's final-pass text into a {@link Verdict}. When set, the loop + * gates on the *outcome* a prompt reports (`{ blockers }`), not just whether it + * executed — a prompt that runs but returns blockers is not passing. Defaults + * to {@link parseVerdict} (reads a fenced ```json `{ "blockers": [...] }`). + * Pass `null` to disable verdict gating entirely (execution-only gate). + */ + verdict?: ((text: string) => Verdict | undefined) | null /** * Observe progress. Isolated: a throwing callback is logged and swallowed, so * an observer bug cannot abort a run. @@ -54,6 +64,7 @@ export class Loop { private readonly prompts: Map private readonly ledger?: DecisionLedger private readonly continueOnError: boolean + private readonly parseVerdict: ((text: string) => Verdict | undefined) | null private readonly emit: (event: LoopProgress) => void constructor(opts: LoopOptions) { @@ -64,6 +75,7 @@ export class Loop { this.prompts = indexPrompts(opts.prompts) if (opts.ledger !== undefined) this.ledger = opts.ledger this.continueOnError = opts.continueOnError ?? true + this.parseVerdict = opts.verdict === undefined ? parseVerdict : opts.verdict this.emit = makeEmitter(opts.onEvent) } @@ -99,14 +111,14 @@ export class Loop { const prompt = this.prompts.get(id) if (!prompt) { this.emit({ type: 'unknown-prompt', promptId: id }) - outcomes.push({ promptId: id, passes: [], ok: false }) + outcomes.push({ promptId: id, passes: [], ok: false, passing: false }) continue } const outcome = await this.runPrompt(prompt, event) outcomes.push(outcome) - if (!outcome.ok && !this.continueOnError) { + if (!outcome.passing && !this.continueOnError) { this.emit({ type: 'gate-stop', promptId: id }) break } @@ -147,9 +159,14 @@ export class Loop { this.emit({ type: 'pass', promptId: prompt.id, result, passes: prompt.passes }) } - const ok = passes[passes.length - 1]?.ok ?? false - this.emit({ type: 'prompt-done', promptId: prompt.id, ok }) - return { promptId: prompt.id, passes, ok } + const last = passes[passes.length - 1] + const ok = last?.ok ?? false + // A verdict only means something when the pass that produced it executed. + const verdict = ok && this.parseVerdict ? this.parseVerdict(last!.text) : undefined + const passing = ok && (verdict ? verdict.blockers.length === 0 : true) + + this.emit({ type: 'prompt-done', promptId: prompt.id, ok, passing, ...(verdict ? { verdict } : {}) }) + return { promptId: prompt.id, passes, ok, passing, ...(verdict ? { verdict } : {}) } } } diff --git a/packages/ai-autopilot/src/loop/policy.ts b/packages/ai-autopilot/src/loop/policy.ts index 1f42a06..201e404 100644 --- a/packages/ai-autopilot/src/loop/policy.ts +++ b/packages/ai-autopilot/src/loop/policy.ts @@ -23,6 +23,8 @@ export const LOOP_PROMPTS = { security: 'security', qa: 'qa', ux: 'ux', + /** The checklist bootstrap's full-fledged loop repeats against; returns a `{ blockers }` verdict. */ + productionGrade: 'production-grade', } as const /** diff --git a/packages/ai-autopilot/src/loop/types.ts b/packages/ai-autopilot/src/loop/types.ts index 8069cf7..7319e39 100644 --- a/packages/ai-autopilot/src/loop/types.ts +++ b/packages/ai-autopilot/src/loop/types.ts @@ -1,4 +1,5 @@ import type { DecisionLedger } from '../decisions/ledger.js' +import type { Verdict } from './verdict.js' /** * "The loop" — the event-to-prompt-chain policy. When the agent does something @@ -95,8 +96,19 @@ export interface PassResult { export interface PromptOutcome { promptId: string passes: PassResult[] - /** True when the final pass succeeded — the result that matters. */ + /** True when the final pass succeeded (executed without throwing). */ ok: boolean + /** + * The structured verdict parsed from the final pass, when the loop has a + * `verdict` parser and the prompt returned one. Absent otherwise. + */ + verdict?: Verdict + /** + * The gating result: executed cleanly *and*, when a verdict was parsed, has no + * blockers. Equals {@link ok} when no verdict parser is configured, so the gate + * is backward-compatible. This is what `continueOnError: false` stops on. + */ + passing: boolean } /** The full result of handling one {@link LoopEvent}. */ @@ -115,6 +127,6 @@ export type LoopProgress = | { type: 'unknown-prompt'; promptId: string } | { type: 'prompt-start'; promptId: string; passes: number } | { type: 'pass'; promptId: string; result: PassResult; passes: number } - | { type: 'prompt-done'; promptId: string; ok: boolean } + | { type: 'prompt-done'; promptId: string; ok: boolean; passing: boolean; verdict?: Verdict } | { type: 'gate-stop'; promptId: string } | { type: 'done'; event: LoopEvent; outcomes: PromptOutcome[] } diff --git a/packages/ai-autopilot/src/loop/verdict.test.ts b/packages/ai-autopilot/src/loop/verdict.test.ts new file mode 100644 index 0000000..7d6d402 --- /dev/null +++ b/packages/ai-autopilot/src/loop/verdict.test.ts @@ -0,0 +1,43 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { parseVerdict, isPassing } from './verdict.js' + +describe('parseVerdict', () => { + it('reads a fenced json block with an empty blockers list (passing)', () => { + const text = 'Everything checks out.\n\n```json\n{ "blockers": [] }\n```' + const v = parseVerdict(text) + assert.deepEqual(v, { blockers: [] }) + assert.equal(isPassing(v), true) + }) + + it('reads blockers and trims/drops empties', () => { + const text = '```json\n{ "blockers": ["no auth", " ", "no tests"] }\n```' + const v = parseVerdict(text) + assert.deepEqual(v?.blockers, ['no auth', 'no tests']) + assert.equal(isPassing(v), false) + }) + + it('keeps an optional notes field', () => { + const v = parseVerdict('```json\n{ "blockers": [], "notes": "ship it" }\n```') + assert.equal(v?.notes, 'ship it') + }) + + it('takes the last verdict when several appear (a corrected pass wins)', () => { + const text = + 'draft:\n```json\n{ "blockers": ["x"] }\n```\nfinal:\n```json\n{ "blockers": [] }\n```' + assert.deepEqual(parseVerdict(text)?.blockers, []) + }) + + it('falls back to a trailing bare object', () => { + const v = parseVerdict('Verdict: { "blockers": ["needs migrations"] }') + assert.deepEqual(v?.blockers, ['needs migrations']) + }) + + it('returns undefined when there is no verdict (distinct from failing)', () => { + assert.equal(parseVerdict('just prose, no json'), undefined) + assert.equal(parseVerdict(''), undefined) + // an object without a blockers array is not a verdict + assert.equal(parseVerdict('```json\n{ "ok": true }\n```'), undefined) + assert.equal(isPassing(undefined), false) + }) +}) diff --git a/packages/ai-autopilot/src/loop/verdict.ts b/packages/ai-autopilot/src/loop/verdict.ts new file mode 100644 index 0000000..4802370 --- /dev/null +++ b/packages/ai-autopilot/src/loop/verdict.ts @@ -0,0 +1,72 @@ +/** + * The verdict convention — a light, structured outcome a prompt can return so + * the loop gates on *what a review concluded*, not just whether the prompt ran. + * + * A verdict is `{ blockers: string[] }`: an empty list means "nothing left to + * fix" (passing); a non-empty list is the concrete work still required. The + * production-grade checklist prompt returns one, and bootstrap's full-fledged + * loop repeats against it until `blockers` is empty. This answers the note left + * in the loop (#113): the v1 gate stopped on execution failure; a verdict lets it + * stop on the review's outcome. + * + * The transport is text (prompts return text), so the convention is: end the + * output with a fenced ```json block holding `{ "blockers": [...] }`. + * {@link parseVerdict} reads it back; {@link isPassing} is the empty-blockers test. + */ + +/** A structured prompt outcome the loop can gate on. */ +export interface Verdict { + /** Concrete work still required; empty means passing. */ + readonly blockers: string[] + /** Optional free-text summary the agent may include. */ + readonly notes?: string +} + +/** True when a verdict has no blockers left. A missing verdict is not passing. */ +export function isPassing(verdict: Verdict | undefined): boolean { + return verdict !== undefined && verdict.blockers.length === 0 +} + +// Matches fenced code blocks, capturing an optional language tag and the body. +const FENCE = /```(?:[a-zA-Z0-9]*)\n([\s\S]*?)```/g + +/** + * Parse a {@link Verdict} out of a prompt's text output. Reads the **last** + * fenced code block that is a JSON object with a `blockers` array (last so a + * later, corrected pass wins over an earlier draft in the same text), falling + * back to a trailing bare `{ ... }` object. Returns `undefined` when no verdict + * is present so the caller can tell "not passing" from "did not report". + */ +export function parseVerdict(text: string): Verdict | undefined { + if (!text) return undefined + + const candidates: string[] = [] + for (const m of text.matchAll(FENCE)) candidates.push(m[1]!) + // Fallback: a bare object somewhere in the text (take the last `{`-to-`}` run). + const lastOpen = text.lastIndexOf('{') + const lastClose = text.lastIndexOf('}') + if (lastOpen !== -1 && lastClose > lastOpen) candidates.push(text.slice(lastOpen, lastClose + 1)) + + // Later candidates win, so scan from the end. + for (let i = candidates.length - 1; i >= 0; i--) { + const verdict = coerce(candidates[i]!) + if (verdict) return verdict + } + return undefined +} + +function coerce(raw: string): Verdict | undefined { + let parsed: unknown + try { + parsed = JSON.parse(raw.trim()) + } catch { + return undefined + } + if (typeof parsed !== 'object' || parsed === null) return undefined + const obj = parsed as Record + if (!Array.isArray(obj.blockers)) return undefined + + const blockers = obj.blockers.map(b => String(b).trim()).filter(Boolean) + const notes = typeof obj.notes === 'string' && obj.notes.trim() ? obj.notes.trim() : undefined + return Object.freeze({ blockers, ...(notes ? { notes } : {}) }) +} diff --git a/packages/ai-autopilot/src/prompts/library.test.ts b/packages/ai-autopilot/src/prompts/library.test.ts index 6cf073a..c6850eb 100644 --- a/packages/ai-autopilot/src/prompts/library.test.ts +++ b/packages/ai-autopilot/src/prompts/library.test.ts @@ -32,6 +32,13 @@ describe('builtin prompts', () => { assert.ok(ids.includes(id), `ships "${id}"`) } }) + + it('ships the production-grade checklist and instructs the { blockers } verdict', async () => { + const gate = (await builtinLibrary()).get(LOOP_PROMPTS.productionGrade) + assert.ok(gate, 'production-grade prompt exists') + assert.match(gate!.instructions, /blockers/) + assert.match(gate!.instructions, /```json/) + }) }) describe('PromptLibrary', () => {