feat(ai-autopilot): production-grade checklist prompt + { blockers } verdict convention (#121)

.changeset/ai-autopilot-generic-surface.md

@@ -0,0 +1,5 @@
+---
+"@gemstack/ai-autopilot": minor
+---
+
+Generalize the surface stream to be event-type generic. `EventStream<E>`, `AutopilotHandle<E, R>`, and `launchAutopilot<E, R>` now take the event (and result) type as parameters, defaulting to the supervisor's `SupervisorEvent` / `SupervisorRun` so every existing supervisor surface is unchanged. This lets bootstrap (and any future surface) stream its own narration events and return its own result over the same replayable, detached transport. Closes #120.

.changeset/ai-autopilot-production-grade-checklist.md

@@ -0,0 +1,5 @@
+---
+"@gemstack/ai-autopilot": minor
+---
+
+Add the `production-grade` checklist prompt and a `{ blockers }` verdict convention. The new built-in prompt judges an app against a production-grade checklist (auth, data layer, error handling, instrumentation, emailing, validation, tests, build/config) and ends with a machine-readable `{ "blockers": [...] }` verdict — an empty list means production-grade. `parseVerdict()` / `isPassing()` read it back, and the loop now gates on that outcome: a `PromptOutcome` carries `verdict` and a `passing` flag (executed *and* no blockers), and `continueOnError: false` stops the chain on `!passing`. Backward compatible — with no verdict reported, `passing === ok`; pass `verdict: null` for an execution-only gate. This is what bootstrap's full-fledged loop repeats against until blockers is empty. Closes #121.

packages/ai-autopilot/prompts/production-grade.md

@@ -0,0 +1,45 @@
+---
+name: production-grade
+description: Checklist gate — is this app production-grade? Returns a { blockers } verdict.
+appliesTo: ["**/*"]
+metadata:
+  title: Production-grade checklist
+  loopId: production-grade
+  passes: 1
+---
+
+You are the production-grade **gate** for a Vike + universal-orm app. Judge the
+app *as it stands now* against the checklist below. You are not reviewing a diff;
+you are deciding whether this is something you would put in front of real users.
+
+Go through the app and check each area. For every item, either it is genuinely
+handled or it is a **blocker** — do not give credit for a stub, a TODO, or a
+console.log standing in for the real thing.
+
+1. **Auth** — real sign-up / sign-in / sign-out, sessions scoped per user, routes
+   and data access gated. No hardcoded or bypassable auth.
+2. **Data layer** — schema + migrations exist and run; every query is scoped to
+   its owner (no IDOR); writes return the data callers rely on.
+3. **Error handling** — no unhandled rejections or naked `throw` reaching the
+   user; failures surface as proper responses, not a white screen or a 500 stack.
+4. **Instrumentation** — logging and error reporting are wired so a failure in
+   production is observable, not silent.
+5. **Emailing** — transactional mail (verification, password reset, receipts) is
+   sent through a real transport, not a stub, where the app's flows need it.
+6. **Validation** — untrusted input is validated at the edge before it reaches
+   the ORM or filesystem.
+7. **Tests** — the core flows have automated tests that actually run and pass.
+8. **Build & config** — the app builds clean, secrets come from env (none
+   committed), and it starts with a documented command.
+
+Weigh each blocker by whether it would actually bite a real user or operator;
+skip theoretical nits. When you are unsure an item is truly done, treat it as a
+blocker and say why.
+
+End your response with **exactly one** fenced JSON block giving the verdict — the
+list of concrete, still-required work. An empty list means the app is
+production-grade.
+
+```json
+{ "blockers": ["short, actionable description of each remaining gap"] }
+```

packages/ai-autopilot/src/index.ts

@@ -50,9 +50,13 @@
  * - {@link Loop} — match an event to a prompt chain and run it (N fresh passes)
  * - {@link definePrompt} / {@link defineRule} — author prompts and policy rules
  * - {@link defaultLoopRules} — the built-in web-app policy
+ * - {@link parseVerdict} / {@link isPassing} — the `{ blockers }` verdict the loop
+ *   gates on, so it stops on a review's *outcome*, not just execution
  *
  * The prompts library supplies the loop's prompt *bodies* as data (stack-aware
- * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base.
+ * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base,
+ * and the `production-grade` checklist bootstrap's full-fledged loop repeats
+ * against.
  *
  * - {@link builtinLibrary} — load the shipped, stack-aware prompt bodies
  * - {@link loopPromptsFor} — materialize a library into loop prompts by id
@@ -140,6 +144,9 @@ export {
   defaultLoopRules,
   LOOP_EVENTS,
   LOOP_PROMPTS,
+  parseVerdict,
+  isPassing,
+  type Verdict,
   type LoopOptions,
   type LoopEvent,
   type LoopContext,

packages/ai-autopilot/src/loop/index.ts

@@ -9,6 +9,7 @@
 export { definePrompt, defineRule, LoopError } from './define.js'
 export { Loop, createLoop, type LoopOptions } from './loop.js'
 export { defaultLoopRules, LOOP_EVENTS, LOOP_PROMPTS } from './policy.js'
+export { parseVerdict, isPassing, type Verdict } from './verdict.js'
 export type {
   LoopEvent,
   LoopContext,

packages/ai-autopilot/src/loop/loop.test.ts

@@ -111,6 +111,53 @@ describe('Loop — failure policy', () => {
   })
 })
 
+describe('Loop — verdict gating', () => {
+  const rules = [defineRule({ on: 'check', run: ['production-grade', 'after'] })]
+  const after = definePrompt({ id: 'after', run: () => 'ran' })
+
+  it('parses a verdict onto the outcome and marks blockers not-passing', async () => {
+    const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no auth"] }\n```' })
+    const loop = new Loop({ rules, prompts: [gate, after] })
+    const result = await loop.handle({ kind: 'check' })
+    const outcome = result.outcomes[0]!
+    assert.equal(outcome.ok, true) // it executed
+    assert.equal(outcome.passing, false) // but it reported blockers
+    assert.deepEqual(outcome.verdict?.blockers, ['no auth'])
+  })
+
+  it('an empty blockers verdict is passing', async () => {
+    const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": [] }\n```' })
+    const loop = new Loop({ rules, prompts: [gate, after] })
+    const result = await loop.handle({ kind: 'check' })
+    assert.equal(result.outcomes[0]?.passing, true)
+  })
+
+  it('gates the chain on blockers when continueOnError is false', async () => {
+    const events: LoopProgress[] = []
+    const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no tests"] }\n```' })
+    const loop = new Loop({ rules, prompts: [gate, after], continueOnError: false, onEvent: e => events.push(e) })
+    const result = await loop.handle({ kind: 'check' })
+    assert.equal(result.outcomes.length, 1) // `after` was gated out on the verdict, not an error
+    assert.ok(events.some(e => e.type === 'gate-stop' && e.promptId === 'production-grade'))
+  })
+
+  it('a prompt with no verdict still passes when it executes (backward compatible)', async () => {
+    const gate = definePrompt({ id: 'production-grade', run: () => 'no json here' })
+    const loop = new Loop({ rules, prompts: [gate, after] })
+    const result = await loop.handle({ kind: 'check' })
+    assert.equal(result.outcomes[0]?.passing, true)
+    assert.equal(result.outcomes[0]?.verdict, undefined)
+  })
+
+  it('verdict: null disables parsing (execution-only gate)', async () => {
+    const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["x"] }\n```' })
+    const loop = new Loop({ rules, prompts: [gate, after], verdict: null })
+    const result = await loop.handle({ kind: 'check' })
+    assert.equal(result.outcomes[0]?.passing, true) // blockers ignored
+    assert.equal(result.outcomes[0]?.verdict, undefined)
+  })
+})
+
 describe('Loop — decisions + watch', () => {
   it('exposes the ledger to prompts via context', async () => {
     const ledger = new DecisionLedger()

packages/ai-autopilot/src/loop/loop.ts

@@ -8,6 +8,8 @@ import type {
   PassResult,
   PromptOutcome,
 } from './types.js'
+import type { Verdict } from './verdict.js'
+import { parseVerdict } from './verdict.js'
 
 /** Options for {@link Loop}. */
 export interface LoopOptions {
@@ -21,12 +23,20 @@ export interface LoopOptions {
    * Failure policy, the sync-vs-async knob:
    * - `true` (default) — fire-and-report: every matched prompt runs regardless
    *   of a prior one failing.
-   * - `false` — blocking gate: if a prompt's final pass fails, stop the chain
-   *   (a `gate-stop` event); the remaining prompts do not run. The v1
-   *   approximation of "block until the review passes" (gates on execution
-   *   failure; gating on a review's *verdict* waits for prompts to return one).
+   * - `false` — blocking gate: if a prompt does not *pass*, stop the chain (a
+   *   `gate-stop` event); the remaining prompts do not run. "Pass" means the
+   *   final pass executed and, when a {@link LoopOptions.verdict} parser is set,
+   *   returned no blockers.
    */
   continueOnError?: boolean
+  /**
+   * Parse a prompt's final-pass text into a {@link Verdict}. When set, the loop
+   * gates on the *outcome* a prompt reports (`{ blockers }`), not just whether it
+   * executed — a prompt that runs but returns blockers is not passing. Defaults
+   * to {@link parseVerdict} (reads a fenced ```json `{ "blockers": [...] }`).
+   * Pass `null` to disable verdict gating entirely (execution-only gate).
+   */
+  verdict?: ((text: string) => Verdict | undefined) | null
   /**
    * Observe progress. Isolated: a throwing callback is logged and swallowed, so
    * an observer bug cannot abort a run.
@@ -54,6 +64,7 @@ export class Loop {
   private readonly prompts: Map<string, LoopPrompt>
   private readonly ledger?: DecisionLedger
   private readonly continueOnError: boolean
+  private readonly parseVerdict: ((text: string) => Verdict | undefined) | null
   private readonly emit: (event: LoopProgress) => void
 
   constructor(opts: LoopOptions) {
@@ -64,6 +75,7 @@ export class Loop {
     this.prompts = indexPrompts(opts.prompts)
     if (opts.ledger !== undefined) this.ledger = opts.ledger
     this.continueOnError = opts.continueOnError ?? true
+    this.parseVerdict = opts.verdict === undefined ? parseVerdict : opts.verdict
     this.emit = makeEmitter(opts.onEvent)
   }
 
@@ -99,14 +111,14 @@ export class Loop {
       const prompt = this.prompts.get(id)
       if (!prompt) {
         this.emit({ type: 'unknown-prompt', promptId: id })
-        outcomes.push({ promptId: id, passes: [], ok: false })
+        outcomes.push({ promptId: id, passes: [], ok: false, passing: false })
         continue
       }
 
       const outcome = await this.runPrompt(prompt, event)
       outcomes.push(outcome)
 
-      if (!outcome.ok && !this.continueOnError) {
+      if (!outcome.passing && !this.continueOnError) {
         this.emit({ type: 'gate-stop', promptId: id })
         break
       }
@@ -147,9 +159,14 @@ export class Loop {
       this.emit({ type: 'pass', promptId: prompt.id, result, passes: prompt.passes })
     }
 
-    const ok = passes[passes.length - 1]?.ok ?? false
-    this.emit({ type: 'prompt-done', promptId: prompt.id, ok })
-    return { promptId: prompt.id, passes, ok }
+    const last = passes[passes.length - 1]
+    const ok = last?.ok ?? false
+    // A verdict only means something when the pass that produced it executed.
+    const verdict = ok && this.parseVerdict ? this.parseVerdict(last!.text) : undefined
+    const passing = ok && (verdict ? verdict.blockers.length === 0 : true)
+
+    this.emit({ type: 'prompt-done', promptId: prompt.id, ok, passing, ...(verdict ? { verdict } : {}) })
+    return { promptId: prompt.id, passes, ok, passing, ...(verdict ? { verdict } : {}) }
   }
 }
 

packages/ai-autopilot/src/loop/policy.ts

@@ -23,6 +23,8 @@ export const LOOP_PROMPTS = {
   security: 'security',
   qa: 'qa',
   ux: 'ux',
+  /** The checklist bootstrap's full-fledged loop repeats against; returns a `{ blockers }` verdict. */
+  productionGrade: 'production-grade',
 } as const
 
 /**

packages/ai-autopilot/src/loop/types.ts

@@ -1,4 +1,5 @@
 import type { DecisionLedger } from '../decisions/ledger.js'
+import type { Verdict } from './verdict.js'
 
 /**
  * "The loop" — the event-to-prompt-chain policy. When the agent does something
@@ -95,8 +96,19 @@ export interface PassResult {
 export interface PromptOutcome {
   promptId: string
   passes: PassResult[]
-  /** True when the final pass succeeded — the result that matters. */
+  /** True when the final pass succeeded (executed without throwing). */
   ok: boolean
+  /**
+   * The structured verdict parsed from the final pass, when the loop has a
+   * `verdict` parser and the prompt returned one. Absent otherwise.
+   */
+  verdict?: Verdict
+  /**
+   * The gating result: executed cleanly *and*, when a verdict was parsed, has no
+   * blockers. Equals {@link ok} when no verdict parser is configured, so the gate
+   * is backward-compatible. This is what `continueOnError: false` stops on.
+   */
+  passing: boolean
 }
 
 /** The full result of handling one {@link LoopEvent}. */
@@ -115,6 +127,6 @@ export type LoopProgress =
   | { type: 'unknown-prompt'; promptId: string }
   | { type: 'prompt-start'; promptId: string; passes: number }
   | { type: 'pass'; promptId: string; result: PassResult; passes: number }
-  | { type: 'prompt-done'; promptId: string; ok: boolean }
+  | { type: 'prompt-done'; promptId: string; ok: boolean; passing: boolean; verdict?: Verdict }
   | { type: 'gate-stop'; promptId: string }
   | { type: 'done'; event: LoopEvent; outcomes: PromptOutcome[] }

packages/ai-autopilot/src/loop/verdict.test.ts

@@ -0,0 +1,43 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { parseVerdict, isPassing } from './verdict.js'
+
+describe('parseVerdict', () => {
+  it('reads a fenced json block with an empty blockers list (passing)', () => {
+    const text = 'Everything checks out.\n\n```json\n{ "blockers": [] }\n```'
+    const v = parseVerdict(text)
+    assert.deepEqual(v, { blockers: [] })
+    assert.equal(isPassing(v), true)
+  })
+
+  it('reads blockers and trims/drops empties', () => {
+    const text = '```json\n{ "blockers": ["no auth", "  ", "no tests"] }\n```'
+    const v = parseVerdict(text)
+    assert.deepEqual(v?.blockers, ['no auth', 'no tests'])
+    assert.equal(isPassing(v), false)
+  })
+
+  it('keeps an optional notes field', () => {
+    const v = parseVerdict('```json\n{ "blockers": [], "notes": "ship it" }\n```')
+    assert.equal(v?.notes, 'ship it')
+  })
+
+  it('takes the last verdict when several appear (a corrected pass wins)', () => {
+    const text =
+      'draft:\n```json\n{ "blockers": ["x"] }\n```\nfinal:\n```json\n{ "blockers": [] }\n```'
+    assert.deepEqual(parseVerdict(text)?.blockers, [])
+  })
+
+  it('falls back to a trailing bare object', () => {
+    const v = parseVerdict('Verdict: { "blockers": ["needs migrations"] }')
+    assert.deepEqual(v?.blockers, ['needs migrations'])
+  })
+
+  it('returns undefined when there is no verdict (distinct from failing)', () => {
+    assert.equal(parseVerdict('just prose, no json'), undefined)
+    assert.equal(parseVerdict(''), undefined)
+    // an object without a blockers array is not a verdict
+    assert.equal(parseVerdict('```json\n{ "ok": true }\n```'), undefined)
+    assert.equal(isPassing(undefined), false)
+  })
+})

packages/ai-autopilot/src/loop/verdict.ts

@@ -0,0 +1,72 @@
+/**
+ * The verdict convention — a light, structured outcome a prompt can return so
+ * the loop gates on *what a review concluded*, not just whether the prompt ran.
+ *
+ * A verdict is `{ blockers: string[] }`: an empty list means "nothing left to
+ * fix" (passing); a non-empty list is the concrete work still required. The
+ * production-grade checklist prompt returns one, and bootstrap's full-fledged
+ * loop repeats against it until `blockers` is empty. This answers the note left
+ * in the loop (#113): the v1 gate stopped on execution failure; a verdict lets it
+ * stop on the review's outcome.
+ *
+ * The transport is text (prompts return text), so the convention is: end the
+ * output with a fenced ```json block holding `{ "blockers": [...] }`.
+ * {@link parseVerdict} reads it back; {@link isPassing} is the empty-blockers test.
+ */
+
+/** A structured prompt outcome the loop can gate on. */
+export interface Verdict {
+  /** Concrete work still required; empty means passing. */
+  readonly blockers: string[]
+  /** Optional free-text summary the agent may include. */
+  readonly notes?: string
+}
+
+/** True when a verdict has no blockers left. A missing verdict is not passing. */
+export function isPassing(verdict: Verdict | undefined): boolean {
+  return verdict !== undefined && verdict.blockers.length === 0
+}
+
+// Matches fenced code blocks, capturing an optional language tag and the body.
+const FENCE = /```(?:[a-zA-Z0-9]*)\n([\s\S]*?)```/g
+
+/**
+ * Parse a {@link Verdict} out of a prompt's text output. Reads the **last**
+ * fenced code block that is a JSON object with a `blockers` array (last so a
+ * later, corrected pass wins over an earlier draft in the same text), falling
+ * back to a trailing bare `{ ... }` object. Returns `undefined` when no verdict
+ * is present so the caller can tell "not passing" from "did not report".
+ */
+export function parseVerdict(text: string): Verdict | undefined {
+  if (!text) return undefined
+
+  const candidates: string[] = []
+  for (const m of text.matchAll(FENCE)) candidates.push(m[1]!)
+  // Fallback: a bare object somewhere in the text (take the last `{`-to-`}` run).
+  const lastOpen = text.lastIndexOf('{')
+  const lastClose = text.lastIndexOf('}')
+  if (lastOpen !== -1 && lastClose > lastOpen) candidates.push(text.slice(lastOpen, lastClose + 1))
+
+  // Later candidates win, so scan from the end.
+  for (let i = candidates.length - 1; i >= 0; i--) {
+    const verdict = coerce(candidates[i]!)
+    if (verdict) return verdict
+  }
+  return undefined
+}
+
+function coerce(raw: string): Verdict | undefined {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw.trim())
+  } catch {
+    return undefined
+  }
+  if (typeof parsed !== 'object' || parsed === null) return undefined
+  const obj = parsed as Record<string, unknown>
+  if (!Array.isArray(obj.blockers)) return undefined
+
+  const blockers = obj.blockers.map(b => String(b).trim()).filter(Boolean)
+  const notes = typeof obj.notes === 'string' && obj.notes.trim() ? obj.notes.trim() : undefined
+  return Object.freeze({ blockers, ...(notes ? { notes } : {}) })
+}