From b662592b2ab9f9c52e925ec0e030321146c0f279 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 20:04:25 +0300 Subject: [PATCH 1/3] feat(ai-autopilot): generalize surface stream to be event-type generic (#120) --- .changeset/ai-autopilot-generic-surface.md | 5 ++++ .../ai-autopilot/src/surface/events.test.ts | 22 ++++++++++++++ packages/ai-autopilot/src/surface/events.ts | 30 +++++++++++-------- .../ai-autopilot/src/surface/launch.test.ts | 30 +++++++++++++++++++ packages/ai-autopilot/src/surface/launch.ts | 20 ++++++++----- 5 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 .changeset/ai-autopilot-generic-surface.md diff --git a/.changeset/ai-autopilot-generic-surface.md b/.changeset/ai-autopilot-generic-surface.md new file mode 100644 index 0000000..f0138e0 --- /dev/null +++ b/.changeset/ai-autopilot-generic-surface.md @@ -0,0 +1,5 @@ +--- +"@gemstack/ai-autopilot": minor +--- + +Generalize the surface stream to be event-type generic. `EventStream`, `AutopilotHandle`, and `launchAutopilot` now take the event (and result) type as parameters, defaulting to the supervisor's `SupervisorEvent` / `SupervisorRun` so every existing supervisor surface is unchanged. This lets bootstrap (and any future surface) stream its own narration events and return its own result over the same replayable, detached transport. Closes #120. diff --git a/packages/ai-autopilot/src/surface/events.test.ts b/packages/ai-autopilot/src/surface/events.test.ts index 30399ea..b45afa1 100644 --- a/packages/ai-autopilot/src/surface/events.test.ts +++ b/packages/ai-autopilot/src/surface/events.test.ts @@ -83,6 +83,28 @@ describe('formatEvent', () => { } }) +describe('EventStream generic element type', () => { + interface BootEvent { + type: 'narrate' | 'decision' + message: string + } + + it('carries a custom event type through push, history, and iteration', async () => { + const s = new EventStream() + s.push({ type: 'narrate', message: 'picking the stack' }) + s.push({ type: 'decision', message: 'Vike + universal-orm' }) + s.close() + + assert.deepEqual( + s.history().map(e => e.message), + ['picking the stack', 'Vike + universal-orm'], + ) + const seen: BootEvent[] = [] + for await (const e of s) seen.push(e) + assert.deepEqual(seen.map(e => e.type), ['narrate', 'decision']) + }) +}) + describe('terminalSink', () => { it('writes a formatted line per event to the provided writer', () => { const lines: string[] = [] diff --git a/packages/ai-autopilot/src/surface/events.ts b/packages/ai-autopilot/src/surface/events.ts index 0bd86a9..ac0188a 100644 --- a/packages/ai-autopilot/src/surface/events.ts +++ b/packages/ai-autopilot/src/surface/events.ts @@ -1,31 +1,35 @@ import type { SupervisorEvent } from '../types.js' /** - * A replayable, multi-consumer stream of {@link SupervisorEvent}s. It is the - * transport shared by the in-page and background surfaces: buffer every event, - * hand out live async iterators, and replay history from an offset (borrowing - * Flue's Durable-Streams `tail=N`). The terminal surface uses {@link terminalSink} - * directly and does not need this. + * A replayable, multi-consumer stream of events. It is the transport shared by + * the in-page and background surfaces: buffer every event, hand out live async + * iterators, and replay history from an offset (borrowing Flue's Durable-Streams + * `tail=N`). The terminal surface uses {@link terminalSink} directly and does not + * need this. + * + * The element type `E` defaults to {@link SupervisorEvent} so existing supervisor + * surfaces are unchanged; bootstrap and any future surface pass their own event + * type (`new EventStream()`). */ -export class EventStream { - private readonly buffer: SupervisorEvent[] = [] +export class EventStream { + private readonly buffer: E[] = [] private readonly waiters: Array<() => void> = [] private closed = false - /** Append an event. Wire this in as a Supervisor `onEvent`. Ignored once closed. */ - readonly push = (event: SupervisorEvent): void => { + /** Append an event. Wire this in as an `onEvent` sink. Ignored once closed. */ + readonly push = (event: E): void => { if (this.closed) return this.buffer.push(event) for (const wake of this.waiters.splice(0)) wake() } /** Alias for {@link push}, reads well at the `onEvent:` call site. */ - get sink(): (event: SupervisorEvent) => void { + get sink(): (event: E) => void { return this.push } /** Events buffered so far, from `fromOffset` (default 0) — Flue-style tail replay. */ - history(fromOffset = 0): SupervisorEvent[] { + history(fromOffset = 0): E[] { return this.buffer.slice(fromOffset) } @@ -52,14 +56,14 @@ export class EventStream { * Independent iterators each keep their own cursor, so late consumers still * see the full history. */ - [Symbol.asyncIterator](): AsyncIterableIterator { + [Symbol.asyncIterator](): AsyncIterableIterator { let index = 0 const stream = this return { [Symbol.asyncIterator]() { return this }, - next(): Promise> { + next(): Promise> { if (index < stream.buffer.length) { return Promise.resolve({ value: stream.buffer[index++]!, done: false }) } diff --git a/packages/ai-autopilot/src/surface/launch.test.ts b/packages/ai-autopilot/src/surface/launch.test.ts index e9825cb..f456970 100644 --- a/packages/ai-autopilot/src/surface/launch.test.ts +++ b/packages/ai-autopilot/src/surface/launch.test.ts @@ -69,6 +69,36 @@ describe('launchAutopilot', () => { assert.deepEqual(seen, []) }) + it('carries a custom event and result type (bootstrap-shaped surface)', async () => { + interface BootEvent { + type: string + message: string + } + interface BootResult { + app: string + blockers: string[] + } + + let emit!: (e: BootEvent) => void + let finish!: (r: BootResult) => void + const handle = launchAutopilot(onEvent => { + emit = onEvent + return new Promise(resolve => { + finish = resolve + }) + }) + + emit({ type: 'narrate', message: 'scaffolding' }) + assert.deepEqual( + handle.events().map(e => e.message), + ['scaffolding'], + ) + finish({ app: 'shop', blockers: [] }) + const result = await handle.result() + assert.equal(result.app, 'shop') + assert.deepEqual(result.blockers, []) + }) + it('honors an explicit id and generates unique ids otherwise', async () => { const a = launchAutopilot(async () => fakeRun(), { id: 'my-run' }) const b = launchAutopilot(async () => fakeRun()) diff --git a/packages/ai-autopilot/src/surface/launch.ts b/packages/ai-autopilot/src/surface/launch.ts index 43cabf0..d27e330 100644 --- a/packages/ai-autopilot/src/surface/launch.ts +++ b/packages/ai-autopilot/src/surface/launch.ts @@ -11,18 +11,22 @@ export type AutopilotStatus = 'running' | 'done' | 'error' * {@link result}. The same handle backs an in-page surface (iterate `stream()` * and push each event over SSE) and a background process (persist the handle, * let a client poll `status()` + `events(offset)`). + * + * The type params default to the supervisor's `E`/`R` so supervisor surfaces are + * unchanged; bootstrap emits its own narration events and returns its own result, + * so it launches with `AutopilotHandle`. */ -export interface AutopilotHandle { +export interface AutopilotHandle { /** Stable id for this run. */ readonly id: string /** Current lifecycle state. */ status(): AutopilotStatus /** Events emitted so far, from `fromOffset` (default 0). */ - events(fromOffset?: number): SupervisorEvent[] + events(fromOffset?: number): E[] /** A fresh async iterator over events: replays history, then live, then ends. */ - stream(): AsyncIterableIterator + stream(): AsyncIterableIterator /** Resolves with the run's result, or rejects if the run threw. */ - result(): Promise + result(): Promise } /** Options for {@link launchAutopilot}. */ @@ -40,11 +44,11 @@ let counter = 0 * Keeping `start` caller-provided means this surface knows nothing about how the * Supervisor is built; it only owns the event stream and lifecycle. */ -export function launchAutopilot( - start: (onEvent: (event: SupervisorEvent) => void) => Promise, +export function launchAutopilot( + start: (onEvent: (event: E) => void) => Promise, opts: LaunchOptions = {}, -): AutopilotHandle { - const stream = new EventStream() +): AutopilotHandle { + const stream = new EventStream() const id = opts.id ?? `autopilot-${++counter}` let state: AutopilotStatus = 'running' From 62faf16506b71d8ed734e0bd4291bc180762f5d8 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 20:09:57 +0300 Subject: [PATCH 2/3] feat(ai-autopilot): production-grade checklist prompt + { blockers } verdict convention (#121) --- ...ai-autopilot-production-grade-checklist.md | 5 ++ .../ai-autopilot/prompts/production-grade.md | 45 ++++++++++++ packages/ai-autopilot/src/index.ts | 9 ++- packages/ai-autopilot/src/loop/index.ts | 1 + packages/ai-autopilot/src/loop/loop.test.ts | 47 ++++++++++++ packages/ai-autopilot/src/loop/loop.ts | 35 ++++++--- packages/ai-autopilot/src/loop/policy.ts | 2 + packages/ai-autopilot/src/loop/types.ts | 16 ++++- .../ai-autopilot/src/loop/verdict.test.ts | 43 +++++++++++ packages/ai-autopilot/src/loop/verdict.ts | 72 +++++++++++++++++++ .../ai-autopilot/src/prompts/library.test.ts | 7 ++ 11 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 .changeset/ai-autopilot-production-grade-checklist.md create mode 100644 packages/ai-autopilot/prompts/production-grade.md create mode 100644 packages/ai-autopilot/src/loop/verdict.test.ts create mode 100644 packages/ai-autopilot/src/loop/verdict.ts diff --git a/.changeset/ai-autopilot-production-grade-checklist.md b/.changeset/ai-autopilot-production-grade-checklist.md new file mode 100644 index 0000000..b5b55ba --- /dev/null +++ b/.changeset/ai-autopilot-production-grade-checklist.md @@ -0,0 +1,5 @@ +--- +"@gemstack/ai-autopilot": minor +--- + +Add the `production-grade` checklist prompt and a `{ blockers }` verdict convention. The new built-in prompt judges an app against a production-grade checklist (auth, data layer, error handling, instrumentation, emailing, validation, tests, build/config) and ends with a machine-readable `{ "blockers": [...] }` verdict — an empty list means production-grade. `parseVerdict()` / `isPassing()` read it back, and the loop now gates on that outcome: a `PromptOutcome` carries `verdict` and a `passing` flag (executed *and* no blockers), and `continueOnError: false` stops the chain on `!passing`. Backward compatible — with no verdict reported, `passing === ok`; pass `verdict: null` for an execution-only gate. This is what bootstrap's full-fledged loop repeats against until blockers is empty. Closes #121. diff --git a/packages/ai-autopilot/prompts/production-grade.md b/packages/ai-autopilot/prompts/production-grade.md new file mode 100644 index 0000000..46cf648 --- /dev/null +++ b/packages/ai-autopilot/prompts/production-grade.md @@ -0,0 +1,45 @@ +--- +name: production-grade +description: Checklist gate — is this app production-grade? Returns a { blockers } verdict. +appliesTo: ["**/*"] +metadata: + title: Production-grade checklist + loopId: production-grade + passes: 1 +--- + +You are the production-grade **gate** for a Vike + universal-orm app. Judge the +app *as it stands now* against the checklist below. You are not reviewing a diff; +you are deciding whether this is something you would put in front of real users. + +Go through the app and check each area. For every item, either it is genuinely +handled or it is a **blocker** — do not give credit for a stub, a TODO, or a +console.log standing in for the real thing. + +1. **Auth** — real sign-up / sign-in / sign-out, sessions scoped per user, routes + and data access gated. No hardcoded or bypassable auth. +2. **Data layer** — schema + migrations exist and run; every query is scoped to + its owner (no IDOR); writes return the data callers rely on. +3. **Error handling** — no unhandled rejections or naked `throw` reaching the + user; failures surface as proper responses, not a white screen or a 500 stack. +4. **Instrumentation** — logging and error reporting are wired so a failure in + production is observable, not silent. +5. **Emailing** — transactional mail (verification, password reset, receipts) is + sent through a real transport, not a stub, where the app's flows need it. +6. **Validation** — untrusted input is validated at the edge before it reaches + the ORM or filesystem. +7. **Tests** — the core flows have automated tests that actually run and pass. +8. **Build & config** — the app builds clean, secrets come from env (none + committed), and it starts with a documented command. + +Weigh each blocker by whether it would actually bite a real user or operator; +skip theoretical nits. When you are unsure an item is truly done, treat it as a +blocker and say why. + +End your response with **exactly one** fenced JSON block giving the verdict — the +list of concrete, still-required work. An empty list means the app is +production-grade. + +```json +{ "blockers": ["short, actionable description of each remaining gap"] } +``` diff --git a/packages/ai-autopilot/src/index.ts b/packages/ai-autopilot/src/index.ts index e230baa..4d84997 100644 --- a/packages/ai-autopilot/src/index.ts +++ b/packages/ai-autopilot/src/index.ts @@ -50,9 +50,13 @@ * - {@link Loop} — match an event to a prompt chain and run it (N fresh passes) * - {@link definePrompt} / {@link defineRule} — author prompts and policy rules * - {@link defaultLoopRules} — the built-in web-app policy + * - {@link parseVerdict} / {@link isPassing} — the `{ blockers }` verdict the loop + * gates on, so it stops on a review's *outcome*, not just execution * * The prompts library supplies the loop's prompt *bodies* as data (stack-aware - * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base. + * markdown): review, code-quality, security, refactor, UX, QA, knowledge-base, + * and the `production-grade` checklist bootstrap's full-fledged loop repeats + * against. * * - {@link builtinLibrary} — load the shipped, stack-aware prompt bodies * - {@link loopPromptsFor} — materialize a library into loop prompts by id @@ -140,6 +144,9 @@ export { defaultLoopRules, LOOP_EVENTS, LOOP_PROMPTS, + parseVerdict, + isPassing, + type Verdict, type LoopOptions, type LoopEvent, type LoopContext, diff --git a/packages/ai-autopilot/src/loop/index.ts b/packages/ai-autopilot/src/loop/index.ts index bd78bab..846d4c5 100644 --- a/packages/ai-autopilot/src/loop/index.ts +++ b/packages/ai-autopilot/src/loop/index.ts @@ -9,6 +9,7 @@ export { definePrompt, defineRule, LoopError } from './define.js' export { Loop, createLoop, type LoopOptions } from './loop.js' export { defaultLoopRules, LOOP_EVENTS, LOOP_PROMPTS } from './policy.js' +export { parseVerdict, isPassing, type Verdict } from './verdict.js' export type { LoopEvent, LoopContext, diff --git a/packages/ai-autopilot/src/loop/loop.test.ts b/packages/ai-autopilot/src/loop/loop.test.ts index b7664eb..2be42b6 100644 --- a/packages/ai-autopilot/src/loop/loop.test.ts +++ b/packages/ai-autopilot/src/loop/loop.test.ts @@ -111,6 +111,53 @@ describe('Loop — failure policy', () => { }) }) +describe('Loop — verdict gating', () => { + const rules = [defineRule({ on: 'check', run: ['production-grade', 'after'] })] + const after = definePrompt({ id: 'after', run: () => 'ran' }) + + it('parses a verdict onto the outcome and marks blockers not-passing', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no auth"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + const outcome = result.outcomes[0]! + assert.equal(outcome.ok, true) // it executed + assert.equal(outcome.passing, false) // but it reported blockers + assert.deepEqual(outcome.verdict?.blockers, ['no auth']) + }) + + it('an empty blockers verdict is passing', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": [] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) + }) + + it('gates the chain on blockers when continueOnError is false', async () => { + const events: LoopProgress[] = [] + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no tests"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after], continueOnError: false, onEvent: e => events.push(e) }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes.length, 1) // `after` was gated out on the verdict, not an error + assert.ok(events.some(e => e.type === 'gate-stop' && e.promptId === 'production-grade')) + }) + + it('a prompt with no verdict still passes when it executes (backward compatible)', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => 'no json here' }) + const loop = new Loop({ rules, prompts: [gate, after] }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) + assert.equal(result.outcomes[0]?.verdict, undefined) + }) + + it('verdict: null disables parsing (execution-only gate)', async () => { + const gate = definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["x"] }\n```' }) + const loop = new Loop({ rules, prompts: [gate, after], verdict: null }) + const result = await loop.handle({ kind: 'check' }) + assert.equal(result.outcomes[0]?.passing, true) // blockers ignored + assert.equal(result.outcomes[0]?.verdict, undefined) + }) +}) + describe('Loop — decisions + watch', () => { it('exposes the ledger to prompts via context', async () => { const ledger = new DecisionLedger() diff --git a/packages/ai-autopilot/src/loop/loop.ts b/packages/ai-autopilot/src/loop/loop.ts index c11c62a..beb380d 100644 --- a/packages/ai-autopilot/src/loop/loop.ts +++ b/packages/ai-autopilot/src/loop/loop.ts @@ -8,6 +8,8 @@ import type { PassResult, PromptOutcome, } from './types.js' +import type { Verdict } from './verdict.js' +import { parseVerdict } from './verdict.js' /** Options for {@link Loop}. */ export interface LoopOptions { @@ -21,12 +23,20 @@ export interface LoopOptions { * Failure policy, the sync-vs-async knob: * - `true` (default) — fire-and-report: every matched prompt runs regardless * of a prior one failing. - * - `false` — blocking gate: if a prompt's final pass fails, stop the chain - * (a `gate-stop` event); the remaining prompts do not run. The v1 - * approximation of "block until the review passes" (gates on execution - * failure; gating on a review's *verdict* waits for prompts to return one). + * - `false` — blocking gate: if a prompt does not *pass*, stop the chain (a + * `gate-stop` event); the remaining prompts do not run. "Pass" means the + * final pass executed and, when a {@link LoopOptions.verdict} parser is set, + * returned no blockers. */ continueOnError?: boolean + /** + * Parse a prompt's final-pass text into a {@link Verdict}. When set, the loop + * gates on the *outcome* a prompt reports (`{ blockers }`), not just whether it + * executed — a prompt that runs but returns blockers is not passing. Defaults + * to {@link parseVerdict} (reads a fenced ```json `{ "blockers": [...] }`). + * Pass `null` to disable verdict gating entirely (execution-only gate). + */ + verdict?: ((text: string) => Verdict | undefined) | null /** * Observe progress. Isolated: a throwing callback is logged and swallowed, so * an observer bug cannot abort a run. @@ -54,6 +64,7 @@ export class Loop { private readonly prompts: Map private readonly ledger?: DecisionLedger private readonly continueOnError: boolean + private readonly parseVerdict: ((text: string) => Verdict | undefined) | null private readonly emit: (event: LoopProgress) => void constructor(opts: LoopOptions) { @@ -64,6 +75,7 @@ export class Loop { this.prompts = indexPrompts(opts.prompts) if (opts.ledger !== undefined) this.ledger = opts.ledger this.continueOnError = opts.continueOnError ?? true + this.parseVerdict = opts.verdict === undefined ? parseVerdict : opts.verdict this.emit = makeEmitter(opts.onEvent) } @@ -99,14 +111,14 @@ export class Loop { const prompt = this.prompts.get(id) if (!prompt) { this.emit({ type: 'unknown-prompt', promptId: id }) - outcomes.push({ promptId: id, passes: [], ok: false }) + outcomes.push({ promptId: id, passes: [], ok: false, passing: false }) continue } const outcome = await this.runPrompt(prompt, event) outcomes.push(outcome) - if (!outcome.ok && !this.continueOnError) { + if (!outcome.passing && !this.continueOnError) { this.emit({ type: 'gate-stop', promptId: id }) break } @@ -147,9 +159,14 @@ export class Loop { this.emit({ type: 'pass', promptId: prompt.id, result, passes: prompt.passes }) } - const ok = passes[passes.length - 1]?.ok ?? false - this.emit({ type: 'prompt-done', promptId: prompt.id, ok }) - return { promptId: prompt.id, passes, ok } + const last = passes[passes.length - 1] + const ok = last?.ok ?? false + // A verdict only means something when the pass that produced it executed. + const verdict = ok && this.parseVerdict ? this.parseVerdict(last!.text) : undefined + const passing = ok && (verdict ? verdict.blockers.length === 0 : true) + + this.emit({ type: 'prompt-done', promptId: prompt.id, ok, passing, ...(verdict ? { verdict } : {}) }) + return { promptId: prompt.id, passes, ok, passing, ...(verdict ? { verdict } : {}) } } } diff --git a/packages/ai-autopilot/src/loop/policy.ts b/packages/ai-autopilot/src/loop/policy.ts index 1f42a06..201e404 100644 --- a/packages/ai-autopilot/src/loop/policy.ts +++ b/packages/ai-autopilot/src/loop/policy.ts @@ -23,6 +23,8 @@ export const LOOP_PROMPTS = { security: 'security', qa: 'qa', ux: 'ux', + /** The checklist bootstrap's full-fledged loop repeats against; returns a `{ blockers }` verdict. */ + productionGrade: 'production-grade', } as const /** diff --git a/packages/ai-autopilot/src/loop/types.ts b/packages/ai-autopilot/src/loop/types.ts index 8069cf7..7319e39 100644 --- a/packages/ai-autopilot/src/loop/types.ts +++ b/packages/ai-autopilot/src/loop/types.ts @@ -1,4 +1,5 @@ import type { DecisionLedger } from '../decisions/ledger.js' +import type { Verdict } from './verdict.js' /** * "The loop" — the event-to-prompt-chain policy. When the agent does something @@ -95,8 +96,19 @@ export interface PassResult { export interface PromptOutcome { promptId: string passes: PassResult[] - /** True when the final pass succeeded — the result that matters. */ + /** True when the final pass succeeded (executed without throwing). */ ok: boolean + /** + * The structured verdict parsed from the final pass, when the loop has a + * `verdict` parser and the prompt returned one. Absent otherwise. + */ + verdict?: Verdict + /** + * The gating result: executed cleanly *and*, when a verdict was parsed, has no + * blockers. Equals {@link ok} when no verdict parser is configured, so the gate + * is backward-compatible. This is what `continueOnError: false` stops on. + */ + passing: boolean } /** The full result of handling one {@link LoopEvent}. */ @@ -115,6 +127,6 @@ export type LoopProgress = | { type: 'unknown-prompt'; promptId: string } | { type: 'prompt-start'; promptId: string; passes: number } | { type: 'pass'; promptId: string; result: PassResult; passes: number } - | { type: 'prompt-done'; promptId: string; ok: boolean } + | { type: 'prompt-done'; promptId: string; ok: boolean; passing: boolean; verdict?: Verdict } | { type: 'gate-stop'; promptId: string } | { type: 'done'; event: LoopEvent; outcomes: PromptOutcome[] } diff --git a/packages/ai-autopilot/src/loop/verdict.test.ts b/packages/ai-autopilot/src/loop/verdict.test.ts new file mode 100644 index 0000000..7d6d402 --- /dev/null +++ b/packages/ai-autopilot/src/loop/verdict.test.ts @@ -0,0 +1,43 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { parseVerdict, isPassing } from './verdict.js' + +describe('parseVerdict', () => { + it('reads a fenced json block with an empty blockers list (passing)', () => { + const text = 'Everything checks out.\n\n```json\n{ "blockers": [] }\n```' + const v = parseVerdict(text) + assert.deepEqual(v, { blockers: [] }) + assert.equal(isPassing(v), true) + }) + + it('reads blockers and trims/drops empties', () => { + const text = '```json\n{ "blockers": ["no auth", " ", "no tests"] }\n```' + const v = parseVerdict(text) + assert.deepEqual(v?.blockers, ['no auth', 'no tests']) + assert.equal(isPassing(v), false) + }) + + it('keeps an optional notes field', () => { + const v = parseVerdict('```json\n{ "blockers": [], "notes": "ship it" }\n```') + assert.equal(v?.notes, 'ship it') + }) + + it('takes the last verdict when several appear (a corrected pass wins)', () => { + const text = + 'draft:\n```json\n{ "blockers": ["x"] }\n```\nfinal:\n```json\n{ "blockers": [] }\n```' + assert.deepEqual(parseVerdict(text)?.blockers, []) + }) + + it('falls back to a trailing bare object', () => { + const v = parseVerdict('Verdict: { "blockers": ["needs migrations"] }') + assert.deepEqual(v?.blockers, ['needs migrations']) + }) + + it('returns undefined when there is no verdict (distinct from failing)', () => { + assert.equal(parseVerdict('just prose, no json'), undefined) + assert.equal(parseVerdict(''), undefined) + // an object without a blockers array is not a verdict + assert.equal(parseVerdict('```json\n{ "ok": true }\n```'), undefined) + assert.equal(isPassing(undefined), false) + }) +}) diff --git a/packages/ai-autopilot/src/loop/verdict.ts b/packages/ai-autopilot/src/loop/verdict.ts new file mode 100644 index 0000000..4802370 --- /dev/null +++ b/packages/ai-autopilot/src/loop/verdict.ts @@ -0,0 +1,72 @@ +/** + * The verdict convention — a light, structured outcome a prompt can return so + * the loop gates on *what a review concluded*, not just whether the prompt ran. + * + * A verdict is `{ blockers: string[] }`: an empty list means "nothing left to + * fix" (passing); a non-empty list is the concrete work still required. The + * production-grade checklist prompt returns one, and bootstrap's full-fledged + * loop repeats against it until `blockers` is empty. This answers the note left + * in the loop (#113): the v1 gate stopped on execution failure; a verdict lets it + * stop on the review's outcome. + * + * The transport is text (prompts return text), so the convention is: end the + * output with a fenced ```json block holding `{ "blockers": [...] }`. + * {@link parseVerdict} reads it back; {@link isPassing} is the empty-blockers test. + */ + +/** A structured prompt outcome the loop can gate on. */ +export interface Verdict { + /** Concrete work still required; empty means passing. */ + readonly blockers: string[] + /** Optional free-text summary the agent may include. */ + readonly notes?: string +} + +/** True when a verdict has no blockers left. A missing verdict is not passing. */ +export function isPassing(verdict: Verdict | undefined): boolean { + return verdict !== undefined && verdict.blockers.length === 0 +} + +// Matches fenced code blocks, capturing an optional language tag and the body. +const FENCE = /```(?:[a-zA-Z0-9]*)\n([\s\S]*?)```/g + +/** + * Parse a {@link Verdict} out of a prompt's text output. Reads the **last** + * fenced code block that is a JSON object with a `blockers` array (last so a + * later, corrected pass wins over an earlier draft in the same text), falling + * back to a trailing bare `{ ... }` object. Returns `undefined` when no verdict + * is present so the caller can tell "not passing" from "did not report". + */ +export function parseVerdict(text: string): Verdict | undefined { + if (!text) return undefined + + const candidates: string[] = [] + for (const m of text.matchAll(FENCE)) candidates.push(m[1]!) + // Fallback: a bare object somewhere in the text (take the last `{`-to-`}` run). + const lastOpen = text.lastIndexOf('{') + const lastClose = text.lastIndexOf('}') + if (lastOpen !== -1 && lastClose > lastOpen) candidates.push(text.slice(lastOpen, lastClose + 1)) + + // Later candidates win, so scan from the end. + for (let i = candidates.length - 1; i >= 0; i--) { + const verdict = coerce(candidates[i]!) + if (verdict) return verdict + } + return undefined +} + +function coerce(raw: string): Verdict | undefined { + let parsed: unknown + try { + parsed = JSON.parse(raw.trim()) + } catch { + return undefined + } + if (typeof parsed !== 'object' || parsed === null) return undefined + const obj = parsed as Record + if (!Array.isArray(obj.blockers)) return undefined + + const blockers = obj.blockers.map(b => String(b).trim()).filter(Boolean) + const notes = typeof obj.notes === 'string' && obj.notes.trim() ? obj.notes.trim() : undefined + return Object.freeze({ blockers, ...(notes ? { notes } : {}) }) +} diff --git a/packages/ai-autopilot/src/prompts/library.test.ts b/packages/ai-autopilot/src/prompts/library.test.ts index 6cf073a..c6850eb 100644 --- a/packages/ai-autopilot/src/prompts/library.test.ts +++ b/packages/ai-autopilot/src/prompts/library.test.ts @@ -32,6 +32,13 @@ describe('builtin prompts', () => { assert.ok(ids.includes(id), `ships "${id}"`) } }) + + it('ships the production-grade checklist and instructs the { blockers } verdict', async () => { + const gate = (await builtinLibrary()).get(LOOP_PROMPTS.productionGrade) + assert.ok(gate, 'production-grade prompt exists') + assert.match(gate!.instructions, /blockers/) + assert.match(gate!.instructions, /```json/) + }) }) describe('PromptLibrary', () => { From 91169281d692bf32599eb2d4c17a6204f2ce7bad Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 20:20:30 +0300 Subject: [PATCH 3/3] =?UTF-8?q?feat(ai-autopilot):=20bootstrap=20orchestra?= =?UTF-8?q?tor=20core=20=E2=80=94=20scope=20->=20architect=20->=20build=20?= =?UTF-8?q?->=20full-fledged=20loop=20(#122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ai-autopilot-bootstrap-orchestrator.md | 5 + .../src/bootstrap/bootstrap.test.ts | 173 ++++++++++++++++ .../ai-autopilot/src/bootstrap/bootstrap.ts | 168 +++++++++++++++ packages/ai-autopilot/src/bootstrap/index.ts | 40 ++++ .../ai-autopilot/src/bootstrap/steps.test.ts | 192 ++++++++++++++++++ packages/ai-autopilot/src/bootstrap/steps.ts | 146 +++++++++++++ packages/ai-autopilot/src/bootstrap/types.ts | 146 +++++++++++++ packages/ai-autopilot/src/index.ts | 35 ++++ 8 files changed, 905 insertions(+) create mode 100644 .changeset/ai-autopilot-bootstrap-orchestrator.md create mode 100644 packages/ai-autopilot/src/bootstrap/bootstrap.test.ts create mode 100644 packages/ai-autopilot/src/bootstrap/bootstrap.ts create mode 100644 packages/ai-autopilot/src/bootstrap/index.ts create mode 100644 packages/ai-autopilot/src/bootstrap/steps.test.ts create mode 100644 packages/ai-autopilot/src/bootstrap/steps.ts create mode 100644 packages/ai-autopilot/src/bootstrap/types.ts diff --git a/.changeset/ai-autopilot-bootstrap-orchestrator.md b/.changeset/ai-autopilot-bootstrap-orchestrator.md new file mode 100644 index 0000000..63fb9c6 --- /dev/null +++ b/.changeset/ai-autopilot-bootstrap-orchestrator.md @@ -0,0 +1,5 @@ +--- +"@gemstack/ai-autopilot": minor +--- + +Add bootstrap mode's orchestrator core: the spine that sequences autopilot's primitives into scope → architect → build → full-fledged loop, taking a user from nothing to a running, production-grade app. `Bootstrap` owns the control flow (the loop, the gate, the interrupt) over four injectable steps, narrating each phase over the generic surface stream and recording the architect's choices to the decisions ledger — no permission asked. The full-fledged loop repeats the production-grade checklist with fresh context, improving against its `{ blockers }` verdict until it is empty or a `maxPasses` budget stops it; prototype scope skips it. Default step builders wire the steps onto the real primitives — `agentArchitect` (an `ai-sdk` agent + the decisions briefing), `supervisorBuild` (the `Supervisor` over personas + runner), and `loopChecklist` / `loopImprove` (the Loop) — so the same orchestrator runs against real agents in production or stubs + `FakeRunner` in a test. Verified end-to-end offline. Closes #122. diff --git a/packages/ai-autopilot/src/bootstrap/bootstrap.test.ts b/packages/ai-autopilot/src/bootstrap/bootstrap.test.ts new file mode 100644 index 0000000..5b6a66d --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/bootstrap.test.ts @@ -0,0 +1,173 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { Bootstrap, createBootstrap, BootstrapAborted } from './bootstrap.js' +import { DecisionLedger } from '../decisions/ledger.js' +import type { BootstrapEvent, BootstrapSteps, ScopeAnswer } from './types.js' +import type { SupervisorRun } from '../types.js' +import type { Verdict } from '../loop/verdict.js' + +const zeroUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 } +const fakeRun: SupervisorRun = { text: 'built', plan: [], results: [], usage: zeroUsage, stoppedEarly: false } + +/** Build a set of stub steps; override any of them per test. */ +function stubSteps(over: Partial = {}): BootstrapSteps { + return { + scope: () => ({ scope: 'full', intent: 'a shop' }) satisfies ScopeAnswer, + architect: () => ({ + stack: 'Vike + universal-orm', + narration: 'Server-rendered shop with a Postgres data layer', + decisions: [{ choice: 'Use Vike for SSR', why: 'SEO + fast first paint' }], + }), + build: () => fakeRun, + checklist: () => ({ blockers: [] }) satisfies Verdict, + ...over, + } +} + +describe('Bootstrap — happy path (full scope, passes first checklist)', () => { + it('sequences scope → architect → build → loop and returns a production-grade result', async () => { + const events: BootstrapEvent[] = [] + const boot = new Bootstrap({ steps: stubSteps(), onEvent: e => events.push(e) }) + const result = await boot.run() + + assert.equal(result.scope, 'full') + assert.equal(result.intent, 'a shop') + assert.equal(result.plan.stack, 'Vike + universal-orm') + assert.equal(result.run, fakeRun) + assert.equal(result.passes, 1) + assert.deepEqual(result.blockers, []) + assert.equal(result.productionGrade, true) + assert.equal(result.stoppedEarly, false) + + // narration order: scope, architect, its narration, build narration, loop, checklist, done + const types = events.map(e => e.type) + assert.deepEqual(types, ['scope', 'architect', 'narrate', 'narrate', 'narrate', 'checklist', 'done']) + const scoped = events[0] + assert.ok(scoped?.type === 'scope' && scoped.scope === 'full') + }) + + it('records the architect decisions to the ledger', async () => { + const ledger = new DecisionLedger() + const boot = new Bootstrap({ steps: stubSteps(), ledger }) + await boot.run() + assert.equal(boot.decisions, ledger) + assert.equal(ledger.size, 1) + assert.equal(ledger.all()[0]?.status, 'accepted') + assert.match(ledger.all()[0]!.title, /Vike for SSR/) + }) +}) + +describe('Bootstrap — full-fledged loop', () => { + it('improves against blockers and repeats until the checklist is clean', async () => { + const verdicts: Verdict[] = [{ blockers: ['no auth', 'no tests'] }, { blockers: [] }] + const improveCalls: readonly string[][] = [] + let call = 0 + const boot = new Bootstrap({ + steps: stubSteps({ + checklist: () => verdicts[call++]!, + improve: ({ blockers }) => { + ;(improveCalls as string[][]).push([...blockers]) + }, + }), + }) + const result = await boot.run() + + assert.equal(result.passes, 2) + assert.deepEqual(result.blockers, []) + assert.equal(result.productionGrade, true) + assert.equal(result.stoppedEarly, false) + // improve ran once, against pass 1's blockers + assert.deepEqual(improveCalls, [['no auth', 'no tests']]) + }) + + it('stops early at maxPasses with blockers still open', async () => { + const events: BootstrapEvent[] = [] + let improves = 0 + const boot = new Bootstrap({ + maxPasses: 2, + steps: stubSteps({ + checklist: () => ({ blockers: ['still no auth'] }), + improve: () => { improves++ }, + }), + onEvent: e => events.push(e), + }) + const result = await boot.run() + + assert.equal(result.passes, 2) + assert.deepEqual(result.blockers, ['still no auth']) + assert.equal(result.productionGrade, false) + assert.equal(result.stoppedEarly, true) + // improve runs only between passes, so once for 2 passes (no improve after the last) + assert.equal(improves, 1) + assert.equal(events.filter(e => e.type === 'checklist').length, 2) + }) +}) + +describe('Bootstrap — prototype scope', () => { + it('skips the full-fledged loop entirely', async () => { + const events: BootstrapEvent[] = [] + let checklistRuns = 0 + const boot = new Bootstrap({ + steps: stubSteps({ + scope: () => ({ scope: 'prototype', intent: 'quick demo' }), + checklist: () => { checklistRuns++; return { blockers: [] } }, + }), + onEvent: e => events.push(e), + }) + const result = await boot.run() + + assert.equal(result.scope, 'prototype') + assert.equal(result.passes, 0) + assert.equal(result.productionGrade, false) // not gated, so not claimed + assert.equal(checklistRuns, 0) + assert.equal(events.some(e => e.type === 'checklist'), false) + }) +}) + +describe('Bootstrap — interrupt + isolation', () => { + it('aborts between phases when the signal fires', async () => { + const controller = new AbortController() + const boot = new Bootstrap({ + signal: controller.signal, + steps: stubSteps({ + architect: () => { + controller.abort() // user interrupts during the architect phase + return { stack: 'x', narration: '', decisions: [] } + }, + }), + }) + await assert.rejects(() => boot.run(), BootstrapAborted) + }) + + it('does not start when already aborted', async () => { + const controller = new AbortController() + controller.abort() + let scopeRan = false + const boot = new Bootstrap({ + signal: controller.signal, + steps: stubSteps({ scope: () => { scopeRan = true; return { scope: 'full', intent: 'x' } } }), + }) + await assert.rejects(() => boot.run(), BootstrapAborted) + assert.equal(scopeRan, false) + }) + + it('isolates a throwing onEvent callback', async () => { + const boot = createBootstrap({ + steps: stubSteps(), + onEvent: () => { throw new Error('observer bug') }, + }) + // run completes despite the observer throwing on every event + const result = await boot.run() + assert.equal(result.productionGrade, true) + }) +}) + +describe('Bootstrap — construction', () => { + it('rejects missing steps and a bad maxPasses', () => { + // @ts-expect-error missing steps + assert.throws(() => new Bootstrap({}), /requires `steps`/) + // @ts-expect-error missing build + assert.throws(() => new Bootstrap({ steps: { scope: () => ({ scope: 'full', intent: '' }), architect: () => ({ stack: '', narration: '', decisions: [] }) } }), /`build` step/) + assert.throws(() => new Bootstrap({ steps: stubSteps(), maxPasses: 0 }), /maxPasses/) + }) +}) diff --git a/packages/ai-autopilot/src/bootstrap/bootstrap.ts b/packages/ai-autopilot/src/bootstrap/bootstrap.ts new file mode 100644 index 0000000..471597d --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/bootstrap.ts @@ -0,0 +1,168 @@ +import { DecisionLedger } from '../decisions/ledger.js' +import { isPassing } from '../loop/verdict.js' +import type { + BootstrapEvent, + BootstrapOptions, + BootstrapResult, + BootstrapSteps, +} from './types.js' + +/** Thrown when a run is aborted via the `AbortSignal`. */ +export class BootstrapAborted extends Error { + constructor() { + super('[ai-autopilot] bootstrap run was aborted') + this.name = 'BootstrapAborted' + } +} + +/** + * The bootstrap orchestrator: sequences the injected {@link BootstrapSteps} into + * scope → architect → build → full-fledged loop, narrating each phase over + * {@link BootstrapOptions.onEvent} and recording the architect's choices to the + * decisions ledger. It owns the control flow (the loop, the gate, the interrupt); + * the steps own the model/runner work, so it runs offline against stubs. + * + * ```ts + * const boot = new Bootstrap({ steps, onEvent: e => render(e) }) + * const result = await boot.run() + * if (result.productionGrade) deploy(result) + * ``` + * + * Pair it with `launchAutopilot` to run detached + * with a replayable narration stream. + */ +export class Bootstrap { + private readonly steps: BootstrapSteps + private readonly maxPasses: number + private readonly ledger: DecisionLedger + private readonly signal?: AbortSignal + private readonly emit: (event: BootstrapEvent) => void + + constructor(opts: BootstrapOptions) { + if (opts?.steps == null) throw new TypeError('[ai-autopilot] Bootstrap requires `steps`') + const { scope, architect, build } = opts.steps + if (typeof scope !== 'function') throw new TypeError('[ai-autopilot] Bootstrap needs a `scope` step') + if (typeof architect !== 'function') throw new TypeError('[ai-autopilot] Bootstrap needs an `architect` step') + if (typeof build !== 'function') throw new TypeError('[ai-autopilot] Bootstrap needs a `build` step') + + const maxPasses = opts.maxPasses ?? 3 + if (!Number.isInteger(maxPasses) || maxPasses < 1) { + throw new TypeError(`[ai-autopilot] Bootstrap maxPasses must be a positive integer, got ${opts.maxPasses}`) + } + + this.steps = opts.steps + this.maxPasses = maxPasses + this.ledger = opts.ledger ?? new DecisionLedger() + if (opts.signal) this.signal = opts.signal + this.emit = makeEmitter(opts.onEvent) + } + + /** The ledger the run records architect choices into (its own, or the injected one). */ + get decisions(): DecisionLedger { + return this.ledger + } + + /** Run the whole flow and resolve with the {@link BootstrapResult}. */ + async run(): Promise { + // 1. Scope — the only question we ask. + this.throwIfAborted() + const { scope, intent } = await this.steps.scope() + this.emit({ type: 'scope', scope, intent }) + + // 2. Architect — pick the stack, narrate, record the choices. No permission asked. + this.throwIfAborted() + const plan = await this.steps.architect({ + intent, + scope, + ledger: this.ledger, + ...(this.signal ? { signal: this.signal } : {}), + }) + for (const d of plan.decisions) this.ledger.accept(d.choice, d.why, ['architecture']) + this.emit({ type: 'architect', stack: plan.stack, decisions: plan.decisions }) + if (plan.narration) this.emit({ type: 'narrate', phase: 'architect', message: plan.narration }) + + // 3. Build — Supervisor over personas + runner; forward its events as narration. + this.throwIfAborted() + this.emit({ type: 'narrate', phase: 'build', message: `Building on ${plan.stack}` }) + const run = await this.steps.build({ + plan, + scope, + intent, + onEvent: event => this.emit({ type: 'build', event }), + ...(this.signal ? { signal: this.signal } : {}), + }) + + // 4. Full-fledged loop — repeat the checklist with fresh context until it is + // clean or the pass budget runs out. Prototype scope skips this. + let passes = 0 + let blockers: readonly string[] = [] + let stoppedEarly = false + if (scope === 'full' && this.steps.checklist) { + this.emit({ type: 'narrate', phase: 'loop', message: 'Checking the app is production-grade' }) + for (let pass = 1; pass <= this.maxPasses; pass++) { + this.throwIfAborted() + const verdict = await this.steps.checklist({ + pass, + plan, + intent, + blockers, + ...(this.signal ? { signal: this.signal } : {}), + }) + passes = pass + blockers = verdict.blockers + const passing = isPassing(verdict) + this.emit({ type: 'checklist', pass, blockers, passing }) + if (passing) break + if (pass === this.maxPasses) { + stoppedEarly = true + break + } + // Not passing and passes remain: improve against the blockers, fresh context. + this.throwIfAborted() + this.emit({ type: 'improve', pass, blockers }) + if (this.steps.improve) { + await this.steps.improve({ + pass, + plan, + intent, + blockers, + ...(this.signal ? { signal: this.signal } : {}), + }) + } + } + } + + const result: BootstrapResult = { + scope, + intent, + plan, + run, + passes, + blockers, + productionGrade: passes > 0 && blockers.length === 0, + stoppedEarly, + } + this.emit({ type: 'done', result }) + return result + } + + private throwIfAborted(): void { + if (this.signal?.aborted) throw new BootstrapAborted() + } +} + +/** Factory mirror of `new Bootstrap(...)`. */ +export function createBootstrap(opts: BootstrapOptions): Bootstrap { + return new Bootstrap(opts) +} + +function makeEmitter(onEvent: BootstrapOptions['onEvent']): (event: BootstrapEvent) => void { + if (!onEvent) return () => {} + return event => { + try { + onEvent(event) + } catch (err) { + console.error('[ai-autopilot] bootstrap onEvent callback threw; ignoring:', err) + } + } +} diff --git a/packages/ai-autopilot/src/bootstrap/index.ts b/packages/ai-autopilot/src/bootstrap/index.ts new file mode 100644 index 0000000..e42f651 --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/index.ts @@ -0,0 +1,40 @@ +/** + * Bootstrap mode — the spine from nothing to a running, production-grade app. + * + * {@link Bootstrap} sequences the injected {@link BootstrapSteps} into + * scope → architect → build → full-fledged loop, narrating each phase and + * recording the architect's choices to the decisions ledger. The default step + * builders wire those steps onto the real primitives (Supervisor, personas, the + * Loop); a test swaps in stubs + a `FakeRunner` to run the whole flow offline. + * + * - {@link Bootstrap} / {@link createBootstrap} — the orchestrator + * - {@link agentArchitect} — architect step over an `ai-sdk` agent + * - {@link supervisorBuild} — build step over the {@link Supervisor} + * - {@link loopChecklist} / {@link loopImprove} — the full-fledged loop steps + */ +export { Bootstrap, createBootstrap, BootstrapAborted } from './bootstrap.js' +export { + agentArchitect, + supervisorBuild, + loopChecklist, + loopImprove, + type ArchitectAgentOptions, + type SupervisorBuildOptions, + type LoopStepOptions, + type LoopChecklistOptions, + type LoopImproveOptions, +} from './steps.js' +export type { + BootstrapScope, + BootstrapPhase, + ScopeAnswer, + ArchitectDecision, + ArchitectPlan, + BootstrapEvent, + BootstrapResult, + BootstrapSteps, + BootstrapOptions, + BuildContext, + ArchitectContext, + LoopPassContext, +} from './types.js' diff --git a/packages/ai-autopilot/src/bootstrap/steps.test.ts b/packages/ai-autopilot/src/bootstrap/steps.test.ts new file mode 100644 index 0000000..3702c99 --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/steps.test.ts @@ -0,0 +1,192 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { AiFake, agent } from '@gemstack/ai-sdk' +import { agentArchitect, supervisorBuild, loopChecklist, loopImprove } from './steps.js' +import { Bootstrap } from './bootstrap.js' +import { DecisionLedger } from '../decisions/ledger.js' +import { Loop } from '../loop/loop.js' +import { definePrompt, defineRule } from '../loop/define.js' +import type { BootstrapEvent } from './types.js' +import type { SupervisorEvent } from '../types.js' +import type { ArchitectContext, BuildContext, LoopPassContext } from './types.js' + +const architectCtx = (over: Partial = {}): ArchitectContext => ({ + intent: 'a bookstore', + scope: 'full', + ledger: new DecisionLedger(), + ...over, +}) + +describe('agentArchitect (default step over an ai-sdk agent)', () => { + it('parses a structured plan and prepends the decisions briefing', async () => { + const fake = AiFake.fake() + try { + fake.respondWithSequence([ + { + text: JSON.stringify({ + stack: 'Vike + universal-orm', + narration: 'A server-rendered bookstore', + decisions: [{ choice: 'Postgres', why: 'relational catalog' }], + }), + }, + ]) + const ledger = new DecisionLedger() + ledger.reject('Use a NoSQL document store', 'the catalog is relational') + + const step = agentArchitect(agent({ instructions: 'architect' })) + const plan = await step(architectCtx({ ledger })) + + assert.equal(plan.stack, 'Vike + universal-orm') + assert.equal(plan.decisions[0]?.choice, 'Postgres') + // the rejected idea reached the model as a briefing so it will not re-pitch it + const sent = JSON.stringify(fake.getCalls()[0]) + assert.match(sent, /NoSQL document store/) + } finally { + fake.restore() + } + }) +}) + +describe('supervisorBuild (default step over the Supervisor)', () => { + it('runs the Supervisor and forwards its events as narration', async () => { + const fake = AiFake.fake() + try { + fake.respondWithSequence([{ text: 'wrote the page' }]) + const events: SupervisorEvent[] = [] + const step = supervisorBuild({ + plan: () => [{ description: 'build the catalog page', worker: 'w' }], + workers: { w: agent({ instructions: 'worker' }) }, + concurrency: 1, + }) + const ctx: BuildContext = { + plan: { stack: 'Vike + universal-orm', narration: '', decisions: [] }, + scope: 'full', + intent: 'a bookstore', + onEvent: e => events.push(e), + } + const run = await step(ctx) + + assert.ok(run.results.length === 1 && run.results[0]?.ok) + assert.ok(events.some(e => e.type === 'plan')) + assert.ok(events.some(e => e.type === 'synthesize')) + } finally { + fake.restore() + } + }) + + it('refuses to start when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const step = supervisorBuild({ plan: () => [], workers: agent({ instructions: 'w' }) }) + await assert.rejects( + async () => step({ plan: { stack: '', narration: '', decisions: [] }, scope: 'full', intent: '', onEvent: () => {}, signal: controller.signal }), + /aborted before start/, + ) + }) +}) + +describe('loopChecklist / loopImprove (default full-fledged loop steps)', () => { + const passCtx = (over: Partial = {}): LoopPassContext => ({ + pass: 1, + plan: { stack: 'Vike + universal-orm', narration: '', decisions: [] }, + intent: 'a bookstore', + blockers: [], + ...over, + }) + + it('reads the { blockers } verdict the production-grade prompt returns', async () => { + const loop = new Loop({ + rules: [defineRule({ on: 'production-check', run: ['production-grade'] })], + prompts: [definePrompt({ id: 'production-grade', run: () => '```json\n{ "blockers": ["no auth"] }\n```' })], + }) + const verdict = await loopChecklist({ loop })(passCtx()) + assert.deepEqual(verdict.blockers, ['no auth']) + }) + + it('treats a missing verdict as a blocker', async () => { + const loop = new Loop({ + rules: [defineRule({ on: 'production-check', run: ['production-grade'] })], + prompts: [definePrompt({ id: 'production-grade', run: () => 'no verdict here' })], + }) + const verdict = await loopChecklist({ loop })(passCtx()) + assert.equal(verdict.blockers.length, 1) + assert.match(verdict.blockers[0]!, /did not return a verdict/) + }) + + it('fires the change events so the review chain runs', async () => { + let reviewRan = 0 + const loop = new Loop({ + rules: [defineRule({ on: 'major-change', run: ['review'] })], + prompts: [definePrompt({ id: 'review', run: () => { reviewRan++; return 'reviewed' } })], + }) + await loopImprove({ loop })(passCtx({ blockers: ['no auth'] })) + assert.equal(reviewRan, 1) + }) +}) + +describe('Bootstrap end-to-end with the default steps (offline)', () => { + it('runs scope → architect → build → full-fledged loop against real primitives', async () => { + const fake = AiFake.fake() + try { + // Two model calls, in order: the architect plan, then the one build worker. + fake.respondWithSequence([ + { + text: JSON.stringify({ + stack: 'Vike + universal-orm', + narration: 'A server-rendered bookstore with a Postgres data layer', + decisions: [{ choice: 'universal-orm on Postgres', why: 'typed, relational catalog' }], + }), + }, + { text: 'scaffolded the catalog page and orders schema' }, + ]) + + // The full-fledged loop: first checklist has a blocker, second is clean. + const verdicts = ['```json\n{ "blockers": ["no auth"] }\n```', '```json\n{ "blockers": [] }\n```'] + let checked = 0 + let improved = 0 + const loop = new Loop({ + rules: [ + defineRule({ on: 'production-check', run: ['production-grade'] }), + defineRule({ on: 'major-change', run: ['fix'] }), + ], + prompts: [ + definePrompt({ id: 'production-grade', run: () => verdicts[checked++] ?? '```json\n{ "blockers": [] }\n```' }), + definePrompt({ id: 'fix', run: () => { improved++; return 'addressed the blockers' } }), + ], + }) + + const ledger = new DecisionLedger() + const events: BootstrapEvent[] = [] + const boot = new Bootstrap({ + ledger, + onEvent: e => events.push(e), + steps: { + scope: () => ({ scope: 'full', intent: 'a bookstore' }), + architect: agentArchitect(agent({ instructions: 'architect' })), + build: supervisorBuild({ + plan: () => [{ description: 'scaffold the app', worker: 'w' }], + workers: { w: agent({ instructions: 'worker' }) }, + concurrency: 1, + }), + checklist: loopChecklist({ loop }), + improve: loopImprove({ loop }), + }, + }) + + const result = await boot.run() + + assert.equal(result.plan.stack, 'Vike + universal-orm') + assert.equal(result.run.results[0]?.ok, true) + assert.equal(result.passes, 2) + assert.deepEqual(result.blockers, []) + assert.equal(result.productionGrade, true) + assert.equal(improved, 1) // improved once, between the two checks + assert.equal(ledger.size, 1) // architect choice recorded + // build events were forwarded into the narration + assert.ok(events.some(e => e.type === 'build' && e.event.type === 'plan')) + assert.ok(events.some(e => e.type === 'checklist' && e.passing)) + } finally { + fake.restore() + } + }) +}) diff --git a/packages/ai-autopilot/src/bootstrap/steps.ts b/packages/ai-autopilot/src/bootstrap/steps.ts new file mode 100644 index 0000000..aa8c0bd --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/steps.ts @@ -0,0 +1,146 @@ +import { Output } from '@gemstack/ai-sdk' +import type { Agent } from '@gemstack/ai-sdk' +import { z } from 'zod' +import { Supervisor } from '../supervisor.js' +import { decisionBriefing } from '../decisions/tools.js' +import { Loop } from '../loop/loop.js' +import { LOOP_EVENTS, LOOP_PROMPTS } from '../loop/policy.js' +import type { Planner, Synthesizer, SupervisorOptions } from '../types.js' +import type { Verdict } from '../loop/verdict.js' +import type { BootstrapSteps, BuildContext } from './types.js' + +/** + * Default wirings of the four bootstrap steps onto the real primitives — + * Supervisor, personas, and the Loop. Each is thin (it constructs a primitive + * and adapts its I/O to the step contract), so the same orchestrator runs + * against these in production or against stubs in a test. The model + runner stay + * injected: you pass the architect agent, the planner/workers, and the loop. + */ + +/** Options for {@link agentArchitect}. */ +export interface ArchitectAgentOptions { + /** Override the architect instruction prepended to the intent. */ + instructions?: string +} + +const DEFAULT_ARCHITECT_INSTRUCTIONS = `You are the lead architect. Choose the stack and structure for the app the user +describes and commit to it — act like a senior engineer who decides and explains, +not one who asks permission. Default to the GemStack stack (Vike + universal-orm) +unless the intent clearly calls for something else. Narrate what you are building +and why in a sentence or two, and list the key choices so they are recorded and +not re-litigated later.` + +/** + * An architect step backed by an `ai-sdk` agent. It prompts the agent for a + * structured `{ stack, narration, decisions }` plan (via `Output.object`), and + * prepends the decisions briefing so it does not re-pitch an already-rejected + * idea. The orchestrator records the returned choices to the ledger. + */ +export function agentArchitect(architect: Agent, opts: ArchitectAgentOptions = {}): BootstrapSteps['architect'] { + const schema = z.object({ + stack: z.string().describe('The chosen stack, one line'), + narration: z.string().describe('What you are building and why, to tell the user'), + decisions: z + .array(z.object({ choice: z.string(), why: z.string() })) + .describe('Key architectural choices and their rationale'), + }) + const output = Output.object({ schema }) + const instructions = opts.instructions ?? DEFAULT_ARCHITECT_INSTRUCTIONS + + return async ({ intent, scope, ledger }) => { + const briefing = decisionBriefing(ledger) + const head = briefing ? `${briefing}\n\n${instructions}` : instructions + const prompt = `${head}\n\n# What the user wants (${scope})\n${intent}\n\n${output.toSystemPrompt()}` + const response = await architect.prompt(prompt) + return output.parse(response.text ?? '') + } +} + +/** Options for {@link supervisorBuild}. */ +export interface SupervisorBuildOptions { + /** How to decompose the build into subtasks (usually `agentPlanner(...)`). */ + plan: Planner + /** The worker agents (usually persona workers with runner tools). */ + workers: SupervisorOptions['workers'] + synthesize?: Synthesizer + concurrency?: number + budget?: SupervisorOptions['budget'] + /** Build the task text from the plan + intent. Default: intent + chosen stack. */ + task?: (ctx: BuildContext) => string +} + +const defaultBuildTask = (ctx: BuildContext): string => + `Build the app.\n\n# Goal\n${ctx.intent}\n\n# Stack\n${ctx.plan.stack}` + +/** + * A build step that runs the {@link Supervisor} over the given planner + workers, + * forwarding its events to bootstrap's narration. The Supervisor has no native + * abort, so the step honors `signal` by not starting once it is already aborted. + */ +export function supervisorBuild(opts: SupervisorBuildOptions): BootstrapSteps['build'] { + const makeTask = opts.task ?? defaultBuildTask + return async ctx => { + if (ctx.signal?.aborted) throw new Error('[ai-autopilot] build aborted before start') + const supervisor = new Supervisor({ + plan: opts.plan, + workers: opts.workers, + ...(opts.synthesize ? { synthesize: opts.synthesize } : {}), + ...(opts.concurrency ? { concurrency: opts.concurrency } : {}), + ...(opts.budget ? { budget: opts.budget } : {}), + onEvent: ctx.onEvent, + }) + return supervisor.run(makeTask(ctx)) + } +} + +/** Options for {@link loopChecklist} and {@link loopImprove}. */ +export interface LoopStepOptions { + /** The loop that resolves the prompt ids to bodies (via `loopPromptsFor`). */ + loop: Loop +} + +/** Options for {@link loopChecklist}. */ +export interface LoopChecklistOptions extends LoopStepOptions { + /** The event kind whose chain runs the checklist prompt. Default `production-check`. */ + kind?: string + /** The prompt id to read the `{ blockers }` verdict from. Default `production-grade`. */ + promptId?: string +} + +/** + * A checklist step that fires a check event into the loop and returns the + * {@link Verdict} the production-grade prompt reported. A missing verdict is + * treated as a blocker (the checklist must return one to pass). + */ +export function loopChecklist(opts: LoopChecklistOptions): NonNullable { + const kind = opts.kind ?? 'production-check' + const promptId = opts.promptId ?? LOOP_PROMPTS.productionGrade + return async ({ intent, blockers }) => { + const summary = blockers.length ? `Re-check after addressing: ${blockers.join('; ')}` : intent + const result = await opts.loop.handle({ kind, summary }) + const outcome = result.outcomes.find(o => o.promptId === promptId) + return outcome?.verdict ?? ({ blockers: [`checklist "${promptId}" did not return a verdict`] } satisfies Verdict) + } +} + +/** Options for {@link loopImprove}. */ +export interface LoopImproveOptions extends LoopStepOptions { + /** Change event kinds to fire so the review / QA chains run. Default `major-change`. */ + kinds?: string[] +} + +/** + * An improve step that fires the change events into the loop, so its review / + * code-quality / security (and QA / UX) prompts run with fresh context against + * the current app before the next checklist. The prompt agents do the fixing + * (they carry the runner tools); this step just triggers the chains. + */ +export function loopImprove(opts: LoopImproveOptions): NonNullable { + const kinds = opts.kinds ?? [LOOP_EVENTS.majorChange] + return async ({ blockers }) => { + const summary = blockers.length ? `Address blockers: ${blockers.join('; ')}` : 'Improve the app toward production-grade' + for (const kind of kinds) { + await opts.loop.handle({ kind, summary }) + } + } +} diff --git a/packages/ai-autopilot/src/bootstrap/types.ts b/packages/ai-autopilot/src/bootstrap/types.ts new file mode 100644 index 0000000..6afaf8c --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/types.ts @@ -0,0 +1,146 @@ +import type { DecisionLedger } from '../decisions/ledger.js' +import type { SupervisorEvent, SupervisorRun } from '../types.js' +import type { Verdict } from '../loop/verdict.js' + +/** + * Bootstrap mode — the spine that takes a user from nothing to a running, + * well-structured app (#116). It sequences autopilot's existing primitives into + * one flow: + * + * scope → architect → build → full-fledged loop + * + * - **Scope** is the one and only interrogation: prototype vs full, plus intent. + * - **Architect** picks the stack (Vike + universal-orm), narrates it, and + * records key choices to the decisions ledger — no permission asked. + * - **Build** runs the Supervisor over the stack personas inside a runner, + * streaming narration; the caller can interrupt via an `AbortSignal`. + * - **Full-fledged loop** (full scope only) repeats the production-grade + * checklist with fresh context, improving against its `{ blockers }` verdict + * until it is empty or a `maxPasses` budget stops it. + * + * The orchestrator ({@link Bootstrap}) owns the sequencing, narration, and loop + * control; the four {@link BootstrapSteps} are injected, so a test drives it with + * stubs + a {@link FakeRunner} while production wires real agents. Its narration + * rides the generic surface stream (`EventStream`). + */ + +/** How much app the user wants: a quick prototype, or the full production thing. */ +export type BootstrapScope = 'prototype' | 'full' + +/** The phase a narration line belongs to. */ +export type BootstrapPhase = 'scope' | 'architect' | 'build' | 'loop' + +/** The answer to the one upfront question. */ +export interface ScopeAnswer { + scope: BootstrapScope + /** What the user wants built, in their words. */ + intent: string +} + +/** One architectural choice the architect made and why — recorded to the ledger. */ +export interface ArchitectDecision { + choice: string + why: string +} + +/** The architect's output: the stack it chose, a narration, and the key choices. */ +export interface ArchitectPlan { + /** The chosen stack, one line (e.g. "Vike + universal-orm, Postgres, vike-auth"). */ + stack: string + /** What it is building and why, to narrate to the user. */ + narration: string + /** Key choices to record to the decisions ledger so they are not re-litigated. */ + decisions: readonly ArchitectDecision[] +} + +/** + * A narration/progress event bootstrap emits over the generic surface stream. + * The build phase forwards the Supervisor's own events verbatim under `build`. + */ +export type BootstrapEvent = + | { type: 'scope'; scope: BootstrapScope; intent: string } + | { type: 'architect'; stack: string; decisions: readonly ArchitectDecision[] } + | { type: 'narrate'; phase: BootstrapPhase; message: string } + | { type: 'build'; event: SupervisorEvent } + | { type: 'checklist'; pass: number; blockers: readonly string[]; passing: boolean } + | { type: 'improve'; pass: number; blockers: readonly string[] } + | { type: 'done'; result: BootstrapResult } + +/** The outcome of a bootstrap run. */ +export interface BootstrapResult { + scope: BootstrapScope + intent: string + /** The architecture chosen. */ + plan: ArchitectPlan + /** The initial build's supervised run. */ + run: SupervisorRun + /** Full-fledged passes performed (0 for a prototype, or when no checklist ran). */ + passes: number + /** Remaining blockers from the last checklist; empty means nothing left to fix. */ + blockers: readonly string[] + /** True when the full-fledged loop ran and ended with no blockers. */ + productionGrade: boolean + /** True when the loop hit `maxPasses` with blockers still open. */ + stoppedEarly: boolean +} + +/** Context handed to the build step. */ +export interface BuildContext { + plan: ArchitectPlan + scope: BootstrapScope + intent: string + /** Forward each Supervisor event here so bootstrap can narrate the build. */ + onEvent: (event: SupervisorEvent) => void + signal?: AbortSignal +} + +/** Context handed to the architect step. */ +export interface ArchitectContext { + intent: string + scope: BootstrapScope + /** The ledger to consult (choices are recorded by the orchestrator, not here). */ + ledger: DecisionLedger + signal?: AbortSignal +} + +/** Context handed to the improve / checklist steps in the full-fledged loop. */ +export interface LoopPassContext { + /** 1-based pass number; each pass is meant to run with fresh context. */ + pass: number + plan: ArchitectPlan + intent: string + /** The blockers the checklist last reported (empty on the improve of pass 1). */ + blockers: readonly string[] + signal?: AbortSignal +} + +/** + * The four injectable steps. Only `scope`, `architect`, and `build` are required; + * `checklist` (and its paired `improve`) drive the full-fledged loop and are used + * only for `scope: 'full'`. + */ +export interface BootstrapSteps { + /** The one upfront question. */ + scope: () => ScopeAnswer | Promise + /** Pick the stack, narrate, and return the key choices. */ + architect: (ctx: ArchitectContext) => ArchitectPlan | Promise + /** Run the build (Supervisor over personas + runner). */ + build: (ctx: BuildContext) => SupervisorRun | Promise + /** Report the production-grade verdict for a pass. Full scope only. */ + checklist?: (ctx: LoopPassContext) => Verdict | Promise + /** Address the current blockers with fresh context, before the next checklist. */ + improve?: (ctx: LoopPassContext) => unknown | Promise +} + +/** Options for {@link Bootstrap}. */ +export interface BootstrapOptions { + steps: BootstrapSteps + /** Max full-fledged passes before stopping with blockers open. Default 3. */ + maxPasses?: number + /** The decisions ledger. A fresh one is created when omitted. */ + ledger?: DecisionLedger + /** Observe narration. Isolated: a throwing callback is logged and swallowed. */ + onEvent?: (event: BootstrapEvent) => void + /** Interrupt the run between phases (the "user can interrupt" affordance). */ + signal?: AbortSignal +} diff --git a/packages/ai-autopilot/src/index.ts b/packages/ai-autopilot/src/index.ts index 4d84997..042c74d 100644 --- a/packages/ai-autopilot/src/index.ts +++ b/packages/ai-autopilot/src/index.ts @@ -61,6 +61,15 @@ * - {@link builtinLibrary} — load the shipped, stack-aware prompt bodies * - {@link loopPromptsFor} — materialize a library into loop prompts by id * - {@link promptInstructions} — compose a body with the decisions briefing + * + * Bootstrap mode is the spine that sequences all of the above into one flow: + * scope → architect → build → full-fledged loop, taking a user from nothing to a + * running, production-grade app. It narrates each phase and repeats the + * production-grade checklist until its `{ blockers }` verdict is empty. + * + * - {@link Bootstrap} — the orchestrator over four injectable steps + * - {@link agentArchitect} / {@link supervisorBuild} — the default step wirings + * - {@link loopChecklist} / {@link loopImprove} — the full-fledged loop steps */ export { Supervisor } from './supervisor.js' export { agentPlanner, type AgentPlannerOptions } from './planner.js' @@ -175,6 +184,32 @@ export { type MakePromptAgent, type PromptAgentContext, } from './prompts/index.js' +export { + Bootstrap, + createBootstrap, + BootstrapAborted, + agentArchitect, + supervisorBuild, + loopChecklist, + loopImprove, + type ArchitectAgentOptions, + type SupervisorBuildOptions, + type LoopStepOptions, + type LoopChecklistOptions, + type LoopImproveOptions, + type BootstrapScope, + type BootstrapPhase, + type ScopeAnswer, + type ArchitectDecision, + type ArchitectPlan, + type BootstrapEvent, + type BootstrapResult, + type BootstrapSteps, + type BootstrapOptions, + type BuildContext, + type ArchitectContext, + type LoopPassContext, +} from './bootstrap/index.js' export type { Subtask, PlannedSubtask,