From 865c32149d4d4138c2687c99b611e6558cb07d7e Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 2 Jul 2026 23:20:24 +0300 Subject: [PATCH] =?UTF-8?q?feat(ai-autopilot):=20serveCheck=20=E2=80=94=20?= =?UTF-8?q?a=20boot-and-serve=20production-grade=20check=20(#140)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-fledged loop gated only on a prompt verdict (loopChecklist asks the model whether the app is production-grade). Now that the runner can start a dev server and reach it (#137), the loop can gate on something concrete: does the app actually boot and serve? Add: - serveCheck(session, { serve, install?, build?, port?, healthPath?, ... }) — a checklist-shaped step that runs inside the build's runner session: install -> (build) -> start the dev server -> preview until reachable -> fetch a health path, turning any failure (install error, server exits on boot, 5xx, unreachable) into a concrete { blockers } verdict the improve loop addresses. Skips (passing, with a note) when the runner can't start/preview. - mergeChecklists(...checks) — union the blockers of several checks, so a pass must BOTH read production-grade AND actually run. Verified: real boot-and-serve success on LocalRunner (clean verdict) plus 5xx, exits-before-serving, install-fail (never starts), incapable-runner-skip, and the merge/dedupe combinator. 220 tests green. MINOR changeset. --- .changeset/serve-check.md | 7 ++ examples/bootstrap-quickstart/README.md | 24 +++- packages/ai-autopilot/src/bootstrap/index.ts | 1 + .../src/bootstrap/serve-check.test.ts | 98 +++++++++++++++ .../ai-autopilot/src/bootstrap/serve-check.ts | 116 ++++++++++++++++++ packages/ai-autopilot/src/index.ts | 3 + 6 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 .changeset/serve-check.md create mode 100644 packages/ai-autopilot/src/bootstrap/serve-check.test.ts create mode 100644 packages/ai-autopilot/src/bootstrap/serve-check.ts diff --git a/.changeset/serve-check.md b/.changeset/serve-check.md new file mode 100644 index 0000000..92045ba --- /dev/null +++ b/.changeset/serve-check.md @@ -0,0 +1,7 @@ +--- +'@gemstack/ai-autopilot': minor +--- + +Bootstrap: `serveCheck` — a production-grade check that actually boots and serves the app. + +Until now the full-fledged loop gated on a prompt verdict (`loopChecklist` asks the model whether the app is production-grade). `serveCheck(session, { serve, install?, build?, port?, healthPath? })` gives it teeth: inside the build's runner session it installs, optionally builds, `start`s the dev server, `preview`s until the port is reachable, and fetches a health path — turning any failure (install error, server exits on boot, a 5xx, unreachable) into a concrete `{ blockers }` verdict the improve loop then addresses. It satisfies the `checklist` step contract, and `mergeChecklists(...)` unions several checks so a pass must BOTH read production-grade AND actually run: `mergeChecklists(loopChecklist({ loop }), serveCheck(session, { serve: 'npm run dev' }))`. A runner that can't `start`/`preview` skips the check (passing, with a note) instead of blocking. Built on the #137 runner seam. diff --git a/examples/bootstrap-quickstart/README.md b/examples/bootstrap-quickstart/README.md index ea4f92c..3648406 100644 --- a/examples/bootstrap-quickstart/README.md +++ b/examples/bootstrap-quickstart/README.md @@ -54,5 +54,25 @@ checklist once on missing auth, then decided SSR → dockploy. Scoped for a first, bounded proof: the production-grade **loop** keeps the scripted verdict (so the run stays deterministic and cheap), and `deploy` uses `planOnlyTarget` -— it decides + narrates but does not actually ship. Booting/serving the generated app -and real deploy adapters remain (tracked by #109). +— it decides + narrates but does not actually ship. Real deploy adapters remain (#109). + +### Giving the loop teeth: `serveCheck` + +The scripted checklist only *asks* whether the app is production-grade. To gate a pass +on whether the app actually **boots and serves**, compose a `serveCheck` into the +checklist — it runs install → start the dev server → `preview` → fetch a health path, +turning failures into blockers the improve loop then fixes: + +```js +import { serveCheck, mergeChecklists, loopChecklist } from '@gemstack/ai-autopilot' + +// A pass is production-grade only when the prompt says so AND the app really runs. +checklist: mergeChecklists( + loopChecklist({ loop }), + serveCheck(session, { install: 'npm install', serve: 'npm run dev', port: 3000 }), +), +``` + +It's left out of the default live run because a model-scaffolded app may not install or +boot first try (that's the loop's job to fix) — enable it when you want the run to prove +the app serves, not just that files were written. diff --git a/packages/ai-autopilot/src/bootstrap/index.ts b/packages/ai-autopilot/src/bootstrap/index.ts index aabb360..bebab34 100644 --- a/packages/ai-autopilot/src/bootstrap/index.ts +++ b/packages/ai-autopilot/src/bootstrap/index.ts @@ -32,6 +32,7 @@ export { type AgentDeployOptions, type FakeDeployTargetOptions, } from './deploy.js' +export { serveCheck, mergeChecklists, type ServeCheckOptions } from './serve-check.js' export type { BootstrapScope, BootstrapPhase, diff --git a/packages/ai-autopilot/src/bootstrap/serve-check.test.ts b/packages/ai-autopilot/src/bootstrap/serve-check.test.ts new file mode 100644 index 0000000..125b6b2 --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/serve-check.test.ts @@ -0,0 +1,98 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { createServer } from 'node:net' +import { serveCheck, mergeChecklists } from './serve-check.js' +import { LocalRunner } from '../runner/local.js' +import { FakeRunner } from '../runner/fake.js' +import type { LoopPassContext } from './types.js' +import type { Verdict } from '../loop/verdict.js' + +/** A minimal loop-pass context — serveCheck ignores it, but the contract needs one. */ +const ctx: LoopPassContext = { pass: 1, plan: { stack: '', narration: '', decisions: [] }, intent: '', blockers: [] } + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const srv = createServer() + srv.on('error', reject) + srv.listen(0, '127.0.0.1', () => { + const addr = srv.address() + const port = typeof addr === 'object' && addr ? addr.port : 0 + srv.close(() => resolve(port)) + }) + }) +} + +const server = (port: number, status: number, body: string): string => + `const http=require('http');http.createServer((_,res)=>{res.writeHead(${status});res.end(${JSON.stringify(body)})}).listen(${port},'127.0.0.1')` + +describe('serveCheck (boots and serves the app)', () => { + it('passes with no blockers when the app boots and serves', async () => { + const port = await freePort() + const s = await new LocalRunner().boot({ files: { 'server.js': server(port, 200, 'ok') } }) + try { + const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 5000 })(ctx) + assert.deepEqual(verdict.blockers, []) + } finally { + await s.dispose() + } + }) + + it('blocks when the app serves a server error', async () => { + const port = await freePort() + const s = await new LocalRunner().boot({ files: { 'server.js': server(port, 500, 'boom') } }) + try { + const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 5000 })(ctx) + assert.equal(verdict.blockers.length, 1) + assert.match(verdict.blockers[0]!, /responded 500/) + } finally { + await s.dispose() + } + }) + + it('blocks when the dev server exits before serving', async () => { + const port = await freePort() + const s = await new LocalRunner().boot({ files: { 'server.js': 'process.exit(1)' } }) + try { + const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 1000 })(ctx) + assert.equal(verdict.blockers.length, 1) + assert.match(verdict.blockers[0]!, /exited before serving/) + } finally { + await s.dispose() + } + }) + + it('blocks on a failed install and never starts the server', async () => { + const runner = new FakeRunner({ + onExec: cmd => (cmd.includes('install') ? { stdout: '', stderr: 'ERESOLVE', exitCode: 1 } : { stdout: '', stderr: '', exitCode: 0 }), + }) + const s = await runner.boot() + const verdict = await serveCheck(s, { install: 'npm install', serve: 'npm run dev' })(ctx) + assert.equal(verdict.blockers.length, 1) + assert.match(verdict.blockers[0]!, /install failed/) + assert.equal(s.startCalls.length, 0) // never got as far as starting + }) + + it('skips (passing, with a note) when the runner cannot serve a preview', async () => { + const s = await new FakeRunner({ background: false }).boot() + const verdict = await serveCheck(s, { serve: 'npm run dev' })(ctx) + assert.deepEqual(verdict.blockers, []) + assert.match(verdict.notes!, /skipped/) + }) +}) + +describe('mergeChecklists', () => { + it('unions and dedupes the blockers of every check', async () => { + const a = async (): Promise => ({ blockers: ['no auth', 'slow query'] }) + const b = async (): Promise => ({ blockers: ['slow query', 'missing tests'], notes: 'serve ok' }) + const verdict = await mergeChecklists(a, b)(ctx) + assert.deepEqual([...verdict.blockers].sort(), ['missing tests', 'no auth', 'slow query']) + assert.equal(verdict.notes, 'serve ok') + }) + + it('passes only when every check is clean', async () => { + const clean = async (): Promise => ({ blockers: [] }) + const dirty = async (): Promise => ({ blockers: ['boom'] }) + assert.deepEqual((await mergeChecklists(clean, clean)(ctx)).blockers, []) + assert.deepEqual((await mergeChecklists(clean, dirty)(ctx)).blockers, ['boom']) + }) +}) diff --git a/packages/ai-autopilot/src/bootstrap/serve-check.ts b/packages/ai-autopilot/src/bootstrap/serve-check.ts new file mode 100644 index 0000000..f93d1fe --- /dev/null +++ b/packages/ai-autopilot/src/bootstrap/serve-check.ts @@ -0,0 +1,116 @@ +import type { RunnerSession } from '../runner/types.js' +import type { Verdict } from '../loop/verdict.js' +import type { BootstrapSteps, LoopPassContext } from './types.js' + +/** + * A production-grade check with teeth: instead of only *asking* whether the app + * is production-grade (the prompt-based {@link loopChecklist}), this one actually + * BOOTS it and confirms it serves. It runs inside the same runner session the + * build wrote into — install deps, optionally build, `start` the dev server, + * `preview` until the port is reachable, then fetch a health path — and turns any + * failure into a concrete `{ blockers }` verdict the full-fledged loop addresses. + * + * It satisfies the `checklist` step contract, so wire it directly + * (`checklist: serveCheck(session, { serve: 'npm run dev' })`) or, more usefully, + * compose it with the prompt checklist via {@link mergeChecklists} so a pass must + * BOTH read production-grade AND actually run. + * + * Needs a runner that can `start` background processes and `preview` them (the + * #137 seam). A runner without those capabilities can't verify, so the check is + * skipped (a passing verdict with a note) rather than blocking forever. + */ +export interface ServeCheckOptions { + /** The command that starts the dev server (e.g. `npm run dev`). Required. */ + serve: string + /** Install command run first (e.g. `npm install`). Skipped when omitted. */ + install?: string + /** Build command run after install (e.g. `npm run build`). Skipped when omitted. */ + build?: string + /** The port the dev server listens on. Default 3000. */ + port?: number + /** How long to wait for the server to accept connections. Default 15000ms. */ + waitMs?: number + /** Path to fetch once the server is up. Default `/`. */ + healthPath?: string + /** A response status at or above this is a blocker (the app errored). Default 500. */ + errorStatusFrom?: number + /** Per-command timeout for install/build. Default 120000ms. */ + commandTimeoutMs?: number + /** Optional progress narration (install / start / fetch). */ + onProgress?: (message: string) => void +} + +/** + * Build a `checklist` step that verifies the app boots and serves in `session`. + * Returns a {@link Verdict}: empty `blockers` means it installed, started, and + * responded without a server error; otherwise each blocker names what failed. + */ +export function serveCheck(session: RunnerSession, opts: ServeCheckOptions): NonNullable { + const port = opts.port ?? 3000 + const waitMs = opts.waitMs ?? 15_000 + const healthPath = opts.healthPath ?? '/' + const errorFrom = opts.errorStatusFrom ?? 500 + const commandTimeoutMs = opts.commandTimeoutMs ?? 120_000 + const say = (m: string): void => opts.onProgress?.(m) + + return async (_ctx: LoopPassContext): Promise => { + if (!session.start || !session.preview) { + return { blockers: [], notes: 'serve check skipped: runner cannot start/preview a dev server' } + } + + // Install / build are prerequisites — if they fail, there is nothing to serve. + for (const [label, command] of [['install', opts.install], ['build', opts.build]] as const) { + if (!command) continue + say(`${label}: ${command}`) + const res = await session.exec(command, { timeoutMs: commandTimeoutMs }) + if (res.exitCode !== 0) { + const tail = (res.stderr || res.stdout).trim().split('\n').slice(-3).join(' ').slice(0, 300) + return { blockers: [`${label} failed (\`${command}\` exited ${res.exitCode})${tail ? `: ${tail}` : ''}`] } + } + } + + say(`start: ${opts.serve}`) + const proc = await session.start(opts.serve) + try { + const { url } = await session.preview({ port, waitMs }) + const target = new URL(healthPath, url.endsWith('/') ? url : url + '/').toString() + + // If the server crashed on boot, its process has already exited — say so. + const raced = await Promise.race([ + fetch(target).then(res => ({ ok: true as const, status: res.status }), err => ({ ok: false as const, error: err as Error })), + proc.exit.then(r => ({ ok: false as const, exited: r.exitCode, log: (r.stderr || r.stdout).trim().split('\n').slice(-3).join(' ').slice(0, 300) })), + ]) + + if ('exited' in raced) { + return { blockers: [`the dev server (\`${opts.serve}\`) exited before serving (code ${raced.exited})${raced.log ? `: ${raced.log}` : ''}`] } + } + if (!raced.ok) { + return { blockers: [`the app did not serve at ${target}: ${raced.error.message}`] } + } + say(`fetch ${target} -> ${raced.status}`) + if (raced.status >= errorFrom) { + return { blockers: [`the app responded ${raced.status} at ${healthPath} (expected < ${errorFrom})`] } + } + return { blockers: [] } + } finally { + await proc.stop() + } + } +} + +/** + * Compose several `checklist` steps into one: run them all and union their + * blockers (deduped). Use it to gate a pass on BOTH the prompt verdict and a real + * serve check — `mergeChecklists(loopChecklist({ loop }), serveCheck(session, ...))`. + * A pass is production-grade only when every check comes back clean. + */ +export function mergeChecklists( + ...checks: ReadonlyArray> +): NonNullable { + return async (ctx: LoopPassContext): Promise => { + const verdicts = await Promise.all(checks.map(check => check(ctx))) + const blockers = [...new Set(verdicts.flatMap(v => v.blockers))] + const notes = verdicts.map(v => v.notes).filter(Boolean).join(' | ') + return { blockers, ...(notes ? { notes } : {}) } + } +} diff --git a/packages/ai-autopilot/src/index.ts b/packages/ai-autopilot/src/index.ts index e1c5760..04aa63b 100644 --- a/packages/ai-autopilot/src/index.ts +++ b/packages/ai-autopilot/src/index.ts @@ -221,6 +221,9 @@ export { planOnlyTarget, FakeDeployTarget, DEFAULT_DEPLOY_TARGETS, + serveCheck, + mergeChecklists, + type ServeCheckOptions, type ArchitectAgentOptions, type SupervisorBuildOptions, type LoopStepOptions,