feat(ai-autopilot): serveCheck — boot-and-serve production-grade check (#140)

.changeset/serve-check.md

@@ -0,0 +1,7 @@
+---
+'@gemstack/ai-autopilot': minor
+---
+
+Bootstrap: `serveCheck` — a production-grade check that actually boots and serves the app.
+
+Until now the full-fledged loop gated on a prompt verdict (`loopChecklist` asks the model whether the app is production-grade). `serveCheck(session, { serve, install?, build?, port?, healthPath? })` gives it teeth: inside the build's runner session it installs, optionally builds, `start`s the dev server, `preview`s until the port is reachable, and fetches a health path — turning any failure (install error, server exits on boot, a 5xx, unreachable) into a concrete `{ blockers }` verdict the improve loop then addresses. It satisfies the `checklist` step contract, and `mergeChecklists(...)` unions several checks so a pass must BOTH read production-grade AND actually run: `mergeChecklists(loopChecklist({ loop }), serveCheck(session, { serve: 'npm run dev' }))`. A runner that can't `start`/`preview` skips the check (passing, with a note) instead of blocking. Built on the #137 runner seam.

examples/bootstrap-quickstart/README.md

@@ -54,5 +54,25 @@ checklist once on missing auth, then decided SSR → dockploy.
 
 Scoped for a first, bounded proof: the production-grade **loop** keeps the scripted
 verdict (so the run stays deterministic and cheap), and `deploy` uses `planOnlyTarget`
-— it decides + narrates but does not actually ship. Booting/serving the generated app
-and real deploy adapters remain (tracked by #109).
+— it decides + narrates but does not actually ship. Real deploy adapters remain (#109).
+
+### Giving the loop teeth: `serveCheck`
+
+The scripted checklist only *asks* whether the app is production-grade. To gate a pass
+on whether the app actually **boots and serves**, compose a `serveCheck` into the
+checklist — it runs install → start the dev server → `preview` → fetch a health path,
+turning failures into blockers the improve loop then fixes:
+
+```js
+import { serveCheck, mergeChecklists, loopChecklist } from '@gemstack/ai-autopilot'
+
+// A pass is production-grade only when the prompt says so AND the app really runs.
+checklist: mergeChecklists(
+  loopChecklist({ loop }),
+  serveCheck(session, { install: 'npm install', serve: 'npm run dev', port: 3000 }),
+),
+```
+
+It's left out of the default live run because a model-scaffolded app may not install or
+boot first try (that's the loop's job to fix) — enable it when you want the run to prove
+the app serves, not just that files were written.

packages/ai-autopilot/src/bootstrap/index.ts

@@ -32,6 +32,7 @@ export {
   type AgentDeployOptions,
   type FakeDeployTargetOptions,
 } from './deploy.js'
+export { serveCheck, mergeChecklists, type ServeCheckOptions } from './serve-check.js'
 export type {
   BootstrapScope,
   BootstrapPhase,

packages/ai-autopilot/src/bootstrap/serve-check.test.ts

@@ -0,0 +1,98 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:net'
+import { serveCheck, mergeChecklists } from './serve-check.js'
+import { LocalRunner } from '../runner/local.js'
+import { FakeRunner } from '../runner/fake.js'
+import type { LoopPassContext } from './types.js'
+import type { Verdict } from '../loop/verdict.js'
+
+/** A minimal loop-pass context — serveCheck ignores it, but the contract needs one. */
+const ctx: LoopPassContext = { pass: 1, plan: { stack: '', narration: '', decisions: [] }, intent: '', blockers: [] }
+
+function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const srv = createServer()
+    srv.on('error', reject)
+    srv.listen(0, '127.0.0.1', () => {
+      const addr = srv.address()
+      const port = typeof addr === 'object' && addr ? addr.port : 0
+      srv.close(() => resolve(port))
+    })
+  })
+}
+
+const server = (port: number, status: number, body: string): string =>
+  `const http=require('http');http.createServer((_,res)=>{res.writeHead(${status});res.end(${JSON.stringify(body)})}).listen(${port},'127.0.0.1')`
+
+describe('serveCheck (boots and serves the app)', () => {
+  it('passes with no blockers when the app boots and serves', async () => {
+    const port = await freePort()
+    const s = await new LocalRunner().boot({ files: { 'server.js': server(port, 200, 'ok') } })
+    try {
+      const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 5000 })(ctx)
+      assert.deepEqual(verdict.blockers, [])
+    } finally {
+      await s.dispose()
+    }
+  })
+
+  it('blocks when the app serves a server error', async () => {
+    const port = await freePort()
+    const s = await new LocalRunner().boot({ files: { 'server.js': server(port, 500, 'boom') } })
+    try {
+      const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 5000 })(ctx)
+      assert.equal(verdict.blockers.length, 1)
+      assert.match(verdict.blockers[0]!, /responded 500/)
+    } finally {
+      await s.dispose()
+    }
+  })
+
+  it('blocks when the dev server exits before serving', async () => {
+    const port = await freePort()
+    const s = await new LocalRunner().boot({ files: { 'server.js': 'process.exit(1)' } })
+    try {
+      const verdict = await serveCheck(s, { serve: 'node server.js', port, waitMs: 1000 })(ctx)
+      assert.equal(verdict.blockers.length, 1)
+      assert.match(verdict.blockers[0]!, /exited before serving/)
+    } finally {
+      await s.dispose()
+    }
+  })
+
+  it('blocks on a failed install and never starts the server', async () => {
+    const runner = new FakeRunner({
+      onExec: cmd => (cmd.includes('install') ? { stdout: '', stderr: 'ERESOLVE', exitCode: 1 } : { stdout: '', stderr: '', exitCode: 0 }),
+    })
+    const s = await runner.boot()
+    const verdict = await serveCheck(s, { install: 'npm install', serve: 'npm run dev' })(ctx)
+    assert.equal(verdict.blockers.length, 1)
+    assert.match(verdict.blockers[0]!, /install failed/)
+    assert.equal(s.startCalls.length, 0) // never got as far as starting
+  })
+
+  it('skips (passing, with a note) when the runner cannot serve a preview', async () => {
+    const s = await new FakeRunner({ background: false }).boot()
+    const verdict = await serveCheck(s, { serve: 'npm run dev' })(ctx)
+    assert.deepEqual(verdict.blockers, [])
+    assert.match(verdict.notes!, /skipped/)
+  })
+})
+
+describe('mergeChecklists', () => {
+  it('unions and dedupes the blockers of every check', async () => {
+    const a = async (): Promise<Verdict> => ({ blockers: ['no auth', 'slow query'] })
+    const b = async (): Promise<Verdict> => ({ blockers: ['slow query', 'missing tests'], notes: 'serve ok' })
+    const verdict = await mergeChecklists(a, b)(ctx)
+    assert.deepEqual([...verdict.blockers].sort(), ['missing tests', 'no auth', 'slow query'])
+    assert.equal(verdict.notes, 'serve ok')
+  })
+
+  it('passes only when every check is clean', async () => {
+    const clean = async (): Promise<Verdict> => ({ blockers: [] })
+    const dirty = async (): Promise<Verdict> => ({ blockers: ['boom'] })
+    assert.deepEqual((await mergeChecklists(clean, clean)(ctx)).blockers, [])
+    assert.deepEqual((await mergeChecklists(clean, dirty)(ctx)).blockers, ['boom'])
+  })
+})

packages/ai-autopilot/src/bootstrap/serve-check.ts

@@ -0,0 +1,116 @@
+import type { RunnerSession } from '../runner/types.js'
+import type { Verdict } from '../loop/verdict.js'
+import type { BootstrapSteps, LoopPassContext } from './types.js'
+
+/**
+ * A production-grade check with teeth: instead of only *asking* whether the app
+ * is production-grade (the prompt-based {@link loopChecklist}), this one actually
+ * BOOTS it and confirms it serves. It runs inside the same runner session the
+ * build wrote into — install deps, optionally build, `start` the dev server,
+ * `preview` until the port is reachable, then fetch a health path — and turns any
+ * failure into a concrete `{ blockers }` verdict the full-fledged loop addresses.
+ *
+ * It satisfies the `checklist` step contract, so wire it directly
+ * (`checklist: serveCheck(session, { serve: 'npm run dev' })`) or, more usefully,
+ * compose it with the prompt checklist via {@link mergeChecklists} so a pass must
+ * BOTH read production-grade AND actually run.
+ *
+ * Needs a runner that can `start` background processes and `preview` them (the
+ * #137 seam). A runner without those capabilities can't verify, so the check is
+ * skipped (a passing verdict with a note) rather than blocking forever.
+ */
+export interface ServeCheckOptions {
+  /** The command that starts the dev server (e.g. `npm run dev`). Required. */
+  serve: string
+  /** Install command run first (e.g. `npm install`). Skipped when omitted. */
+  install?: string
+  /** Build command run after install (e.g. `npm run build`). Skipped when omitted. */
+  build?: string
+  /** The port the dev server listens on. Default 3000. */
+  port?: number
+  /** How long to wait for the server to accept connections. Default 15000ms. */
+  waitMs?: number
+  /** Path to fetch once the server is up. Default `/`. */
+  healthPath?: string
+  /** A response status at or above this is a blocker (the app errored). Default 500. */
+  errorStatusFrom?: number
+  /** Per-command timeout for install/build. Default 120000ms. */
+  commandTimeoutMs?: number
+  /** Optional progress narration (install / start / fetch). */
+  onProgress?: (message: string) => void
+}
+
+/**
+ * Build a `checklist` step that verifies the app boots and serves in `session`.
+ * Returns a {@link Verdict}: empty `blockers` means it installed, started, and
+ * responded without a server error; otherwise each blocker names what failed.
+ */
+export function serveCheck(session: RunnerSession, opts: ServeCheckOptions): NonNullable<BootstrapSteps['checklist']> {
+  const port = opts.port ?? 3000
+  const waitMs = opts.waitMs ?? 15_000
+  const healthPath = opts.healthPath ?? '/'
+  const errorFrom = opts.errorStatusFrom ?? 500
+  const commandTimeoutMs = opts.commandTimeoutMs ?? 120_000
+  const say = (m: string): void => opts.onProgress?.(m)
+
+  return async (_ctx: LoopPassContext): Promise<Verdict> => {
+    if (!session.start || !session.preview) {
+      return { blockers: [], notes: 'serve check skipped: runner cannot start/preview a dev server' }
+    }
+
+    // Install / build are prerequisites — if they fail, there is nothing to serve.
+    for (const [label, command] of [['install', opts.install], ['build', opts.build]] as const) {
+      if (!command) continue
+      say(`${label}: ${command}`)
+      const res = await session.exec(command, { timeoutMs: commandTimeoutMs })
+      if (res.exitCode !== 0) {
+        const tail = (res.stderr || res.stdout).trim().split('\n').slice(-3).join(' ').slice(0, 300)
+        return { blockers: [`${label} failed (\`${command}\` exited ${res.exitCode})${tail ? `: ${tail}` : ''}`] }
+      }
+    }
+
+    say(`start: ${opts.serve}`)
+    const proc = await session.start(opts.serve)
+    try {
+      const { url } = await session.preview({ port, waitMs })
+      const target = new URL(healthPath, url.endsWith('/') ? url : url + '/').toString()
+
+      // If the server crashed on boot, its process has already exited — say so.
+      const raced = await Promise.race([
+        fetch(target).then(res => ({ ok: true as const, status: res.status }), err => ({ ok: false as const, error: err as Error })),
+        proc.exit.then(r => ({ ok: false as const, exited: r.exitCode, log: (r.stderr || r.stdout).trim().split('\n').slice(-3).join(' ').slice(0, 300) })),
+      ])
+
+      if ('exited' in raced) {
+        return { blockers: [`the dev server (\`${opts.serve}\`) exited before serving (code ${raced.exited})${raced.log ? `: ${raced.log}` : ''}`] }
+      }
+      if (!raced.ok) {
+        return { blockers: [`the app did not serve at ${target}: ${raced.error.message}`] }
+      }
+      say(`fetch ${target} -> ${raced.status}`)
+      if (raced.status >= errorFrom) {
+        return { blockers: [`the app responded ${raced.status} at ${healthPath} (expected < ${errorFrom})`] }
+      }
+      return { blockers: [] }
+    } finally {
+      await proc.stop()
+    }
+  }
+}
+
+/**
+ * Compose several `checklist` steps into one: run them all and union their
+ * blockers (deduped). Use it to gate a pass on BOTH the prompt verdict and a real
+ * serve check — `mergeChecklists(loopChecklist({ loop }), serveCheck(session, ...))`.
+ * A pass is production-grade only when every check comes back clean.
+ */
+export function mergeChecklists(
+  ...checks: ReadonlyArray<NonNullable<BootstrapSteps['checklist']>>
+): NonNullable<BootstrapSteps['checklist']> {
+  return async (ctx: LoopPassContext): Promise<Verdict> => {
+    const verdicts = await Promise.all(checks.map(check => check(ctx)))
+    const blockers = [...new Set(verdicts.flatMap(v => v.blockers))]
+    const notes = verdicts.map(v => v.notes).filter(Boolean).join(' | ')
+    return { blockers, ...(notes ? { notes } : {}) }
+  }
+}

packages/ai-autopilot/src/index.ts

@@ -221,6 +221,9 @@ export {
   planOnlyTarget,
   FakeDeployTarget,
   DEFAULT_DEPLOY_TARGETS,
+  serveCheck,
+  mergeChecklists,
+  type ServeCheckOptions,
   type ArchitectAgentOptions,
   type SupervisorBuildOptions,
   type LoopStepOptions,