From 5a261fef7be62bbc9bb7a5ab4982c7b31ef39411 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Sun, 5 Jul 2026 21:11:04 +0300 Subject: [PATCH 1/2] feat(framework): AI meta-select for the domain preset + modes + build kind (#270) On a live run with no --preset (and none in the-framework.yml), infer the best-fit Open Loop domain preset, its modes, and the build event kind from the intent + workspace, then run under it. --no-auto-preset opts out; --fake stays deterministic; a failed or empty pick falls back to the plain flow. New meta-select module (router prompt + validated parse) plus a CLI seam that feeds the pick into the same resolve path a flag would. Closes #270. --- .changeset/ai-meta-select.md | 7 + packages/framework/src/cli.test.ts | 52 +++++++ packages/framework/src/cli.ts | 151 +++++++++++++++---- packages/framework/src/index.ts | 12 +- packages/framework/src/meta-select.test.ts | 112 ++++++++++++++ packages/framework/src/meta-select.ts | 161 +++++++++++++++++++++ 6 files changed, 464 insertions(+), 31 deletions(-) create mode 100644 .changeset/ai-meta-select.md create mode 100644 packages/framework/src/meta-select.test.ts create mode 100644 packages/framework/src/meta-select.ts diff --git a/.changeset/ai-meta-select.md b/.changeset/ai-meta-select.md new file mode 100644 index 0000000..d41494a --- /dev/null +++ b/.changeset/ai-meta-select.md @@ -0,0 +1,7 @@ +--- +'@gemstack/framework': minor +--- + +feat(framework): AI meta-select — auto-pick the Open Loop domain preset, modes, and build event kind from the prompt + workspace (#270) + +A live run with no `--preset` (and none in `the-framework.yml`) now infers the best-fit domain preset, its modes (technical / autopilot), and the build event kind from what you asked for and the project you are in, then runs under it. So `framework "add a login page"` in a web app picks Web Development on its own. `--no-auto-preset` opts out (plain framework flow); `--fake` stays deterministic. Any failed or empty pick falls back to the plain flow, so the auto-pick never blocks a run. diff --git a/packages/framework/src/cli.test.ts b/packages/framework/src/cli.test.ts index 7368eca..243a99c 100644 --- a/packages/framework/src/cli.test.ts +++ b/packages/framework/src/cli.test.ts @@ -1,7 +1,11 @@ import { strict as assert } from 'node:assert' import { test } from 'node:test' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { activeModes, + autoSelectPreset, buildDeployTarget, chooseSessionLink, claudeDriverOptions, @@ -10,8 +14,10 @@ import { parseArgs, resolveDomainPreset, runCli, + workspaceSummary, type CliIO, } from './cli.js' +import { FakeDriver } from './driver/index.js' function capture(): { io: CliIO; out: string[]; err: string[] } { const out: string[] = [] @@ -77,6 +83,52 @@ test('parseArgs reads --kind as the build event (#265)', () => { assert.equal(parseArgs(['x']).buildEvent, undefined) }) +test('parseArgs defaults autoPreset on, and --no-auto-preset turns it off (#204)', () => { + assert.equal(parseArgs(['x']).autoPreset, true) + assert.equal(parseArgs(['--no-auto-preset', 'x']).autoPreset, false) +}) + +test('workspaceSummary describes an empty vs an existing workspace', async () => { + const empty = await mkdtemp(join(tmpdir(), 'framework-ws-')) + try { + assert.match(workspaceSummary(empty, {}), /empty/) + await writeFile(join(empty, 'index.ts'), 'export const x = 1\n') + const summary = workspaceSummary(empty, { dependencies: { react: '^19', vite: '^5' } }) + assert.match(summary, /existing project/) + assert.match(summary, /react/) + assert.match(summary, /vite/) + } finally { + await rm(empty, { recursive: true, force: true }) + } +}) + +test('autoSelectPreset routes the pick through an injected driver and returns the selection (#204)', async () => { + const empty = await mkdtemp(join(tmpdir(), 'framework-ws-')) + try { + const driver = new FakeDriver({ + turns: [{ text: '```json\n{ "preset": "software-development", "modes": ["technical"], "event": "bug-fix", "why": "a fix" }\n```' }], + }) + const { io } = capture() + const selection = await autoSelectPreset({ intent: 'fix a crash', cwd: empty, signals: {}, claudeOpts: {}, io, driver }) + assert.deepEqual(selection, { preset: 'software-development', modes: ['technical'], buildEvent: 'bug-fix', why: 'a fix' }) + } finally { + await rm(empty, { recursive: true, force: true }) + } +}) + +test('autoSelectPreset degrades to undefined when the driver errors', async () => { + const empty = await mkdtemp(join(tmpdir(), 'framework-ws-')) + try { + const driver = new FakeDriver({ respond: () => { throw new Error('agent unavailable') } }) + const { io, err } = capture() + const selection = await autoSelectPreset({ intent: 'anything', cwd: empty, signals: {}, claudeOpts: {}, io, driver }) + assert.equal(selection, undefined) + assert.ok(err.some(l => /auto-select skipped/.test(l))) + } finally { + await rm(empty, { recursive: true, force: true }) + } +}) + test('activeModes maps the mode flags to Open Loop mode names', () => { assert.deepEqual(activeModes({ autopilot: false, technical: false }), []) assert.deepEqual(activeModes({ autopilot: true, technical: false }), ['autopilot']) diff --git a/packages/framework/src/cli.ts b/packages/framework/src/cli.ts index e197560..c31c8dd 100644 --- a/packages/framework/src/cli.ts +++ b/packages/framework/src/cli.ts @@ -9,8 +9,9 @@ import { selectPreset, type DeployTarget, type DomainPreset, + type FrameworkSignals, } from '@gemstack/ai-autopilot' -import { ClaudeCodeDriver, type ClaudeCodeDriverOptions, type Driver, type PermissionMode } from './driver/index.js' +import { ClaudeCodeDriver, type ClaudeCodeDriverOptions, type Driver, type DriverSession, type PermissionMode } from './driver/index.js' import { hostExecutor } from './host-exec.js' import { startDashboard, type Dashboard } from './dashboard/index.js' import { formatFrameworkEvent, CLAUDE_CODE_SESSION_LINK, type FrameworkEvent } from './events.js' @@ -23,6 +24,13 @@ import { } from './run.js' import { FAKE_DEPLOY, FAKE_INTENT, FAKE_SIGNALS, fakeDriver } from './fake-script.js' import { discoverExtensions, readProjectSignals } from './extensions.js' +import { + META_SELECT_SYSTEM, + metaSelect, + presetCatalog, + type MetaSelection, +} from './meta-select.js' +import { isWorkspaceEmpty } from './steps.js' import { loadFrameworkConfig, type FrameworkFileConfig } from './config.js' import { loadRepoMemory } from './memory.js' import { preflight } from './preflight.js' @@ -74,6 +82,10 @@ Options: --scope How much app to build (default: full). --preset Run under an Open Loop domain preset (its loops + prompts + skills frame the build), e.g. software-development. + Omit it and a live run auto-picks the best-fit preset + + modes from your prompt + workspace (--no-auto-preset off). + --no-auto-preset Do not auto-pick a preset; run the plain framework flow + unless --preset / the-framework.yml sets one. --autopilot Activate the preset's Autopilot mode variants. --technical Activate the preset's Technical mode variants. (--preset / --autopilot / --technical / --kind can also be @@ -133,6 +145,7 @@ export interface CliOptions { model?: string | undefined scope: 'prototype' | 'full' preset?: string | undefined + autoPreset: boolean autopilot: boolean technical: boolean buildEvent?: string | undefined @@ -167,6 +180,7 @@ export function parseArgs(argv: string[]): CliOptions { skipPreflight: false, intent: '', scope: 'full', + autoPreset: true, autopilot: false, technical: false, dashboard: true, @@ -200,6 +214,9 @@ export function parseArgs(argv: string[]): CliOptions { case '--preset': opts.preset = argv[++i] break + case '--no-auto-preset': + opts.autoPreset = false + break case '--autopilot': opts.autopilot = true break @@ -373,6 +390,57 @@ export async function resolveDomainPreset( return { preset } } +/** A one-line summary of the workspace for the meta-select router: empty vs existing + a few deps. */ +export function workspaceSummary(cwd: string, signals: FrameworkSignals): string { + if (isWorkspaceEmpty(cwd)) return 'empty (a from-scratch build)' + const deps = signals.dependencies + const names = Array.isArray(deps) ? deps : Object.keys(deps ?? {}) + if (names.length === 0) return 'an existing project' + const shown = names.slice(0, 12).join(', ') + return `an existing project (dependencies: ${shown}${names.length > 12 ? ', …' : ''})` +} + +/** + * AI meta-select (#204): infer the best-fit domain preset (+ modes + build event) + * from the intent + workspace when the user did not pick one. Spins up a + * short-lived driver session for a single routing prompt, then disposes it. Any + * failure (agent error, junk reply) degrades to `undefined` = the plain flow, so + * a run is never blocked by the auto-pick. + */ +export async function autoSelectPreset(opts: { + intent: string + cwd: string + signals: FrameworkSignals + claudeOpts: ClaudeCodeDriverOptions + signal?: AbortSignal + io: CliIO + /** The driver to route the pick through. Defaults to a Claude Code driver; injected in tests. */ + driver?: Driver +}): Promise { + const catalog = presetCatalog(await builtinDomainPresets()) + if (catalog.length === 0) return undefined + const driver = opts.driver ?? new ClaudeCodeDriver(opts.claudeOpts) + let session: DriverSession | undefined + try { + session = await driver.start({ + cwd: opts.cwd, + system: META_SELECT_SYSTEM, + ...(opts.signal ? { signal: opts.signal } : {}), + }) + return await metaSelect(session, { + intent: opts.intent, + catalog, + workspace: workspaceSummary(opts.cwd, opts.signals), + ...(opts.signal ? { signal: opts.signal } : {}), + }) + } catch (err) { + opts.io.err(`auto-select skipped (${err instanceof Error ? err.message : String(err)}); using the plain framework flow.`) + return undefined + } finally { + await session?.dispose() + } +} + /** * The `framework` command. Wires the parsed options into {@link runFramework} * over a live dashboard + terminal narration, and resolves with an exit code. @@ -422,25 +490,8 @@ export async function runCli(argv: string[], io: CliIO = defaultIO): Promise { const link = chooseSessionLink(opts, fake) diff --git a/packages/framework/src/index.ts b/packages/framework/src/index.ts index af31fd7..78de5db 100644 --- a/packages/framework/src/index.ts +++ b/packages/framework/src/index.ts @@ -85,7 +85,17 @@ export { type RunStatus, type OpenStoreOptions, } from './store/index.js' -export { runCli, parseArgs, buildDeployTarget, type CliIO, type CliOptions } from './cli.js' +export { runCli, parseArgs, buildDeployTarget, workspaceSummary, autoSelectPreset, type CliIO, type CliOptions } from './cli.js' +export { + metaSelect, + metaSelectPrompt, + parseMetaSelection, + presetCatalog, + META_SELECT_MODES, + META_SELECT_SYSTEM, + type MetaSelection, + type PresetCatalogEntry, +} from './meta-select.js' export { loadFrameworkConfig, parseFrameworkConfig, diff --git a/packages/framework/src/meta-select.test.ts b/packages/framework/src/meta-select.test.ts new file mode 100644 index 0000000..e8548b4 --- /dev/null +++ b/packages/framework/src/meta-select.test.ts @@ -0,0 +1,112 @@ +import { strict as assert } from 'node:assert' +import { test } from 'node:test' +import type { DomainPreset } from '@gemstack/ai-autopilot' +import { FakeDriver } from './driver/index.js' +import { + META_SELECT_MODES, + metaSelect, + metaSelectPrompt, + parseMetaSelection, + presetCatalog, + type PresetCatalogEntry, +} from './meta-select.js' + +/** Two tiny presets whose loops declare distinct build kinds, for catalog + validation tests. */ +function fakePresets(): DomainPreset[] { + const loop = (on: string[]) => ({ on, run: [] as string[] }) as DomainPreset['loops'][number] + return [ + { + name: 'web-development', + title: 'Web Development', + description: 'Building web apps.', + defaultEvent: 'major-change', + loops: [loop(['major-change']), loop(['bug-fix'])], + prompts: [], + skills: [], + }, + { + name: 'data-science', + title: 'Data Science', + description: 'Notebooks and models.', + loops: [loop(['major-change'])], + prompts: [], + skills: [], + }, + ] +} + +const CATALOG = (): PresetCatalogEntry[] => presetCatalog(fakePresets()) + +test('presetCatalog derives name/title/description + deduped event kinds + defaultEvent', () => { + assert.deepEqual(presetCatalog(fakePresets()), [ + { + name: 'web-development', + title: 'Web Development', + description: 'Building web apps.', + eventKinds: ['major-change', 'bug-fix'], + defaultEvent: 'major-change', + }, + { + name: 'data-science', + title: 'Data Science', + description: 'Notebooks and models.', + eventKinds: ['major-change'], + }, + ]) +}) + +test('parseMetaSelection accepts a valid preset + modes + event + why', () => { + const text = '```json\n{ "preset": "web-development", "modes": ["technical"], "event": "bug-fix", "why": "fixing a defect" }\n```' + assert.deepEqual(parseMetaSelection(text, CATALOG()), { + preset: 'web-development', + modes: ['technical'], + buildEvent: 'bug-fix', + why: 'fixing a defect', + }) +}) + +test('parseMetaSelection drops an unknown preset (plain flow), keeping only why', () => { + const text = '{ "preset": "biology", "modes": ["technical"], "event": "bug-fix", "why": "no fit" }' + assert.deepEqual(parseMetaSelection(text, CATALOG()), { modes: [], why: 'no fit' }) +}) + +test('parseMetaSelection filters unknown modes and drops an event the preset has no loop for', () => { + // data-science only has a major-change loop; "bug-fix" and mode "wizard" are not valid there. + const text = '{ "preset": "data-science", "modes": ["wizard", "autopilot"], "event": "bug-fix" }' + assert.deepEqual(parseMetaSelection(text, CATALOG()), { preset: 'data-science', modes: ['autopilot'] }) +}) + +test('parseMetaSelection returns the plain-flow selection for junk text', () => { + assert.deepEqual(parseMetaSelection('I could not decide.', CATALOG()), { modes: [] }) +}) + +test('parseMetaSelection knows exactly the two shipped modes', () => { + assert.deepEqual([...META_SELECT_MODES], ['autopilot', 'technical']) +}) + +test('metaSelectPrompt names the task, the workspace, and every preset', () => { + const prompt = metaSelectPrompt('add a login page', CATALOG(), 'an existing project (dependencies: react)') + assert.match(prompt, /add a login page/) + assert.match(prompt, /an existing project \(dependencies: react\)/) + assert.match(prompt, /web-development/) + assert.match(prompt, /data-science/) +}) + +test('metaSelect routes one prompt through the driver and parses the reply', async () => { + const driver = new FakeDriver({ + turns: [{ text: '```json\n{ "preset": "web-development", "modes": ["technical"], "event": "major-change", "why": "a web feature" }\n```' }], + }) + const session = await driver.start({ cwd: '/tmp/x', system: 'router' }) + const selection = await metaSelect(session, { + intent: 'add a settings page', + catalog: CATALOG(), + workspace: 'an existing project', + }) + await session.dispose() + assert.deepEqual(selection, { + preset: 'web-development', + modes: ['technical'], + buildEvent: 'major-change', + why: 'a web feature', + }) +}) diff --git a/packages/framework/src/meta-select.ts b/packages/framework/src/meta-select.ts new file mode 100644 index 0000000..1effeee --- /dev/null +++ b/packages/framework/src/meta-select.ts @@ -0,0 +1,161 @@ +import type { DomainPreset } from '@gemstack/ai-autopilot' +import type { DriverSession } from './driver/index.js' + +/** + * AI meta-select (#204, Rom's "meta-meta prompt"): before a run, infer which Open + * Loop domain preset (plus its modes and build event kind) best fits what the user + * asked for and the workspace they are in — so `framework "add a login page"` in a + * web app picks the Web Development preset on its own, with no `--preset` flag or + * `the-framework.yml`. It is a *selection* over the shipped catalog, not a new + * kind of thing: it returns the same inputs a user would have typed, then the run + * proceeds exactly as if they had. + * + * Kept out of `runFramework`: the CLI already owns preset discovery + resolution + * (flags, the-framework.yml), and this is one more source that feeds the same + * `resolveDomainPreset` path. Live only — `--fake` stays deterministic. + */ + +/** The Open Loop modes a run can activate. Mirrors the CLI's `--autopilot` / `--technical`. */ +export const META_SELECT_MODES = ['autopilot', 'technical'] as const + +/** The system framing for the meta-select turn: a tiny, fast classifier, not a builder. */ +export const META_SELECT_SYSTEM = + 'You are a router that picks the best-fit build policy for a coding task. ' + + 'You do not write or run any code — you only choose from the given options and reply with one JSON object.' + +/** What the model needs to know about one shipped preset to choose it. */ +export interface PresetCatalogEntry { + name: string + title: string + description: string + /** The build event kinds this preset actually has a review loop for (e.g. `major-change`, `bug-fix`). */ + eventKinds: string[] + /** The kind dispatched when the run does not pick one; absent means `major-change`. */ + defaultEvent?: string +} + +/** The inferred selection: the same inputs `--preset` / `--autopilot` / `--kind` would have supplied. */ +export interface MetaSelection { + /** The chosen preset name, validated against the catalog. Absent = no preset fits; run the plain flow. */ + preset?: string + /** The chosen modes, a validated subset of {@link META_SELECT_MODES}. */ + modes: string[] + /** The chosen build event kind, validated against the chosen preset's `eventKinds`. Absent = the preset default. */ + buildEvent?: string + /** One-line rationale, shown to the user so the auto-pick is legible. */ + why?: string +} + +/** Derive the {@link PresetCatalogEntry} list the router chooses from. */ +export function presetCatalog(presets: readonly DomainPreset[]): PresetCatalogEntry[] { + return presets.map(p => { + const eventKinds = [...new Set(p.loops.flatMap(l => [...l.on]))] + return { + name: p.name, + title: p.title, + description: p.description, + eventKinds, + ...(p.defaultEvent ? { defaultEvent: p.defaultEvent } : {}), + } + }) +} + +/** + * Compose the router prompt: the task, a one-line summary of the workspace, and + * the catalog of presets + modes to choose from. Asks for exactly one JSON object. + */ +export function metaSelectPrompt(intent: string, catalog: readonly PresetCatalogEntry[], workspace: string): string { + const presetLines = catalog.map( + p => `- ${p.name}: ${p.title} — ${p.description} (build kinds: ${p.eventKinds.join(', ') || 'none'})`, + ) + return [ + 'Pick the build policy that best fits this coding task.', + `Task: ${intent}`, + `Workspace: ${workspace}`, + '', + 'Available domain presets (a preset frames the review loop for the build):', + ...presetLines, + '', + 'Modes (optional, activate any that apply):', + '- technical: the user is technical; keep the review lean and skip hand-holding.', + '- autopilot: run with minimal check-ins.', + '', + 'Choose the single best preset, or "none" if none clearly fits (then the plain framework flow runs).', + 'If you pick a preset, choose its build kind from that preset\'s list: "bug-fix" when the task is fixing', + 'a defect, otherwise "major-change" (or leave it out to use the preset default).', + '', + 'Respond with ONLY a fenced ```json block of the shape:', + '{ "preset": "", "modes": ["technical"?, "autopilot"?], "event": "", "why": "" }', + ].join('\n') +} + +/** + * Parse and validate the router's reply into a {@link MetaSelection}. Everything is + * validated against what actually ships: an unknown preset name, mode, or event + * kind is dropped rather than trusted, so a loose reply can only ever *narrow* to a + * safe selection (worst case: no preset, i.e. the plain flow). Never throws. + */ +export function parseMetaSelection(text: string, catalog: readonly PresetCatalogEntry[]): MetaSelection { + const obj = lastJsonObject(text) + const rawPreset = typeof obj?.['preset'] === 'string' ? obj['preset'].trim() : '' + const entry = catalog.find(p => p.name === rawPreset) + const modes = coerceStrings(obj?.['modes']).filter((m): m is string => + (META_SELECT_MODES as readonly string[]).includes(m), + ) + const why = typeof obj?.['why'] === 'string' && obj['why'].trim() ? obj['why'].trim() : undefined + // No matching preset: the modes/event have nothing to act on, so return the + // plain-flow selection (only `why`, if the model explained itself). + if (!entry) return { modes: [], ...(why ? { why } : {}) } + const rawEvent = typeof obj?.['event'] === 'string' ? obj['event'].trim() : '' + const buildEvent = entry.eventKinds.includes(rawEvent) ? rawEvent : undefined + return { + preset: entry.name, + modes, + ...(buildEvent ? { buildEvent } : {}), + ...(why ? { why } : {}), + } +} + +/** + * Run the meta-select turn against a live driver session and parse its reply. + * One fresh prompt, no tools — a fast classification the wrapped agent answers as + * JSON. The caller supplies a short-lived session (dispose it after); on any error + * the caller falls back to the plain flow. + */ +export async function metaSelect( + session: DriverSession, + opts: { intent: string; catalog: readonly PresetCatalogEntry[]; workspace: string; signal?: AbortSignal }, +): Promise { + const turn = await session.prompt(metaSelectPrompt(opts.intent, opts.catalog, opts.workspace), { + ...(opts.signal ? { signal: opts.signal } : {}), + }) + return parseMetaSelection(turn.text, opts.catalog) +} + +function coerceStrings(value: unknown): string[] { + if (!Array.isArray(value)) return [] + return value.filter((v): v is string => typeof v === 'string' && v.trim() !== '').map(v => v.trim()) +} + +const FENCE = /```(?:[a-zA-Z0-9]*)\n([\s\S]*?)```/g + +/** Extract the last JSON object from text: last fenced block, else a trailing `{...}`. */ +function lastJsonObject(text: string): Record | undefined { + if (!text) return undefined + const candidates: string[] = [] + for (const m of text.matchAll(FENCE)) candidates.push(m[1]!) + const open = text.lastIndexOf('{') + const close = text.lastIndexOf('}') + if (open !== -1 && close > open) candidates.push(text.slice(open, close + 1)) + for (let i = candidates.length - 1; i >= 0; i--) { + try { + const parsed = JSON.parse(candidates[i]!.trim()) + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) { + return parsed as Record + } + } catch { + // try the next candidate + } + } + return undefined +} From 7dc76c5f0c10454db9c38a3142d1cef7ef7e7b33 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Sun, 5 Jul 2026 21:15:54 +0300 Subject: [PATCH 2/2] fix(framework): validate an explicit --preset before preflight A bad --preset name is a usage error independent of the environment, so it must fail the same way whether or not the agent is installed. Moving preflight ahead of preset resolution made a bad --preset report a preflight failure on a machine without Claude Code (e.g. CI) instead of the unknown-preset usage error. Resolve an explicit preset up front; only auto-select runs after preflight. --- packages/framework/src/cli.ts | 47 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/packages/framework/src/cli.ts b/packages/framework/src/cli.ts index c31c8dd..82f9cfe 100644 --- a/packages/framework/src/cli.ts +++ b/packages/framework/src/cli.ts @@ -490,8 +490,31 @@ export async function runCli(argv: string[], io: CliIO = defaultIO): Promise