diff --git a/benchmarks/runs/2026-06-28-task-001-tags/results.md b/benchmarks/runs/2026-06-28-task-001-tags/results.md new file mode 100644 index 0000000..0cae81c --- /dev/null +++ b/benchmarks/runs/2026-06-28-task-001-tags/results.md @@ -0,0 +1,48 @@ +# Phase 0 run — task-001-tags + +Date: 2026-06-28 +Task: [`spec/task-001-tags.md`](../../spec/task-001-tags.md) — add tagging to notes (data model + HTTP contract + UI). +Gate: [`tasks/task-001-tags/accept.mjs`](../../tasks/task-001-tags/accept.mjs), objective contract check, exit 0 = pass. + +This is the first end-to-end Phase 0 run. Its purpose (per [#78](https://github.com/gemstack-land/gemstack/issues/78)) is to prove the method and the rubric are runnable, not to draw a final conclusion. + +## Method + +- One autonomous AI coding agent per app, same agent type and model on both sides (fair-harness rule). Each was handed the same task spec verbatim and run fully autonomously — no human in the loop. +- Each agent reset its DB for a clean seed, started its own dev server on the app's fixed port, and iterated against `accept.mjs` until exit 0 or a 30-minute / 5-intervention cap. +- Wall clock was stamped by each agent with `date +%s` around its own run. +- **Harness validated first:** both committed baselines were run against the gate before any code was written — both failed identically (the same 5 checks: `?tag` filtering and the `tags` array), confirming a real, symmetric gap to measure. +- **Independently re-verified:** after both agents reported PASS, the gate was re-run by hand against fresh DBs + freshly restarted servers on both apps. Both passed (exit 0), reproducible. + +## Results + +| App | Status | Time | Interventions | Notes | +|---|---|---|---|---| +| `bench-app-next` (vanilla Next.js) | PASS | 139s | 0 | first-run green, no server restart needed | +| `bench-app-gemstack` (Vike + React + `@gemstack/ai-sdk`) | PASS | 145s | 0 | first-run green after one server restart (server-code change) | + +Both passed all 14 contract checks. Diffs were comparable in size (Next: 6 files / ~130 lines; GemStack: 5 files / ~150 lines). The agents' full solutions are saved as patches under [`solutions/`](./solutions) for audit and reproduction; they are intentionally **not** applied to the committed baseline apps, which stay the pristine Phase 0 starting point. + +## Reading + +The rubric runs end to end and the gate is objective and reproducible — Phase 0's goal is met. + +The signal this task produces is "both frameworks build a basic CRUD-extension feature in roughly equal time with zero interventions." That is expected: a plain tags feature does not exercise the orchestration layer the benchmark exists to measure. The intervention metric only differentiates on tasks where the layer is in reach — AI-integration, multi-step, and refactor tasks where an agent on the vanilla side has to be unblocked. Phase 1 ([#79](https://github.com/gemstack-land/gemstack/issues/79)) should weight the task set toward those. + +## Reproduce + +```bash +# from the repo root +pnpm install +pnpm --filter @gemstack/ai-sdk build + +# baseline should FAIL the gate (proves the gap) +cd examples/bench-app-next && rm -f data/notes.db* && pnpm_config_verify_deps_before_run=false pnpm dev # :4311 +cd examples/bench-app-gemstack && rm -f data/bench.sqlite* && PORT=3100 pnpm dev # :3100 +BASE_URL=http://localhost:4311 node benchmarks/tasks/task-001-tags/accept.mjs # FAIL on baseline +BASE_URL=http://localhost:3100 node benchmarks/tasks/task-001-tags/accept.mjs # FAIL on baseline + +# apply an agent's solution to re-check a PASS +git apply benchmarks/runs/2026-06-28-task-001-tags/solutions/next.patch +git apply benchmarks/runs/2026-06-28-task-001-tags/solutions/gemstack.patch +``` diff --git a/benchmarks/runs/2026-06-28-task-001-tags/solutions/gemstack.patch b/benchmarks/runs/2026-06-28-task-001-tags/solutions/gemstack.patch new file mode 100644 index 0000000..a9110c5 --- /dev/null +++ b/benchmarks/runs/2026-06-28-task-001-tags/solutions/gemstack.patch @@ -0,0 +1,307 @@ +diff --git a/examples/bench-app-gemstack/pages/api.ts b/examples/bench-app-gemstack/pages/api.ts +index e782e91..bfbed59 100644 +--- a/examples/bench-app-gemstack/pages/api.ts ++++ b/examples/bench-app-gemstack/pages/api.ts +@@ -5,6 +5,7 @@ export interface Note { + body: string + summary: string | null + createdAt: string ++ tags: string[] + } + + async function json(res: Response): Promise { +@@ -24,15 +25,16 @@ export const api = { + async logout(): Promise { + await fetch('/api/logout', { method: 'POST' }) + }, +- async list(): Promise { +- const { notes } = await json<{ notes: Note[] }>(await fetch('/api/notes')) ++ async list(tag?: string): Promise { ++ const url = tag ? `/api/notes?tag=${encodeURIComponent(tag)}` : '/api/notes' ++ const { notes } = await json<{ notes: Note[] }>(await fetch(url)) + return notes + }, +- async create(title: string, body: string): Promise { ++ async create(title: string, body: string, tags: string[] = []): Promise { + const res = await fetch('/api/notes', { + method: 'POST', + headers: { 'content-type': 'application/json' }, +- body: JSON.stringify({ title, body }), ++ body: JSON.stringify({ title, body, tags }), + }) + return (await json<{ note: Note }>(res)).note + }, +diff --git a/examples/bench-app-gemstack/pages/index/+Page.tsx b/examples/bench-app-gemstack/pages/index/+Page.tsx +index 089f9ac..1ce76ff 100644 +--- a/examples/bench-app-gemstack/pages/index/+Page.tsx ++++ b/examples/bench-app-gemstack/pages/index/+Page.tsx +@@ -5,11 +5,20 @@ export default function NotesPage() { + const [notes, setNotes] = useState([]) + const [title, setTitle] = useState('') + const [body, setBody] = useState('') ++ const [tags, setTags] = useState('') ++ const [filter, setFilter] = useState(null) + const [loading, setLoading] = useState(true) + +- async function refresh() { ++ function parseTagsInput(raw: string): string[] { ++ return raw ++ .split(',') ++ .map((t) => t.trim()) ++ .filter(Boolean) ++ } ++ ++ async function refresh(tag: string | null = filter) { + try { +- setNotes(await api.list()) ++ setNotes(await api.list(tag ?? undefined)) + } catch { + window.location.href = '/login' + } finally { +@@ -18,15 +27,22 @@ export default function NotesPage() { + } + + useEffect(() => { +- void refresh() ++ void refresh(null) + }, []) + ++ async function applyFilter(tag: string | null) { ++ setFilter(tag) ++ setLoading(true) ++ await refresh(tag) ++ } ++ + async function onCreate(e: React.FormEvent) { + e.preventDefault() + if (!title.trim() || !body.trim()) return +- await api.create(title, body) ++ await api.create(title, body, parseTagsInput(tags)) + setTitle('') + setBody('') ++ setTags('') + await refresh() + } + +@@ -66,9 +82,22 @@ export default function NotesPage() { + rows={3} + style={{ display: 'block', width: '100%', marginBottom: 8 }} + /> ++ setTags(e.target.value)} ++ style={{ display: 'block', width: '100%', marginBottom: 8 }} ++ /> + + + ++ {filter && ( ++

++ Filtering by #{filter}{' '} ++ ++

++ )} ++ + {loading ? ( +

Loading…

+ ) : notes.length === 0 ? ( +@@ -81,6 +110,26 @@ export default function NotesPage() { + {note.title} + +

{note.body}

++ {note.tags.length > 0 && ( ++
++ {note.tags.map((tag) => ( ++ ++ ))} ++
++ )} + {note.summary && ( +

+ Summary: {note.summary} +diff --git a/examples/bench-app-gemstack/pages/note/@id/+Page.tsx b/examples/bench-app-gemstack/pages/note/@id/+Page.tsx +index 5762453..b59aa27 100644 +--- a/examples/bench-app-gemstack/pages/note/@id/+Page.tsx ++++ b/examples/bench-app-gemstack/pages/note/@id/+Page.tsx +@@ -35,6 +35,27 @@ export default function NoteDetailPage() { +

+

{note.title}

+

{new Date(note.createdAt).toLocaleString()}

++ {note.tags.length > 0 && ( ++
++ {note.tags.map((tag) => ( ++ ++ #{tag} ++ ++ ))} ++
++ )} +

{note.body}

+ {note.summary ? ( +

+diff --git a/examples/bench-app-gemstack/server/api.ts b/examples/bench-app-gemstack/server/api.ts +index 126ad94..6398700 100644 +--- a/examples/bench-app-gemstack/server/api.ts ++++ b/examples/bench-app-gemstack/server/api.ts +@@ -1,6 +1,6 @@ + import { randomUUID } from 'node:crypto' + import type { Express, Request, Response, NextFunction } from 'express' +-import { createNote, deleteNote, findUser, getNote, listNotes, setSummary } from './db.js' ++import { createNote, deleteNote, findUser, getNote, listNotes, normalizeTags, setSummary } from './db.js' + import { summarize } from './ai.js' + + const COOKIE = 'session' +@@ -58,19 +58,21 @@ export function registerApi(app: Express): Express { + res.status(200).json({ ok: true }) + }) + +- // GET /api/notes +- app.get('/api/notes', requireAuth, (_req: Request, res: Response) => { +- res.status(200).json({ notes: listNotes() }) ++ // GET /api/notes (optional ?tag= filter) ++ app.get('/api/notes', requireAuth, (req: Request, res: Response) => { ++ const tagParam = req.query.tag ++ const tag = typeof tagParam === 'string' ? tagParam : undefined ++ res.status(200).json({ notes: listNotes(tag) }) + }) + + // POST /api/notes + app.post('/api/notes', requireAuth, (req: Request, res: Response) => { +- const { title, body } = (req.body ?? {}) as { title?: string; body?: string } ++ const { title, body, tags } = (req.body ?? {}) as { title?: string; body?: string; tags?: unknown } + if (typeof title !== 'string' || typeof body !== 'string') { + res.status(400).json({ error: 'title and body are required' }) + return + } +- res.status(201).json({ note: createNote(title, body) }) ++ res.status(201).json({ note: createNote(title, body, normalizeTags(tags)) }) + }) + + // GET /api/notes/:id +diff --git a/examples/bench-app-gemstack/server/db.ts b/examples/bench-app-gemstack/server/db.ts +index daff925..2f18f4f 100644 +--- a/examples/bench-app-gemstack/server/db.ts ++++ b/examples/bench-app-gemstack/server/db.ts +@@ -12,6 +12,7 @@ export interface NoteRow { + body: string + summary: string | null + created_at: string ++ tags: string | null + } + + /** Public `Note` shape from the HTTP contract. */ +@@ -21,6 +22,32 @@ export interface Note { + body: string + summary: string | null + createdAt: string ++ tags: string[] ++} ++ ++function parseTags(raw: string | null): string[] { ++ if (!raw) return [] ++ try { ++ const parsed = JSON.parse(raw) ++ return Array.isArray(parsed) ? parsed.filter((t): t is string => typeof t === 'string') : [] ++ } catch { ++ return [] ++ } ++} ++ ++/** Normalize an incoming tags value into a clean string[] (trimmed, non-empty, de-duped). */ ++export function normalizeTags(input: unknown): string[] { ++ if (!Array.isArray(input)) return [] ++ const seen = new Set() ++ const out: string[] = [] ++ for (const t of input) { ++ if (typeof t !== 'string') continue ++ const trimmed = t.trim() ++ if (!trimmed || seen.has(trimmed)) continue ++ seen.add(trimmed) ++ out.push(trimmed) ++ } ++ return out + } + + function toNote(row: NoteRow): Note { +@@ -30,6 +57,7 @@ function toNote(row: NoteRow): Note { + body: row.body, + summary: row.summary, + createdAt: row.created_at, ++ tags: parseTags(row.tags), + } + } + +@@ -54,10 +82,17 @@ export function db(): Database.Database { + title TEXT NOT NULL, + body TEXT NOT NULL, + summary TEXT, +- created_at TEXT NOT NULL ++ created_at TEXT NOT NULL, ++ tags TEXT NOT NULL DEFAULT '[]' + ); + `) + ++ // Additive migration for pre-existing databases that lack the tags column. ++ const cols = conn.prepare(`PRAGMA table_info(notes)`).all() as Array<{ name: string }> ++ if (!cols.some((c) => c.name === 'tags')) { ++ conn.exec(`ALTER TABLE notes ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'`) ++ } ++ + // Seed the single demo user on first boot. + const seeded = conn.prepare('SELECT 1 FROM users WHERE email = ?').get('demo@example.com') + if (!seeded) { +@@ -74,9 +109,11 @@ export function findUser(email: string, password: string): { id: number } | unde + .get(email, password) as { id: number } | undefined + } + +-export function listNotes(): Note[] { ++export function listNotes(tag?: string): Note[] { + const rows = db().prepare('SELECT * FROM notes ORDER BY id DESC').all() as NoteRow[] +- return rows.map(toNote) ++ const notes = rows.map(toNote) ++ if (tag === undefined) return notes ++ return notes.filter((n) => n.tags.includes(tag)) + } + + export function getNote(id: number): Note | undefined { +@@ -84,11 +121,11 @@ export function getNote(id: number): Note | undefined { + return row ? toNote(row) : undefined + } + +-export function createNote(title: string, body: string): Note { ++export function createNote(title: string, body: string, tags: string[] = []): Note { + const createdAt = new Date().toISOString() + const info = db() +- .prepare('INSERT INTO notes (title, body, summary, created_at) VALUES (?, ?, NULL, ?)') +- .run(title, body, createdAt) ++ .prepare('INSERT INTO notes (title, body, summary, created_at, tags) VALUES (?, ?, NULL, ?, ?)') ++ .run(title, body, createdAt, JSON.stringify(normalizeTags(tags))) + return getNote(Number(info.lastInsertRowid))! + } + diff --git a/benchmarks/runs/2026-06-28-task-001-tags/solutions/next.patch b/benchmarks/runs/2026-06-28-task-001-tags/solutions/next.patch new file mode 100644 index 0000000..2d7b2d5 --- /dev/null +++ b/benchmarks/runs/2026-06-28-task-001-tags/solutions/next.patch @@ -0,0 +1,279 @@ +diff --git a/examples/bench-app-next/app/api/notes/route.ts b/examples/bench-app-next/app/api/notes/route.ts +index fb97033..0746ec5 100644 +--- a/examples/bench-app-next/app/api/notes/route.ts ++++ b/examples/bench-app-next/app/api/notes/route.ts +@@ -1,12 +1,13 @@ + import { NextResponse } from 'next/server'; + import { isAuthenticated } from '@/lib/auth'; +-import { createNote, listNotes } from '@/lib/notes'; ++import { createNote, listNotes, normalizeTags } from '@/lib/notes'; + +-export async function GET() { ++export async function GET(req: Request) { + if (!(await isAuthenticated())) { + return NextResponse.json({ error: 'unauthorized' }, { status: 401 }); + } +- return NextResponse.json({ notes: listNotes() }); ++ const tag = new URL(req.url).searchParams.get('tag') ?? undefined; ++ return NextResponse.json({ notes: listNotes(tag || undefined) }); + } + + export async function POST(req: Request) { +@@ -16,10 +17,12 @@ export async function POST(req: Request) { + + let title = ''; + let body = ''; ++ let tags: string[] = []; + try { + const json = await req.json(); + title = typeof json?.title === 'string' ? json.title : ''; + body = typeof json?.body === 'string' ? json.body : ''; ++ tags = normalizeTags(json?.tags); + } catch { + // fall through to validation + } +@@ -28,5 +31,5 @@ export async function POST(req: Request) { + return NextResponse.json({ error: 'title and body are required' }, { status: 400 }); + } + +- return NextResponse.json({ note: createNote(title, body) }, { status: 201 }); ++ return NextResponse.json({ note: createNote(title, body, tags) }, { status: 201 }); + } +diff --git a/examples/bench-app-next/app/create-note-form.tsx b/examples/bench-app-next/app/create-note-form.tsx +index 51b7de8..29edf09 100644 +--- a/examples/bench-app-next/app/create-note-form.tsx ++++ b/examples/bench-app-next/app/create-note-form.tsx +@@ -7,17 +7,23 @@ export default function CreateNoteForm() { + const router = useRouter(); + const [title, setTitle] = useState(''); + const [body, setBody] = useState(''); ++ const [tags, setTags] = useState(''); + + async function onSubmit(e: React.FormEvent) { + e.preventDefault(); ++ const tagList = tags ++ .split(',') ++ .map((t) => t.trim()) ++ .filter(Boolean); + const res = await fetch('/api/notes', { + method: 'POST', + headers: { 'content-type': 'application/json' }, +- body: JSON.stringify({ title, body }), ++ body: JSON.stringify({ title, body, tags: tagList }), + }); + if (res.ok) { + setTitle(''); + setBody(''); ++ setTags(''); + router.refresh(); + } + } +@@ -37,6 +43,11 @@ export default function CreateNoteForm() { + rows={3} + required + /> ++ setTags(e.target.value)} ++ placeholder="Tags (comma-separated)" ++ /> + +diff --git a/examples/bench-app-next/app/notes/[id]/page.tsx b/examples/bench-app-next/app/notes/[id]/page.tsx +index b1e1fff..07b8b5b 100644 +--- a/examples/bench-app-next/app/notes/[id]/page.tsx ++++ b/examples/bench-app-next/app/notes/[id]/page.tsx +@@ -22,6 +22,26 @@ export default async function NoteDetailPage({ + ← Back to notes +

+

{note.title}

++ {note.tags.length > 0 && ( ++

++ {note.tags.map((t) => ( ++ ++ #{t} ++ ++ ))} ++

++ )} +

{note.body}

+ {note.summary && ( +

Summary: {note.summary}

+diff --git a/examples/bench-app-next/app/page.tsx b/examples/bench-app-next/app/page.tsx +index 8e24698..321bfca 100644 +--- a/examples/bench-app-next/app/page.tsx ++++ b/examples/bench-app-next/app/page.tsx +@@ -8,9 +8,14 @@ import LogoutButton from './logout-button'; + + export const dynamic = 'force-dynamic'; + +-export default async function HomePage() { ++export default async function HomePage({ ++ searchParams, ++}: { ++ searchParams: Promise<{ tag?: string }>; ++}) { + if (!(await isAuthenticated())) redirect('/login'); +- const notes = listNotes(); ++ const { tag } = await searchParams; ++ const notes = listNotes(tag || undefined); + + return ( +
+@@ -21,6 +26,15 @@ export default async function HomePage() { + + + ++ {tag && ( ++

++ Filtering by tag: {tag}{' '} ++ ++ (clear) ++ ++

++ )} ++ + {notes.length === 0 ? ( +

No notes yet.

+ ) : ( +@@ -31,6 +45,26 @@ export default async function HomePage() { + {note.title} + +

{note.body}

++ {note.tags.length > 0 && ( ++

++ {note.tags.map((t) => ( ++ ++ #{t} ++ ++ ))} ++

++ )} + {note.summary && ( +

+ Summary: {note.summary} +diff --git a/examples/bench-app-next/lib/db.ts b/examples/bench-app-next/lib/db.ts +index 6d35960..22757c8 100644 +--- a/examples/bench-app-next/lib/db.ts ++++ b/examples/bench-app-next/lib/db.ts +@@ -25,10 +25,17 @@ export function getDb(): Database.Database { + title TEXT NOT NULL, + body TEXT NOT NULL, + summary TEXT, +- created_at TEXT NOT NULL ++ created_at TEXT NOT NULL, ++ tags TEXT NOT NULL DEFAULT '[]' + ); + `); + ++ // Migrate older databases that predate the tags column. ++ const cols = db.prepare('PRAGMA table_info(notes)').all() as { name: string }[]; ++ if (!cols.some((c) => c.name === 'tags')) { ++ db.exec("ALTER TABLE notes ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'"); ++ } ++ + // Seed the single demo user once. + const seeded = db.prepare('SELECT id FROM users WHERE email = ?').get('demo@example.com'); + if (!seeded) { +diff --git a/examples/bench-app-next/lib/notes.ts b/examples/bench-app-next/lib/notes.ts +index 74e45e1..f109f63 100644 +--- a/examples/bench-app-next/lib/notes.ts ++++ b/examples/bench-app-next/lib/notes.ts +@@ -6,6 +6,7 @@ export interface Note { + body: string; + summary: string | null; + createdAt: string; ++ tags: string[]; + } + + interface NoteRow { +@@ -14,6 +15,17 @@ interface NoteRow { + body: string; + summary: string | null; + created_at: string; ++ tags: string | null; ++} ++ ++function parseTags(raw: string | null): string[] { ++ if (!raw) return []; ++ try { ++ const parsed = JSON.parse(raw); ++ return Array.isArray(parsed) ? parsed.filter((t): t is string => typeof t === 'string') : []; ++ } catch { ++ return []; ++ } + } + + function toNote(row: NoteRow): Note { +@@ -23,14 +35,31 @@ function toNote(row: NoteRow): Note { + body: row.body, + summary: row.summary, + createdAt: row.created_at, ++ tags: parseTags(row.tags), + }; + } + +-export function listNotes(): Note[] { ++/** Normalize an arbitrary tags input into a clean string[] (trimmed, de-duped, no empties). */ ++export function normalizeTags(input: unknown): string[] { ++ if (!Array.isArray(input)) return []; ++ const out: string[] = []; ++ for (const item of input) { ++ if (typeof item !== 'string') continue; ++ const tag = item.trim(); ++ if (tag && !out.includes(tag)) out.push(tag); ++ } ++ return out; ++} ++ ++export function listNotes(tag?: string): Note[] { + const rows = getDb() + .prepare('SELECT * FROM notes ORDER BY id DESC') + .all() as NoteRow[]; +- return rows.map(toNote); ++ const notes = rows.map(toNote); ++ if (tag) { ++ return notes.filter((n) => n.tags.includes(tag)); ++ } ++ return notes; + } + + export function getNote(id: number): Note | null { +@@ -40,11 +69,11 @@ export function getNote(id: number): Note | null { + return row ? toNote(row) : null; + } + +-export function createNote(title: string, body: string): Note { ++export function createNote(title: string, body: string, tags: string[] = []): Note { + const createdAt = new Date().toISOString(); + const info = getDb() +- .prepare('INSERT INTO notes (title, body, summary, created_at) VALUES (?, ?, NULL, ?)') +- .run(title, body, createdAt); ++ .prepare('INSERT INTO notes (title, body, summary, created_at, tags) VALUES (?, ?, NULL, ?, ?)') ++ .run(title, body, createdAt, JSON.stringify(normalizeTags(tags))); + return getNote(Number(info.lastInsertRowid))!; + } +