From 206816e98bcad5699e907152fe67c70a8e63648d Mon Sep 17 00:00:00 2001 From: Jude Gao Date: Thu, 2 Jul 2026 23:03:48 -0400 Subject: [PATCH 1/3] evals: run via the AI Gateway with a fixed haiku judge; adapt to agent-eval 1.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump @vercel/agent-eval 0.9.5 -> 1.3.0 (ships the agentic judge runtime) and adapt run-evals.js to the 1.3 CLI: `run baseline agents-md --force`, with --dry mapping to the read-only `status` command. Generated experiments now use vercel-ai-gateway/claude-code (default model claude-opus-4-8), so the OIDC token from `vc env pull` is the only credential needed — sandbox, codegen model, and judge. The judge is pinned to claude-haiku-4-5: a cheap fixed grader, identical for every run regardless of the model under test. --- package.json | 2 +- pnpm-lock.yaml | 44 ++++++++------------------------------------ run-evals.js | 18 +++++++++++++----- 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/package.json b/package.json index ede6cfa64fec..3c37bc9e4e5e 100644 --- a/package.json +++ b/package.json @@ -166,7 +166,7 @@ "@types/string-hash": "1.1.1", "@types/trusted-types": "2.0.3", "@types/yargs": "16.0.11", - "@vercel/agent-eval": "0.9.5", + "@vercel/agent-eval": "1.3.0", "@vercel/blob": "2.3.2", "@vercel/devlow-bench": "workspace:*", "@vercel/kv": "3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61df21237b54..b93339e67465 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -244,8 +244,8 @@ importers: specifier: 16.0.11 version: 16.0.11 '@vercel/agent-eval': - specifier: 0.9.5 - version: 0.9.5 + specifier: 1.3.0 + version: 1.3.0 '@vercel/blob': specifier: 2.3.2 version: 2.3.2(patch_hash=cb53bfa5effb15b3a361e176003df667cadf757b27b7c9b458c2f0fce86ea893) @@ -6971,8 +6971,8 @@ packages: '@upstash/redis@1.35.3': resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==} - '@vercel/agent-eval@0.9.5': - resolution: {integrity: sha512-wSs0K39zZFQDNniG0UNeZMy3GgVDSuTyhVKZVqxm33nu3sq2KJjOSJCpvoFaXKUWkk2Ea3DTaxJsVgp8b1Xq5w==} + '@vercel/agent-eval@1.3.0': + resolution: {integrity: sha512-x8mxdvteWp7z4I5VuOm2mknlyMUTevbbp7RCiPg7gsVSknOcVxzJxBCaccenHoux2JKwK63svnzfoeIfB2dDWg==} engines: {node: '>=18.0.0'} hasBin: true @@ -8204,10 +8204,6 @@ packages: resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} engines: {node: '>=8'} - cli-cursor@4.0.0: - resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - cli-cursor@5.0.0: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} @@ -12664,10 +12660,6 @@ packages: resolution: {integrity: sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==} engines: {node: '>=10'} - log-update@6.0.0: - resolution: {integrity: sha512-niTvB4gqvtof056rRIrTZvjNYE4rCUzO6X/X+kYjd7WFxXeJ0NwEFnRxX6ehkvv3jTwrXnNdtAak5XYZuIyPFw==} - engines: {node: '>=18'} - log-update@6.1.0: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} @@ -15756,10 +15748,6 @@ packages: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} - restore-cursor@4.0.0: - resolution: {integrity: sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -17655,6 +17643,7 @@ packages: uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@11.1.0: @@ -23684,7 +23673,7 @@ snapshots: dependencies: uncrypto: 0.1.3 - '@vercel/agent-eval@0.9.5': + '@vercel/agent-eval@1.3.0': dependencies: '@ai-sdk/anthropic': 1.2.12(zod@3.25.76) '@vercel/sandbox': 1.8.0 @@ -25174,10 +25163,6 @@ snapshots: dependencies: restore-cursor: 3.1.0 - cli-cursor@4.0.0: - dependencies: - restore-cursor: 4.0.0 - cli-cursor@5.0.0: dependencies: restore-cursor: 5.1.0 @@ -30649,7 +30634,7 @@ snapshots: cli-truncate: 4.0.0 colorette: 2.0.20 eventemitter3: 5.0.1 - log-update: 6.0.0 + log-update: 6.1.0 rfdc: 1.3.0 wrap-ansi: 9.0.0 @@ -30844,14 +30829,6 @@ snapshots: slice-ansi: 4.0.0 wrap-ansi: 6.2.0 - log-update@6.0.0: - dependencies: - ansi-escapes: 6.2.1 - cli-cursor: 4.0.0 - slice-ansi: 7.1.0 - strip-ansi: 7.1.0 - wrap-ansi: 9.0.0 - log-update@6.1.0: dependencies: ansi-escapes: 7.3.0 @@ -34767,11 +34744,6 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - restore-cursor@4.0.0: - dependencies: - onetime: 5.1.2 - signal-exit: 3.0.7 - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -37540,7 +37512,7 @@ snapshots: dependencies: ansi-styles: 4.3.0 string-width: 4.2.3 - strip-ansi: 6.0.1 + strip-ansi: 6.0.0 wrap-ansi@8.1.0: dependencies: diff --git a/run-evals.js b/run-evals.js index 224811794de6..f7279670d32e 100644 --- a/run-evals.js +++ b/run-evals.js @@ -72,8 +72,13 @@ function writeExperiments(evalName) { ${v.imports} const config: ExperimentConfig = { - agent: 'claude-code', - model: 'claude-opus-4-6',${evalsField} + // Via the Vercel AI Gateway, so the OIDC token from \`vc env pull\` is the only + // credential needed (it auths the sandbox, the codegen model, and the judge). + agent: 'vercel-ai-gateway/claude-code', + model: 'claude-opus-4-8',${evalsField} + // Cheap fixed grader for the agentic judge clauses in EVAL.ts files — every + // run is graded by the same model regardless of the model under test. + judge: { model: 'claude-haiku-4-5' }, scripts: ['build'], runs: 1, earlyExit: true, @@ -136,8 +141,11 @@ function main() { /** @type {string | null} */ const evalName = argv.all ? null : /** @type {string} */ (argv.evalName) - // Flags not consumed here are forwarded to agent-eval. - const forward = argv.dry ? ['--dry'] : [] + // agent-eval 1.3 dropped run-all/--dry: `run` takes explicit experiment names, + // and `status` is the read-only preview. + const agentEvalArgs = argv.dry + ? ['status'] + : ['run', ...VARIANTS.map((v) => v.suffix), '--force'] if (!fs.existsSync(path.join(ROOT, 'packages/next/dist'))) { console.error( @@ -180,7 +188,7 @@ function main() { // the bin directly rather than via `pnpm exec` because pnpm resets cwd to // the workspace root, but agent-eval resolves experiments/ from process.cwd(). const bin = path.join(ROOT, 'node_modules/.bin/agent-eval') - const result = spawnSync(bin, ['run-all', '--force', ...forward], { + const result = spawnSync(bin, agentEvalArgs, { cwd: EVALS_DIR, stdio: 'inherit', env: { ...process.env, NEXT_EVAL_TARBALL: TARBALL }, From 58a13bb1cc35d086950900142802a2ed4ef585cb Mon Sep 17 00:00:00 2001 From: Jude Gao Date: Thu, 2 Jul 2026 23:03:48 -0400 Subject: [PATCH 2/3] evals: fix false-negative regex checks with the agentic LLM judge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent-034-async-cookies: /await\s+cookies\(\)/ only matched the naive form and rejected the correct `await Promise.all([cookies(), headers()])`, and the no-sync-call lookbehind wrongly flagged the bare cookies() inside that array. The four await-mechanics tests collapse into one judge criterion (the promises must actually be awaited before use, in any correct form); the theme-cookie and Accept-Language content checks stay regex. agent-030-app-router-migration-hard: the home-page test rejected solutions that didn't match one exact shape (e.g. fetching extracted to a helper means no literal `fetch(` in page.tsx). Its body becomes one judge criterion; the file-existence check and the other seven tests stay deterministic. Both fixtures exclude EVAL.ts from tsconfig: the @vercel/agent-eval/eval import has no type declarations, and `next build` (run by experiments with scripts: ['build']) type-checks the fixture — same reason EVAL.tsx was already excluded. --- .../EVAL.ts | 32 ++++++----- .../tsconfig.json | 2 +- evals/evals/agent-034-async-cookies/EVAL.ts | 53 +++++++------------ .../agent-034-async-cookies/tsconfig.json | 2 +- 4 files changed, 41 insertions(+), 48 deletions(-) diff --git a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts index bfde69883bf0..a36e8f5a6722 100644 --- a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts +++ b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts @@ -13,6 +13,7 @@ import { expect, test } from 'vitest' import { readFileSync, existsSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' /** Strip JS/TS comments so we only test actual code, not migration notes */ function stripComments(code: string): string { @@ -36,25 +37,30 @@ test('Root layout exists and replaces _app/_document', () => { expect(layoutContent).toMatch(/children.*ReactNode/) }) -test('Home page migrated to Server Component with async data fetching', () => { +// The "is it a Server Component fetching data" check is semantic, so it uses the +// agentic LLM judge rather than regex. The old regexes rejected correct solutions +// that didn't match one exact shape — e.g. data fetching extracted to a helper +// (no literal `fetch(` in page.tsx) or a component not declared with the +// `export default async function` form. +test('Home page migrated to Server Component with async data fetching', async () => { const pagePath = join(process.cwd(), 'app', 'page.tsx') expect(existsSync(pagePath)).toBe(true) - const pageContent = readFileSync(pagePath, 'utf-8') - - // Should be async Server Component - expect(pageContent).toMatch( - /export\s+default\s+async\s+function|async\s+function.*Page/ - ) + await expect(environment).toSatisfyCriterion( + `app/page.tsx is the migrated home page: an async Server Component that fetches its data during server render, with no Pages Router data-fetching API left in actual code (mentions in comments are fine). - // Should NOT have 'use client' directive - expect(pageContent).not.toMatch(/['"]use client['"];?/) +For reference, a correct solution looks like: - // Should use fetch instead of getServerSideProps - expect(pageContent).toMatch(/await\s+fetch|fetch\(/) + // app/page.tsx — note: NO 'use client' directive + export default async function HomePage() { + // inline fetch(...) here, or an imported helper that fetches — + // any organization of the data fetching is fine + const posts = await getPosts() + return
{/* renders the fetched data */}
+ } - // Should not have getServerSideProps in actual code (comments OK) - expect(stripComments(pageContent)).not.toMatch(/getServerSideProps/) +WRONG (fails the criterion): the file has a 'use client' directive; or actual code still defines or uses getServerSideProps; or the page renders without fetching its data on the server.` + ) }) test('Blog index migrated with ISR equivalent', () => { diff --git a/evals/evals/agent-030-app-router-migration-hard/tsconfig.json b/evals/evals/agent-030-app-router-migration-hard/tsconfig.json index 00978ef407fd..cc321ed0658b 100644 --- a/evals/evals/agent-030-app-router-migration-hard/tsconfig.json +++ b/evals/evals/agent-030-app-router-migration-hard/tsconfig.json @@ -23,5 +23,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-034-async-cookies/EVAL.ts b/evals/evals/agent-034-async-cookies/EVAL.ts index ed9cebe78768..497dc4473277 100644 --- a/evals/evals/agent-034-async-cookies/EVAL.ts +++ b/evals/evals/agent-034-async-cookies/EVAL.ts @@ -6,11 +6,19 @@ * * Tricky because agents trained on Next.js 15 call cookies()/headers() * synchronously — Next.js 16 removed synchronous access entirely. + * + * The awaited-correctly check is semantic, so it uses the agentic LLM judge + * rather than regex. The old /await\s+cookies\(\)/ regex only matched the naive + * `await cookies()` form and rejected correct (arguably better) code like + * `await Promise.all([cookies(), headers()])`, and its no-sync-call lookbehind + * wrongly flagged the bare `cookies()` inside that array. The judge reasons + * about whether the promises are actually awaited before use, whatever the form. */ import { expect, test } from 'vitest' import { readFileSync, existsSync, readdirSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' function readAppFiles(): string { const appDir = join(process.cwd(), 'app') @@ -20,29 +28,23 @@ function readAppFiles(): string { return files.map((f) => readFileSync(join(appDir, f), 'utf-8')).join('\n') } -test('Component uses await with cookies()', () => { - const content = readAppFiles() +test('cookies() and headers() are consumed as async APIs', async () => { + await expect(environment).toSatisfyCriterion( + `In the app/ directory, every call to Next.js's cookies() and headers() (from 'next/headers') has its returned promise awaited before the value is used. Nothing treats the return value as a synchronous store. - // In Next.js 16, cookies() returns a Promise and MUST be awaited - // Correct: const cookieStore = await cookies() - // Wrong: const cookieStore = cookies() - expect(content).toMatch(/await\s+cookies\s*\(\s*\)/) -}) +For reference, each of these consumptions is CORRECT (any one of them, or an equivalent, passes): -test('Component uses await with headers()', () => { - const content = readAppFiles() + const cookieStore = await cookies() + const headersList = await headers() + const [cookieStore, headersList] = await Promise.all([cookies(), headers()]) - // In Next.js 16, headers() returns a Promise and MUST be awaited - // Correct: const headersList = await headers() - // Wrong: const headersList = headers() - expect(content).toMatch(/await\s+headers\s*\(\s*\)/) -}) +Note the bare cookies() inside Promise.all([...]) IS correctly awaited — the await applies to the whole array. Do not fail it. -test('Component is async', () => { - const content = readAppFiles() +This is WRONG (fails the criterion) — using the value without awaiting: - // Component must be async to use await - expect(content).toMatch(/async\s+function|export\s+default\s+async/) + const cookieStore = cookies() // no await + const theme = cookieStore.get('theme') // .get() on a Promise` + ) }) test('Component reads theme cookie', () => { @@ -61,18 +63,3 @@ test('Component reads Accept-Language header', () => { // Should read Accept-Language header expect(content).toMatch(/accept-language/i) }) - -test('Does NOT use synchronous cookies() pattern', () => { - const content = readAppFiles() - - // Should NOT have synchronous pattern like: - // const cookieStore = cookies() - // (without await) - - // This regex matches "cookies()" that is NOT preceded by "await" - // We check that every cookies() call is preceded by await - const syncCookiesPattern = /(? Date: Thu, 2 Jul 2026 23:25:09 -0400 Subject: [PATCH 3/3] lint: ignore eval fixtures in the type-checked eslint configs Eval fixtures are sandbox workspaces with their own package.json/tsconfig, not repo code. Their EVAL.ts files may import modules that only resolve inside the sandbox (@vercel/agent-eval/eval) and are excluded from the fixture tsconfig, so type-aware linting with `project: true` fails to place them in a project. Both ignore lists (eslint.cli.config.mjs and its mirror block) stay in sync. --- eslint.cli.config.mjs | 4 ++++ eslint.config.mjs | 4 ++++ .../agent-030-app-router-migration-hard/EVAL.ts | 12 +++++------- evals/evals/agent-034-async-cookies/EVAL.ts | 15 +++++---------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/eslint.cli.config.mjs b/eslint.cli.config.mjs index 32b7d7529ec1..98389b5f5117 100644 --- a/eslint.cli.config.mjs +++ b/eslint.cli.config.mjs @@ -15,6 +15,10 @@ export default defineConfig([ // files. Keep both in sync if you change the globs. ignores: [ 'bench/**/*', + // Eval fixtures are sandbox workspaces with their own package.json and + // tsconfig, not repo code — EVAL.ts files may import modules that only + // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). + 'evals/evals/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/eslint.config.mjs b/eslint.config.mjs index 29974435598f..be2e89305256 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -404,6 +404,10 @@ export default defineConfig([ files: ['**/*.ts', '**/*.tsx'], ignores: [ 'bench/**/*', + // Eval fixtures are sandbox workspaces with their own package.json and + // tsconfig, not repo code — EVAL.ts files may import modules that only + // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). + 'evals/evals/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts index a36e8f5a6722..8665d81079bd 100644 --- a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts +++ b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts @@ -47,19 +47,17 @@ test('Home page migrated to Server Component with async data fetching', async () expect(existsSync(pagePath)).toBe(true) await expect(environment).toSatisfyCriterion( - `app/page.tsx is the migrated home page: an async Server Component that fetches its data during server render, with no Pages Router data-fetching API left in actual code (mentions in comments are fine). + `app/page.tsx is the home page migrated to the App Router: an async Server Component that fetches its data during server render. -For reference, a correct solution looks like: +For reference, one correct solution shape: - // app/page.tsx — note: NO 'use client' directive + // app/page.tsx export default async function HomePage() { - // inline fetch(...) here, or an imported helper that fetches — - // any organization of the data fetching is fine - const posts = await getPosts() + const posts = await getPosts() // fetched inline or via an imported helper return
{/* renders the fetched data */}
} -WRONG (fails the criterion): the file has a 'use client' directive; or actual code still defines or uses getServerSideProps; or the page renders without fetching its data on the server.` +Judge runtime behavior, not style: any organization that renders the fetched data from a Server Component is correct.` ) }) diff --git a/evals/evals/agent-034-async-cookies/EVAL.ts b/evals/evals/agent-034-async-cookies/EVAL.ts index 497dc4473277..78485756fb83 100644 --- a/evals/evals/agent-034-async-cookies/EVAL.ts +++ b/evals/evals/agent-034-async-cookies/EVAL.ts @@ -30,20 +30,15 @@ function readAppFiles(): string { test('cookies() and headers() are consumed as async APIs', async () => { await expect(environment).toSatisfyCriterion( - `In the app/ directory, every call to Next.js's cookies() and headers() (from 'next/headers') has its returned promise awaited before the value is used. Nothing treats the return value as a synchronous store. + `Next.js 16's cookies() and headers() (from 'next/headers') are async: they return Promises. In the app/ directory, the code must consume them correctly at runtime — every read of the cookie store or headers list operates on a resolved value, never on a pending Promise. -For reference, each of these consumptions is CORRECT (any one of them, or an equivalent, passes): +For reference, one correct solution: - const cookieStore = await cookies() - const headersList = await headers() const [cookieStore, headersList] = await Promise.all([cookies(), headers()]) + const theme = cookieStore.get('theme')?.value + const lang = headersList.get('accept-language') -Note the bare cookies() inside Promise.all([...]) IS correctly awaited — the await applies to the whole array. Do not fail it. - -This is WRONG (fails the criterion) — using the value without awaiting: - - const cookieStore = cookies() // no await - const theme = cookieStore.get('theme') // .get() on a Promise` +Judge runtime correctness, not style: any form that resolves the promises before using the values is correct, even if unidiomatic or redundant.` ) })