diff --git a/eslint.cli.config.mjs b/eslint.cli.config.mjs index 32b7d7529ec1..98389b5f5117 100644 --- a/eslint.cli.config.mjs +++ b/eslint.cli.config.mjs @@ -15,6 +15,10 @@ export default defineConfig([ // files. Keep both in sync if you change the globs. ignores: [ 'bench/**/*', + // Eval fixtures are sandbox workspaces with their own package.json and + // tsconfig, not repo code — EVAL.ts files may import modules that only + // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). + 'evals/evals/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/eslint.config.mjs b/eslint.config.mjs index 29974435598f..be2e89305256 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -404,6 +404,10 @@ export default defineConfig([ files: ['**/*.ts', '**/*.tsx'], ignores: [ 'bench/**/*', + // Eval fixtures are sandbox workspaces with their own package.json and + // tsconfig, not repo code — EVAL.ts files may import modules that only + // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). + 'evals/evals/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts index bfde69883bf0..8665d81079bd 100644 --- a/evals/evals/agent-030-app-router-migration-hard/EVAL.ts +++ b/evals/evals/agent-030-app-router-migration-hard/EVAL.ts @@ -13,6 +13,7 @@ import { expect, test } from 'vitest' import { readFileSync, existsSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' /** Strip JS/TS comments so we only test actual code, not migration notes */ function stripComments(code: string): string { @@ -36,25 +37,28 @@ test('Root layout exists and replaces _app/_document', () => { expect(layoutContent).toMatch(/children.*ReactNode/) }) -test('Home page migrated to Server Component with async data fetching', () => { +// The "is it a Server Component fetching data" check is semantic, so it uses the +// agentic LLM judge rather than regex. The old regexes rejected correct solutions +// that didn't match one exact shape — e.g. data fetching extracted to a helper +// (no literal `fetch(` in page.tsx) or a component not declared with the +// `export default async function` form. +test('Home page migrated to Server Component with async data fetching', async () => { const pagePath = join(process.cwd(), 'app', 'page.tsx') expect(existsSync(pagePath)).toBe(true) - const pageContent = readFileSync(pagePath, 'utf-8') - - // Should be async Server Component - expect(pageContent).toMatch( - /export\s+default\s+async\s+function|async\s+function.*Page/ - ) + await expect(environment).toSatisfyCriterion( + `app/page.tsx is the home page migrated to the App Router: an async Server Component that fetches its data during server render. - // Should NOT have 'use client' directive - expect(pageContent).not.toMatch(/['"]use client['"];?/) +For reference, one correct solution shape: - // Should use fetch instead of getServerSideProps - expect(pageContent).toMatch(/await\s+fetch|fetch\(/) + // app/page.tsx + export default async function HomePage() { + const posts = await getPosts() // fetched inline or via an imported helper + return
{/* renders the fetched data */}
+ } - // Should not have getServerSideProps in actual code (comments OK) - expect(stripComments(pageContent)).not.toMatch(/getServerSideProps/) +Judge runtime behavior, not style: any organization that renders the fetched data from a Server Component is correct.` + ) }) test('Blog index migrated with ISR equivalent', () => { diff --git a/evals/evals/agent-030-app-router-migration-hard/tsconfig.json b/evals/evals/agent-030-app-router-migration-hard/tsconfig.json index 00978ef407fd..cc321ed0658b 100644 --- a/evals/evals/agent-030-app-router-migration-hard/tsconfig.json +++ b/evals/evals/agent-030-app-router-migration-hard/tsconfig.json @@ -23,5 +23,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-034-async-cookies/EVAL.ts b/evals/evals/agent-034-async-cookies/EVAL.ts index ed9cebe78768..78485756fb83 100644 --- a/evals/evals/agent-034-async-cookies/EVAL.ts +++ b/evals/evals/agent-034-async-cookies/EVAL.ts @@ -6,11 +6,19 @@ * * Tricky because agents trained on Next.js 15 call cookies()/headers() * synchronously — Next.js 16 removed synchronous access entirely. + * + * The awaited-correctly check is semantic, so it uses the agentic LLM judge + * rather than regex. The old /await\s+cookies\(\)/ regex only matched the naive + * `await cookies()` form and rejected correct (arguably better) code like + * `await Promise.all([cookies(), headers()])`, and its no-sync-call lookbehind + * wrongly flagged the bare `cookies()` inside that array. The judge reasons + * about whether the promises are actually awaited before use, whatever the form. */ import { expect, test } from 'vitest' import { readFileSync, existsSync, readdirSync } from 'fs' import { join } from 'path' +import { environment } from '@vercel/agent-eval/eval' function readAppFiles(): string { const appDir = join(process.cwd(), 'app') @@ -20,29 +28,18 @@ function readAppFiles(): string { return files.map((f) => readFileSync(join(appDir, f), 'utf-8')).join('\n') } -test('Component uses await with cookies()', () => { - const content = readAppFiles() - - // In Next.js 16, cookies() returns a Promise and MUST be awaited - // Correct: const cookieStore = await cookies() - // Wrong: const cookieStore = cookies() - expect(content).toMatch(/await\s+cookies\s*\(\s*\)/) -}) +test('cookies() and headers() are consumed as async APIs', async () => { + await expect(environment).toSatisfyCriterion( + `Next.js 16's cookies() and headers() (from 'next/headers') are async: they return Promises. In the app/ directory, the code must consume them correctly at runtime — every read of the cookie store or headers list operates on a resolved value, never on a pending Promise. -test('Component uses await with headers()', () => { - const content = readAppFiles() - - // In Next.js 16, headers() returns a Promise and MUST be awaited - // Correct: const headersList = await headers() - // Wrong: const headersList = headers() - expect(content).toMatch(/await\s+headers\s*\(\s*\)/) -}) +For reference, one correct solution: -test('Component is async', () => { - const content = readAppFiles() + const [cookieStore, headersList] = await Promise.all([cookies(), headers()]) + const theme = cookieStore.get('theme')?.value + const lang = headersList.get('accept-language') - // Component must be async to use await - expect(content).toMatch(/async\s+function|export\s+default\s+async/) +Judge runtime correctness, not style: any form that resolves the promises before using the values is correct, even if unidiomatic or redundant.` + ) }) test('Component reads theme cookie', () => { @@ -61,18 +58,3 @@ test('Component reads Accept-Language header', () => { // Should read Accept-Language header expect(content).toMatch(/accept-language/i) }) - -test('Does NOT use synchronous cookies() pattern', () => { - const content = readAppFiles() - - // Should NOT have synchronous pattern like: - // const cookieStore = cookies() - // (without await) - - // This regex matches "cookies()" that is NOT preceded by "await" - // We check that every cookies() call is preceded by await - const syncCookiesPattern = /(?=18.0.0'} hasBin: true @@ -8204,10 +8204,6 @@ packages: resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} engines: {node: '>=8'} - cli-cursor@4.0.0: - resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - cli-cursor@5.0.0: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} @@ -12664,10 +12660,6 @@ packages: resolution: {integrity: sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==} engines: {node: '>=10'} - log-update@6.0.0: - resolution: {integrity: sha512-niTvB4gqvtof056rRIrTZvjNYE4rCUzO6X/X+kYjd7WFxXeJ0NwEFnRxX6ehkvv3jTwrXnNdtAak5XYZuIyPFw==} - engines: {node: '>=18'} - log-update@6.1.0: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} @@ -15756,10 +15748,6 @@ packages: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} - restore-cursor@4.0.0: - resolution: {integrity: sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -17655,6 +17643,7 @@ packages: uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@11.1.0: @@ -23684,7 +23673,7 @@ snapshots: dependencies: uncrypto: 0.1.3 - '@vercel/agent-eval@0.9.5': + '@vercel/agent-eval@1.3.0': dependencies: '@ai-sdk/anthropic': 1.2.12(zod@3.25.76) '@vercel/sandbox': 1.8.0 @@ -25174,10 +25163,6 @@ snapshots: dependencies: restore-cursor: 3.1.0 - cli-cursor@4.0.0: - dependencies: - restore-cursor: 4.0.0 - cli-cursor@5.0.0: dependencies: restore-cursor: 5.1.0 @@ -30649,7 +30634,7 @@ snapshots: cli-truncate: 4.0.0 colorette: 2.0.20 eventemitter3: 5.0.1 - log-update: 6.0.0 + log-update: 6.1.0 rfdc: 1.3.0 wrap-ansi: 9.0.0 @@ -30844,14 +30829,6 @@ snapshots: slice-ansi: 4.0.0 wrap-ansi: 6.2.0 - log-update@6.0.0: - dependencies: - ansi-escapes: 6.2.1 - cli-cursor: 4.0.0 - slice-ansi: 7.1.0 - strip-ansi: 7.1.0 - wrap-ansi: 9.0.0 - log-update@6.1.0: dependencies: ansi-escapes: 7.3.0 @@ -34767,11 +34744,6 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - restore-cursor@4.0.0: - dependencies: - onetime: 5.1.2 - signal-exit: 3.0.7 - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -37540,7 +37512,7 @@ snapshots: dependencies: ansi-styles: 4.3.0 string-width: 4.2.3 - strip-ansi: 6.0.1 + strip-ansi: 6.0.0 wrap-ansi@8.1.0: dependencies: diff --git a/run-evals.js b/run-evals.js index 224811794de6..f7279670d32e 100644 --- a/run-evals.js +++ b/run-evals.js @@ -72,8 +72,13 @@ function writeExperiments(evalName) { ${v.imports} const config: ExperimentConfig = { - agent: 'claude-code', - model: 'claude-opus-4-6',${evalsField} + // Via the Vercel AI Gateway, so the OIDC token from \`vc env pull\` is the only + // credential needed (it auths the sandbox, the codegen model, and the judge). + agent: 'vercel-ai-gateway/claude-code', + model: 'claude-opus-4-8',${evalsField} + // Cheap fixed grader for the agentic judge clauses in EVAL.ts files — every + // run is graded by the same model regardless of the model under test. + judge: { model: 'claude-haiku-4-5' }, scripts: ['build'], runs: 1, earlyExit: true, @@ -136,8 +141,11 @@ function main() { /** @type {string | null} */ const evalName = argv.all ? null : /** @type {string} */ (argv.evalName) - // Flags not consumed here are forwarded to agent-eval. - const forward = argv.dry ? ['--dry'] : [] + // agent-eval 1.3 dropped run-all/--dry: `run` takes explicit experiment names, + // and `status` is the read-only preview. + const agentEvalArgs = argv.dry + ? ['status'] + : ['run', ...VARIANTS.map((v) => v.suffix), '--force'] if (!fs.existsSync(path.join(ROOT, 'packages/next/dist'))) { console.error( @@ -180,7 +188,7 @@ function main() { // the bin directly rather than via `pnpm exec` because pnpm resets cwd to // the workspace root, but agent-eval resolves experiments/ from process.cwd(). const bin = path.join(ROOT, 'node_modules/.bin/agent-eval') - const result = spawnSync(bin, ['run-all', '--force', ...forward], { + const result = spawnSync(bin, agentEvalArgs, { cwd: EVALS_DIR, stdio: 'inherit', env: { ...process.env, NEXT_EVAL_TARBALL: TARBALL },