evals: fix false-negative regex checks with the agentic LLM judge

eslint.cli.config.mjs

@@ -15,6 +15,10 @@ export default defineConfig([
     // files. Keep both in sync if you change the globs.
     ignores: [
       'bench/**/*',
+      // Eval fixtures are sandbox workspaces with their own package.json and
+      // tsconfig, not repo code — EVAL.ts files may import modules that only
+      // resolve inside the sandbox (e.g. @vercel/agent-eval/eval).
+      'evals/evals/**/*',
       'examples/**/*',
       'test/**/*',
       '**/*.d.ts',

eslint.config.mjs

@@ -404,6 +404,10 @@ export default defineConfig([
     files: ['**/*.ts', '**/*.tsx'],
     ignores: [
       'bench/**/*',
+      // Eval fixtures are sandbox workspaces with their own package.json and
+      // tsconfig, not repo code — EVAL.ts files may import modules that only
+      // resolve inside the sandbox (e.g. @vercel/agent-eval/eval).
+      'evals/evals/**/*',
       'examples/**/*',
       'test/**/*',
       '**/*.d.ts',

evals/evals/agent-030-app-router-migration-hard/EVAL.ts

@@ -13,6 +13,7 @@
 import { expect, test } from 'vitest'
 import { readFileSync, existsSync } from 'fs'
 import { join } from 'path'
+import { environment } from '@vercel/agent-eval/eval'
 
 /** Strip JS/TS comments so we only test actual code, not migration notes */
 function stripComments(code: string): string {
@@ -36,25 +37,28 @@ test('Root layout exists and replaces _app/_document', () => {
   expect(layoutContent).toMatch(/children.*ReactNode/)
 })
 
-test('Home page migrated to Server Component with async data fetching', () => {
+// The "is it a Server Component fetching data" check is semantic, so it uses the
+// agentic LLM judge rather than regex. The old regexes rejected correct solutions
+// that didn't match one exact shape — e.g. data fetching extracted to a helper
+// (no literal `fetch(` in page.tsx) or a component not declared with the
+// `export default async function` form.
+test('Home page migrated to Server Component with async data fetching', async () => {
   const pagePath = join(process.cwd(), 'app', 'page.tsx')
   expect(existsSync(pagePath)).toBe(true)
 
-  const pageContent = readFileSync(pagePath, 'utf-8')
-
-  // Should be async Server Component
-  expect(pageContent).toMatch(
-    /export\s+default\s+async\s+function|async\s+function.*Page/
-  )
+  await expect(environment).toSatisfyCriterion(
+    `app/page.tsx is the home page migrated to the App Router: an async Server Component that fetches its data during server render.
 
-  // Should NOT have 'use client' directive
-  expect(pageContent).not.toMatch(/['"]use client['"];?/)
+For reference, one correct solution shape:
 
-  // Should use fetch instead of getServerSideProps
-  expect(pageContent).toMatch(/await\s+fetch|fetch\(/)
+  // app/page.tsx
+  export default async function HomePage() {
+    const posts = await getPosts() // fetched inline or via an imported helper
+    return <main>{/* renders the fetched data */}</main>
+  }
 
-  // Should not have getServerSideProps in actual code (comments OK)
-  expect(stripComments(pageContent)).not.toMatch(/getServerSideProps/)
+Judge runtime behavior, not style: any organization that renders the fetched data from a Server Component is correct.`
+  )
 })
 
 test('Blog index migrated with ISR equivalent', () => {

evals/evals/agent-030-app-router-migration-hard/tsconfig.json

@@ -23,5 +23,5 @@
     }
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules", "EVAL.ts"]
 }

evals/evals/agent-034-async-cookies/EVAL.ts

@@ -6,11 +6,19 @@
  *
  * Tricky because agents trained on Next.js 15 call cookies()/headers()
  * synchronously — Next.js 16 removed synchronous access entirely.
+ *
+ * The awaited-correctly check is semantic, so it uses the agentic LLM judge
+ * rather than regex. The old /await\s+cookies\(\)/ regex only matched the naive
+ * `await cookies()` form and rejected correct (arguably better) code like
+ * `await Promise.all([cookies(), headers()])`, and its no-sync-call lookbehind
+ * wrongly flagged the bare `cookies()` inside that array. The judge reasons
+ * about whether the promises are actually awaited before use, whatever the form.
  */
 
 import { expect, test } from 'vitest'
 import { readFileSync, existsSync, readdirSync } from 'fs'
 import { join } from 'path'
+import { environment } from '@vercel/agent-eval/eval'
 
 function readAppFiles(): string {
   const appDir = join(process.cwd(), 'app')
@@ -20,29 +28,18 @@ function readAppFiles(): string {
   return files.map((f) => readFileSync(join(appDir, f), 'utf-8')).join('\n')
 }
 
-test('Component uses await with cookies()', () => {
-  const content = readAppFiles()
-
-  // In Next.js 16, cookies() returns a Promise and MUST be awaited
-  // Correct: const cookieStore = await cookies()
-  // Wrong: const cookieStore = cookies()
-  expect(content).toMatch(/await\s+cookies\s*\(\s*\)/)
-})
+test('cookies() and headers() are consumed as async APIs', async () => {
+  await expect(environment).toSatisfyCriterion(
+    `Next.js 16's cookies() and headers() (from 'next/headers') are async: they return Promises. In the app/ directory, the code must consume them correctly at runtime — every read of the cookie store or headers list operates on a resolved value, never on a pending Promise.
 
-test('Component uses await with headers()', () => {
-  const content = readAppFiles()
-
-  // In Next.js 16, headers() returns a Promise and MUST be awaited
-  // Correct: const headersList = await headers()
-  // Wrong: const headersList = headers()
-  expect(content).toMatch(/await\s+headers\s*\(\s*\)/)
-})
+For reference, one correct solution:
 
-test('Component is async', () => {
-  const content = readAppFiles()
+  const [cookieStore, headersList] = await Promise.all([cookies(), headers()])
+  const theme = cookieStore.get('theme')?.value
+  const lang = headersList.get('accept-language')
 
-  // Component must be async to use await
-  expect(content).toMatch(/async\s+function|export\s+default\s+async/)
+Judge runtime correctness, not style: any form that resolves the promises before using the values is correct, even if unidiomatic or redundant.`
+  )
 })
 
 test('Component reads theme cookie', () => {
@@ -61,18 +58,3 @@ test('Component reads Accept-Language header', () => {
   // Should read Accept-Language header
   expect(content).toMatch(/accept-language/i)
 })
-
-test('Does NOT use synchronous cookies() pattern', () => {
-  const content = readAppFiles()
-
-  // Should NOT have synchronous pattern like:
-  // const cookieStore = cookies()
-  // (without await)
-
-  // This regex matches "cookies()" that is NOT preceded by "await"
-  // We check that every cookies() call is preceded by await
-  const syncCookiesPattern = /(?<!await\s)cookies\s*\(\s*\)(?!\s*\.then)/
-
-  // Verify the synchronous cookies() pattern is NOT used
-  expect(content).not.toMatch(syncCookiesPattern)
-})

evals/evals/agent-034-async-cookies/tsconfig.json

@@ -23,5 +23,5 @@
     }
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules", "EVAL.ts"]
 }

package.json

@@ -166,7 +166,7 @@
     "@types/string-hash": "1.1.1",
     "@types/trusted-types": "2.0.3",
     "@types/yargs": "16.0.11",
-    "@vercel/agent-eval": "0.9.5",
+    "@vercel/agent-eval": "1.3.0",
     "@vercel/blob": "2.3.2",
     "@vercel/devlow-bench": "workspace:*",
     "@vercel/kv": "3.0.0",

run-evals.js

@@ -72,8 +72,13 @@ function writeExperiments(evalName) {
 ${v.imports}
 
 const config: ExperimentConfig = {
-  agent: 'claude-code',
-  model: 'claude-opus-4-6',${evalsField}
+  // Via the Vercel AI Gateway, so the OIDC token from \`vc env pull\` is the only
+  // credential needed (it auths the sandbox, the codegen model, and the judge).
+  agent: 'vercel-ai-gateway/claude-code',
+  model: 'claude-opus-4-8',${evalsField}
+  // Cheap fixed grader for the agentic judge clauses in EVAL.ts files — every
+  // run is graded by the same model regardless of the model under test.
+  judge: { model: 'claude-haiku-4-5' },
   scripts: ['build'],
   runs: 1,
   earlyExit: true,
@@ -136,8 +141,11 @@ function main() {
 
   /** @type {string | null} */
   const evalName = argv.all ? null : /** @type {string} */ (argv.evalName)
-  // Flags not consumed here are forwarded to agent-eval.
-  const forward = argv.dry ? ['--dry'] : []
+  // agent-eval 1.3 dropped run-all/--dry: `run` takes explicit experiment names,
+  // and `status` is the read-only preview.
+  const agentEvalArgs = argv.dry
+    ? ['status']
+    : ['run', ...VARIANTS.map((v) => v.suffix), '--force']
 
   if (!fs.existsSync(path.join(ROOT, 'packages/next/dist'))) {
     console.error(
@@ -180,7 +188,7 @@ function main() {
   // the bin directly rather than via `pnpm exec` because pnpm resets cwd to
   // the workspace root, but agent-eval resolves experiments/ from process.cwd().
   const bin = path.join(ROOT, 'node_modules/.bin/agent-eval')
-  const result = spawnSync(bin, ['run-all', '--force', ...forward], {
+  const result = spawnSync(bin, agentEvalArgs, {
     cwd: EVALS_DIR,
     stdio: 'inherit',
     env: { ...process.env, NEXT_EVAL_TARBALL: TARBALL },