feat(bench): resolution benchmark v2 — dynamic tracing, 14 languages, per-mode categories

scripts/build-wasm.ts

@@ -128,7 +128,14 @@ let failed = 0;
 let rejected = 0;
 
 for (const g of grammars) {
-  const pkgDir = dirname(require.resolve(`${g.pkg}/package.json`));
+  let pkgDir: string;
+  try {
+    pkgDir = dirname(require.resolve(`${g.pkg}/package.json`));
+  } catch {
+    failed++;
+    console.warn(`  WARN: Skipping ${g.name}.wasm — package '${g.pkg}' not installed`);
+    continue;
+  }
   const grammarDir = g.sub ? resolve(pkgDir, g.sub) : pkgDir;
 
   console.log(`Building ${g.name}.wasm from ${grammarDir}...`);

scripts/resolution-benchmark.ts

@@ -60,11 +60,14 @@ interface LangResult {
 
 // ── Helpers ──────────────────────────────────────────────────────────────
 
+// Files to skip when copying fixtures (not source code for codegraph)
+const SKIP_FILES = new Set(['expected-edges.json', 'driver.mjs']);
+
 function copyFixture(lang: string): string {
 	const src = path.join(FIXTURES_DIR, lang);
 	const tmp = fs.mkdtempSync(path.join(os.tmpdir(), `codegraph-resolution-${lang}-`));
 	for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
-		if (entry.name === 'expected-edges.json') continue;
+		if (SKIP_FILES.has(entry.name)) continue;
 		if (!entry.isFile()) {
 			console.error(`  Warning: skipping subdirectory "${entry.name}" in ${lang} fixture (flat copy only)`);
 			continue;

scripts/update-benchmark-report.ts

@@ -398,14 +398,14 @@ if (fs.existsSync(readmePath)) {
 		benchmarkLinks = linksMatch[1];
 	}
 
-	// Resolution precision/recall — from resolution-benchmark.ts JSON merged into entry
-	// Resolution is engine-independent, so show single value (span both columns when needed)
+	// Resolution precision/recall — aggregate row in the main table
+	let resolutionTable = '';
 	if (latest.resolution) {
-		const langs = Object.values(latest.resolution);
-		if (langs.length > 0) {
-			const totalResolved = langs.reduce((s, l) => s + l.totalResolved, 0);
-			const totalExpected = langs.reduce((s, l) => s + l.totalExpected, 0);
-			const totalTP = langs.reduce((s, l) => s + l.truePositives, 0);
+		const langEntries = Object.entries(latest.resolution);
+		if (langEntries.length > 0) {
+			const totalResolved = langEntries.reduce((s, [, l]) => s + l.totalResolved, 0);
+			const totalExpected = langEntries.reduce((s, [, l]) => s + l.totalExpected, 0);
+			const totalTP = langEntries.reduce((s, [, l]) => s + l.truePositives, 0);
 			const aggPrecision = totalResolved > 0 ? `${((totalTP / totalResolved) * 100).toFixed(1)}%` : 'n/a';
 			const aggRecall = totalExpected > 0 ? `${((totalTP / totalExpected) * 100).toFixed(1)}%` : 'n/a';
 			if (hasBoth) {
@@ -415,6 +415,49 @@ if (fs.existsSync(readmePath)) {
 				rows += `| Resolution precision | **${aggPrecision}** |\n`;
 				rows += `| Resolution recall | **${aggRecall}** |\n`;
 			}
+
+			// Per-language resolution breakdown table
+			// Sort: JS/TS first, then alphabetical
+			const sortOrder = ['javascript', 'typescript'];
+			const sorted = langEntries.sort(([a], [b]) => {
+				const ai = sortOrder.indexOf(a);
+				const bi = sortOrder.indexOf(b);
+				if (ai !== -1 && bi !== -1) return ai - bi;
+				if (ai !== -1) return -1;
+				if (bi !== -1) return 1;
+				return a.localeCompare(b);
+			});
+
+			resolutionTable += '\n<details><summary>Per-language resolution precision/recall</summary>\n\n';
+			resolutionTable += '| Language | Precision | Recall | TP | FP | FN | Edges |\n';
+			resolutionTable += '|----------|----------:|-------:|---:|---:|---:|------:|\n';
+			for (const [lang, m] of sorted) {
+				const p = (m.precision * 100).toFixed(1);
+				const r = (m.recall * 100).toFixed(1);
+				resolutionTable += `| ${lang} | ${p}% | ${r}% | ${m.truePositives} | ${m.falsePositives} | ${m.falseNegatives} | ${m.totalExpected} |\n`;
+			}
+
+			// Per-mode breakdown across all languages
+			const allModes: Record<string, { expected: number; resolved: number }> = {};
+			for (const [, m] of langEntries) {
+				if (!m.byMode) continue;
+				for (const [mode, data] of Object.entries(m.byMode)) {
+					if (!allModes[mode]) allModes[mode] = { expected: 0, resolved: 0 };
+					allModes[mode].expected += data.expected;
+					allModes[mode].resolved += data.resolved;
+				}
+			}
+			if (Object.keys(allModes).length > 0) {
+				resolutionTable += '\n**By resolution mode (all languages):**\n\n';
+				resolutionTable += '| Mode | Resolved | Expected | Recall |\n';
+				resolutionTable += '|------|--------:|---------:|-------:|\n';
+				for (const [mode, data] of Object.entries(allModes).sort(([, a], [, b]) => b.expected - a.expected)) {
+					const recall = data.expected > 0 ? ((data.resolved / data.expected) * 100).toFixed(1) : 'n/a';
+					resolutionTable += `| ${mode} | ${data.resolved} | ${data.expected} | ${recall}% |\n`;
+				}
+			}
+
+			resolutionTable += '\n</details>\n';
 		}
 	}
 
@@ -431,7 +474,7 @@ Self-measured on every release via CI (${benchmarkLinks}):
 ${tableHeader}
 ${rows}
 Metrics are normalized per file for cross-version comparability. Times above are for a full initial build — incremental rebuilds only re-parse changed files.
-`;
+${resolutionTable}`;
 
 	// Match the performance section from header to next h2/h3 header or end.
 	// The lookahead stops at ## (h2) or ### (h3) so subsections like

tests/benchmarks/resolution/expected-edges.schema.json

@@ -41,8 +41,23 @@
           },
           "mode": {
             "type": "string",
-            "enum": ["static", "receiver-typed", "interface-dispatched"],
-            "description": "Resolution mode that should produce this edge"
+            "enum": [
+              "static",
+              "receiver-typed",
+              "interface-dispatched",
+              "closure",
+              "re-export",
+              "dynamic-import",
+              "class-inheritance",
+              "same-file",
+              "constructor",
+              "callback",
+              "higher-order",
+              "trait-dispatch",
+              "module-function",
+              "package-function"
+            ],
+            "description": "Resolution category — describes the language feature exercised by this edge"
           },
           "notes": {
             "type": "string",

tests/benchmarks/resolution/fixtures/bash/expected-edges.json

@@ -0,0 +1,91 @@
+{
+  "$schema": "../../expected-edges.schema.json",
+  "language": "bash",
+  "description": "Hand-annotated call edges for Bash resolution benchmark",
+  "edges": [
+    {
+      "source": { "name": "validate_user", "file": "validators.sh" },
+      "target": { "name": "valid_name", "file": "validators.sh" },
+      "kind": "calls",
+      "mode": "same-file",
+      "notes": "Same-file helper call within validators"
+    },
+    {
+      "source": { "name": "validate_user", "file": "validators.sh" },
+      "target": { "name": "valid_email", "file": "validators.sh" },
+      "kind": "calls",
+      "mode": "same-file",
+      "notes": "Same-file helper call within validators"
+    },
+    {
+      "source": { "name": "create_user", "file": "service.sh" },
+      "target": { "name": "validate_user", "file": "validators.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to validate_user via source"
+    },
+    {
+      "source": { "name": "create_user", "file": "service.sh" },
+      "target": { "name": "format_user", "file": "service.sh" },
+      "kind": "calls",
+      "mode": "same-file",
+      "notes": "Same-file helper call within service"
+    },
+    {
+      "source": { "name": "create_user", "file": "service.sh" },
+      "target": { "name": "repo_save", "file": "repository.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to repo_save via source"
+    },
+    {
+      "source": { "name": "get_user", "file": "service.sh" },
+      "target": { "name": "repo_find_by_id", "file": "repository.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to repo_find_by_id via source"
+    },
+    {
+      "source": { "name": "remove_user", "file": "service.sh" },
+      "target": { "name": "repo_delete", "file": "repository.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to repo_delete via source"
+    },
+    {
+      "source": { "name": "list_users", "file": "service.sh" },
+      "target": { "name": "repo_list_all", "file": "repository.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to repo_list_all via source"
+    },
+    {
+      "source": { "name": "run", "file": "main.sh" },
+      "target": { "name": "create_user", "file": "service.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to create_user via source"
+    },
+    {
+      "source": { "name": "run", "file": "main.sh" },
+      "target": { "name": "get_user", "file": "service.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to get_user via source"
+    },
+    {
+      "source": { "name": "run", "file": "main.sh" },
+      "target": { "name": "list_users", "file": "service.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to list_users via source"
+    },
+    {
+      "source": { "name": "run", "file": "main.sh" },
+      "target": { "name": "remove_user", "file": "service.sh" },
+      "kind": "calls",
+      "mode": "static",
+      "notes": "Cross-file call to remove_user via source"
+    }
+  ]
+}

tests/benchmarks/resolution/fixtures/bash/main.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")/service.sh"
+
+run() {
+  create_user "u1" "Alice" "alice@example.com"
+  local found
+  found=$(get_user "u1")
+  if [[ -n "$found" ]]; then
+    echo "Found: $found"
+  fi
+  list_users
+  remove_user "u1"
+}
+
+run

tests/benchmarks/resolution/fixtures/bash/repository.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+declare -A STORE
+
+repo_save() {
+  local id="$1"
+  local data="$2"
+  STORE["$id"]="$data"
+}
+
+repo_find_by_id() {
+  local id="$1"
+  echo "${STORE[$id]}"
+}
+
+repo_delete() {
+  local id="$1"
+  unset STORE["$id"]
+}
+
+repo_list_all() {
+  for key in "${!STORE[@]}"; do
+    echo "${STORE[$key]}"
+  done
+}

tests/benchmarks/resolution/fixtures/bash/service.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")/validators.sh"
+source "$(dirname "$0")/repository.sh"
+
+format_user() {
+  local id="$1"
+  local name="$2"
+  local email="$3"
+  echo "${id}:${name}:${email}"
+}
+
+create_user() {
+  local id="$1"
+  local name="$2"
+  local email="$3"
+  if ! validate_user "$name" "$email"; then
+    echo "Invalid user data" >&2
+    return 1
+  fi
+  local data
+  data=$(format_user "$id" "$name" "$email")
+  repo_save "$id" "$data"
+}
+
+get_user() {
+  local id="$1"
+  repo_find_by_id "$id"
+}
+
+remove_user() {
+  local id="$1"
+  repo_delete "$id"
+}
+
+list_users() {
+  repo_list_all
+}

tests/benchmarks/resolution/fixtures/bash/validators.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+valid_email() {
+  local email="$1"
+  [[ "$email" == *@*.* ]]
+}
+
+valid_name() {
+  local name="$1"
+  [[ ${#name} -ge 2 ]]
+}
+
+validate_user() {
+  local name="$1"
+  local email="$2"
+  valid_name "$name" && valid_email "$email"
+}