From a04ee4550a49814196f1704a2e8314aeb87ea7d4 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:33:17 -0600 Subject: [PATCH 1/3] perf(native): fix 1238% incremental rebuild regression by skipping redundant structure phase The native orchestrator's post-processing unconditionally ran a full JS structure rebuild (changedFilePaths=null) and loaded all nodes from DB after every native build, regardless of whether the Rust fast-path already handled structure. For a 1-file change on a 567-file codebase, this turned a 42ms rebuild into 562ms. Three fixes: 1. Rust reports `structureHandled: bool` in BuildPipelineResult so the JS side knows whether the small-incremental fast path ran 2. When Rust handled structure, JS skips the structure phase entirely and scopes the DB reconstruction query to changed files only 3. When Rust didn't handle structure (full/large incremental), JS now passes changedFiles to buildStructureFn instead of null, matching the JS pipeline's medium-incremental path behavior --- crates/codegraph-core/src/build_pipeline.rs | 7 ++ src/domain/graph/builder/pipeline.ts | 81 +++++++++++++++------ 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/crates/codegraph-core/src/build_pipeline.rs b/crates/codegraph-core/src/build_pipeline.rs index 09e745e9..a525e2ea 100644 --- a/crates/codegraph-core/src/build_pipeline.rs +++ b/crates/codegraph-core/src/build_pipeline.rs @@ -67,6 +67,11 @@ pub struct BuildPipelineResult { pub changed_count: usize, pub removed_count: usize, pub is_full_build: bool, + /// Whether the Rust pipeline handled the structure phase (directory nodes, + /// contains edges, file metrics). True when the small-incremental fast path + /// ran (≤5 changed files, >20 existing files). When false, the JS caller + /// must run its own structure phase as a post-processing step. + pub structure_handled: bool, } /// Normalize path to forward slashes. @@ -205,6 +210,7 @@ pub fn run_pipeline( changed_count: 0, removed_count: 0, is_full_build: false, + structure_handled: true, }); } @@ -587,6 +593,7 @@ pub fn run_pipeline( changed_count: parse_changes.len(), removed_count: change_result.removed.len(), is_full_build: change_result.is_full_build, + structure_handled: use_fast_path, }) } diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index 8caddaf0..f478ac13 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -263,6 +263,8 @@ interface NativeOrchestratorResult { changedCount?: number; removedCount?: number; isFullBuild?: boolean; + /** Whether the Rust pipeline handled the structure phase (small-incremental fast path). */ + structureHandled?: boolean; } // ── Native orchestrator helpers ─────────────────────────────────────── @@ -295,13 +297,26 @@ function handoffWalAfterNativeBuild(ctx: PipelineContext): boolean { } } -/** Reconstruct fileSymbols from the DB after a native orchestrator build. */ -function reconstructFileSymbolsFromDb(ctx: PipelineContext): Map { - const allFileRows = ctx.db - .prepare( - 'SELECT file, name, kind, line, end_line as endLine FROM nodes WHERE file IS NOT NULL ORDER BY file, line', - ) - .all() as { +/** + * Reconstruct fileSymbols from the DB after a native orchestrator build. + * When `scopeFiles` is provided, only loads those files (for analysis-only). + * When omitted, loads all files (needed for structure rebuilds). + */ +function reconstructFileSymbolsFromDb( + ctx: PipelineContext, + scopeFiles?: string[], +): Map { + let query = + 'SELECT file, name, kind, line, end_line as endLine FROM nodes WHERE file IS NOT NULL'; + const params: string[] = []; + if (scopeFiles && scopeFiles.length > 0) { + const placeholders = scopeFiles.map(() => '?').join(','); + query += ` AND file IN (${placeholders})`; + params.push(...scopeFiles); + } + query += ' ORDER BY file, line'; + + const rows = ctx.db.prepare(query).all(...params) as { file: string; name: string; kind: string; @@ -309,9 +324,9 @@ function reconstructFileSymbolsFromDb(ctx: PipelineContext): Map(); - for (const row of allFileRows) { - let entry = allFileSymbols.get(row.file); + const fileSymbols = new Map(); + for (const row of rows) { + let entry = fileSymbols.get(row.file); if (!entry) { entry = { definitions: [], @@ -321,7 +336,7 @@ function reconstructFileSymbolsFromDb(ctx: PipelineContext): Map, + isFullBuild: boolean, + changedFiles: string[] | undefined, ): Promise { const structureStart = performance.now(); try { @@ -396,7 +417,10 @@ async function runPostNativeStructure( lineCountMap.set(row.file, row.line_count); } - const changedFilePaths = null; // full rebuild — every directory gets nodes + // Full builds need null (rebuild everything). Incremental builds pass the + // changed file list so buildStructure only updates those files' metrics + // and contains edges — matching the JS pipeline's medium-incremental path. + const changedFilePaths = isFullBuild ? null : (changedFiles ?? null); const { buildStructure: buildStructureFn } = (await import( '../../../features/structure.js' )) as { @@ -417,7 +441,9 @@ async function runPostNativeStructure( directories, changedFilePaths, ); - debug('Structure phase completed after native orchestrator'); + debug( + `Structure phase completed after native orchestrator${changedFilePaths ? ` (${changedFilePaths.length} files)` : ' (full)'}`, + ); } catch (err) { warn(`Structure phase failed after native build: ${toErrorMessage(err)}`); } @@ -573,8 +599,10 @@ async function tryNativeOrchestrator( ctx.opts.complexity !== false || ctx.opts.cfg !== false || ctx.opts.dataflow !== false; - // Always run JS structure — native fast-path guard can't be reliably detected. - const needsStructure = true; + // Skip JS structure when the Rust pipeline's small-incremental fast path + // already handled it. For full builds and large incrementals where Rust + // skipped structure, we must run the JS fallback. + const needsStructure = !result.structureHandled; if (needsAnalysis || needsStructure) { if (!handoffWalAfterNativeBuild(ctx)) { @@ -582,14 +610,23 @@ async function tryNativeOrchestrator( return formatNativeTimingResult(p, 0, analysisTiming); } - const allFileSymbols = reconstructFileSymbolsFromDb(ctx); + // When structure was handled by Rust, we only need changed files for + // analysis — no need to load the entire graph from DB. When structure + // was NOT handled, we need all files to build the complete directory tree. + const scopeFiles = needsStructure ? undefined : result.changedFiles; + const fileSymbols = reconstructFileSymbolsFromDb(ctx, scopeFiles); if (needsStructure) { - structurePatchMs = await runPostNativeStructure(ctx, allFileSymbols); + structurePatchMs = await runPostNativeStructure( + ctx, + fileSymbols, + !!result.isFullBuild, + result.changedFiles, + ); } if (needsAnalysis) { - analysisTiming = await runPostNativeAnalysis(ctx, allFileSymbols, result.changedFiles); + analysisTiming = await runPostNativeAnalysis(ctx, fileSymbols, result.changedFiles); } } From 1716f4d5c98b3cf29c25a7842f384d44077114f8 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sun, 5 Apr 2026 00:00:07 -0600 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20address=20Greptile=20review=20?= =?UTF-8?q?=E2=80=94=20empty-array=20guard,=20eprintln=20noise,=20structur?= =?UTF-8?q?e=20scope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Guard against empty changedFiles array passing through ?? operator in runPostNativeStructure, which would skip structure updates for removal-only builds (behavioral regression vs pre-PR) - Remove noisy eprintln! that fires on every large-incremental build with a misleading message about running JS pipeline - Add structureScope field to BuildPipelineResult containing the full changed_files list (including reverse-dep files) so the JS structure fallback path can update metrics for files whose edges changed even though their content didn't --- crates/codegraph-core/src/build_pipeline.rs | 20 +++++++++++--------- src/domain/graph/builder/pipeline.ts | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/codegraph-core/src/build_pipeline.rs b/crates/codegraph-core/src/build_pipeline.rs index a525e2ea..a8285efd 100644 --- a/crates/codegraph-core/src/build_pipeline.rs +++ b/crates/codegraph-core/src/build_pipeline.rs @@ -67,6 +67,11 @@ pub struct BuildPipelineResult { pub changed_count: usize, pub removed_count: usize, pub is_full_build: bool, + /// Full set of changed files including reverse-dep files. Used by the JS + /// structure fallback path so it can update metrics for files whose edges + /// changed even though their content didn't. `None` for full builds. + #[serde(skip_serializing_if = "Option::is_none")] + pub structure_scope: Option>, /// Whether the Rust pipeline handled the structure phase (directory nodes, /// contains edges, file metrics). True when the small-incremental fast path /// ran (≤5 changed files, >20 existing files). When false, the JS caller @@ -210,6 +215,7 @@ pub fn run_pipeline( changed_count: 0, removed_count: 0, is_full_build: false, + structure_scope: Some(vec![]), structure_handled: true, }); } @@ -508,15 +514,6 @@ pub fn run_pipeline( &line_count_map, &file_symbols, ); - } else { - // Emit a debug-level warning so users of `codegraph stats` know - // structure metrics were not updated on this build path. - eprintln!( - "[codegraph] note: structure metrics skipped (native fast-path not applicable — \ - {} changed files, full_build={}). Run JS pipeline for full structure.", - parse_changes.len(), - change_result.is_full_build, - ); } // For full/larger builds, the JS fallback handles full structure via // `features/structure.ts`. The Rust orchestrator handles the fast path @@ -593,6 +590,11 @@ pub fn run_pipeline( changed_count: parse_changes.len(), removed_count: change_result.removed.len(), is_full_build: change_result.is_full_build, + structure_scope: if change_result.is_full_build { + None + } else { + Some(changed_files) + }, structure_handled: use_fast_path, }) } diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index f478ac13..c0c02d71 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -263,6 +263,8 @@ interface NativeOrchestratorResult { changedCount?: number; removedCount?: number; isFullBuild?: boolean; + /** Full changed files including reverse-dep files — used by JS structure fallback. */ + structureScope?: string[]; /** Whether the Rust pipeline handled the structure phase (small-incremental fast path). */ structureHandled?: boolean; } @@ -420,7 +422,7 @@ async function runPostNativeStructure( // Full builds need null (rebuild everything). Incremental builds pass the // changed file list so buildStructure only updates those files' metrics // and contains edges — matching the JS pipeline's medium-incremental path. - const changedFilePaths = isFullBuild ? null : (changedFiles ?? null); + const changedFilePaths = isFullBuild || !changedFiles?.length ? null : changedFiles; const { buildStructure: buildStructureFn } = (await import( '../../../features/structure.js' )) as { @@ -621,7 +623,7 @@ async function tryNativeOrchestrator( ctx, fileSymbols, !!result.isFullBuild, - result.changedFiles, + result.structureScope ?? result.changedFiles, ); } From 9f83fd41846b5e5837228465fffa6b7f8f31820a Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sun, 5 Apr 2026 00:48:46 -0600 Subject: [PATCH 3/3] fix(native): resolve use-after-move compile error in structure_scope changed_files was moved into changed_file_list on line 544 and then referenced again for structure_scope on line 607, which Rust rejects as a use-after-move. Reuse changed_file_list.clone() which already holds the correct None/Some(changed_files) value. --- crates/codegraph-core/src/build_pipeline.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/codegraph-core/src/build_pipeline.rs b/crates/codegraph-core/src/build_pipeline.rs index 1df1939e..10ef11ae 100644 --- a/crates/codegraph-core/src/build_pipeline.rs +++ b/crates/codegraph-core/src/build_pipeline.rs @@ -601,11 +601,7 @@ pub fn run_pipeline( changed_count: parse_changes.len(), removed_count: change_result.removed.len(), is_full_build: change_result.is_full_build, - structure_scope: if change_result.is_full_build { - None - } else { - Some(changed_files) - }, + structure_scope: changed_file_list.clone(), structure_handled: use_fast_path, }) }