ISS-141 - feat: overlay system — merge remote base graph with local branch diffs

src/index.ts

@@ -47,6 +47,13 @@ import { GraphTraverser, GraphQueryManager } from './graph';
 import { ContextBuilder, createContextBuilder } from './context';
 import { Mutex, FileLock } from './utils';
 import { FileWatcher, WatchOptions } from './sync';
+import * as fs from 'fs';
+import {
+  RemoteGraphClient,
+  BranchDiffIndexer,
+  OverlayQueryEngine,
+} from './overlay';
+import type { RemoteGraphConfig } from './overlay';
 
 // Re-export types for consumers
 export * from './types';
@@ -77,6 +84,12 @@ export {
 export { Mutex, FileLock, processInBatches, debounce, throttle, MemoryMonitor } from './utils';
 export { FileWatcher, WatchOptions } from './sync';
 export { MCPServer } from './mcp';
+export {
+  RemoteGraphClient,
+  BranchDiffIndexer,
+  OverlayQueryEngine,
+} from './overlay';
+export type { RemoteGraphConfig, BranchDiffResult } from './overlay';
 
 /**
  * Options for initializing a new CodeGraph project
@@ -138,6 +151,9 @@ export class CodeGraph {
   // File watcher for auto-sync on file changes
   private watcher: FileWatcher | null = null;
 
+  // Remote graph client (non-null only in overlay mode)
+  private remoteClient: RemoteGraphClient | null = null;
+
   private constructor(
     db: DatabaseConnection,
     queries: QueryBuilder,
@@ -292,13 +308,118 @@ export class CodeGraph {
   }
 
   /**
-   * Close the CodeGraph instance and release resources
+   * Open a CodeGraph project in overlay mode.
+   *
+   * Instead of indexing the entire repository locally, this method:
+   * 1. Fetches a pre-built base graph from a remote source (CI artifact,
+   *    shared filesystem, HTTP server).
+   * 2. Detects files changed on the current feature branch relative to
+   *    the base branch using `git diff`.
+   * 3. Indexes only the changed files into a lightweight local overlay
+   *    database.
+   * 4. Returns a CodeGraph instance whose queries seamlessly merge the
+   *    remote base graph with the local overlay, so the LLM sees a
+   *    complete, up-to-date view.
+   *
+   * This is the team-friendly workflow: CI builds the main-branch graph
+   * once, every developer downloads it and layers on their branch diffs.
+   *
+   * Falls back gracefully: if no files changed, the base graph is
+   * returned as-is. The existing local-only CodeGraph API is unaffected.
+   *
+   * @param projectRoot   - Path to the project root (git repo)
+   * @param remoteConfig  - Remote base graph configuration
+   * @param options        - Indexing options (progress callback, etc.)
+   * @returns A CodeGraph instance with overlay-merged queries
+   *
+   * @example
+   * ```ts
+   * const cg = await CodeGraph.openOverlay('/path/to/repo', {
+   *   url: 'https://ci.example.com/codegraph/main.db',
+   *   baseBranch: 'main',
+   * });
+   * // All queries now merge the base graph + local branch changes
+   * const results = cg.searchNodes('AuthService');
+   * cg.close();
+   * ```
+   */
+  static async openOverlay(
+    projectRoot: string,
+    remoteConfig: RemoteGraphConfig,
+    _options: IndexOptions = {}
+  ): Promise<CodeGraph> {
+    await initGrammars();
+    const resolvedRoot = path.resolve(projectRoot);
+
+    // Ensure .codegraph directory exists
+    if (!isInitialized(resolvedRoot)) {
+      createDirectory(resolvedRoot);
+    }
+
+    // Step 1: Fetch remote base graph
+    const cacheDir =
+      remoteConfig.cacheDir || path.join(resolvedRoot, '.codegraph');
+    const client = new RemoteGraphClient({ ...remoteConfig, cacheDir });
+    await client.fetch();
+    const baseQueries = client.open();
+
+    // Step 2: Detect changed files on the feature branch
+    const diffIndexer = new BranchDiffIndexer(resolvedRoot);
+    const diff = diffIndexer.getChangedFiles(remoteConfig.baseBranch);
+    const overlayFiles = new Set([...diff.added, ...diff.modified]);
+    const deletedFiles = new Set(diff.deleted);
+
+    // Step 3: Initialize local overlay database
+    const overlayDbPath = path.join(resolvedRoot, '.codegraph', 'overlay.db');
+    // Remove stale overlay DB so we start fresh
+    if (fs.existsSync(overlayDbPath)) {
+      fs.unlinkSync(overlayDbPath);
+    }
+    const overlayDb = DatabaseConnection.initialize(overlayDbPath);
+    // Disable FK enforcement on the overlay DB: its edges legitimately
+    // reference nodes that live in the base DB, not in the local overlay.
+    overlayDb.getDb().pragma('foreign_keys = OFF');
+
+    // Step 4: Create overlay query engine (merges base + overlay)
+    const engine = new OverlayQueryEngine(
+      overlayDb.getDb(),
+      baseQueries,
+      overlayFiles,
+      deletedFiles
+    );
+
+    // Step 5: Create CodeGraph instance wired to the overlay engine
+    const instance = new CodeGraph(overlayDb, engine, resolvedRoot);
+    instance.remoteClient = client;
+
+    // Step 6: Index changed files into the overlay database
+    if (overlayFiles.size > 0) {
+      const absolutePaths = [...overlayFiles].map((f) =>
+        path.resolve(resolvedRoot, f)
+      );
+      await instance.indexFiles(absolutePaths);
+      // Resolve references using the merged engine so cross-boundary
+      // calls (overlay → base) are correctly linked
+      instance.resolveReferences();
+    }
+
+    return instance;
+  }
+
+  /**
+   * Close the CodeGraph instance and release resources.
+   * In overlay mode, also closes the remote base graph connection.
    */
   close(): void {
     this.unwatch();
     // Release file lock if held
     this.fileLock.release();
     this.db.close();
+    // Close remote graph client if in overlay mode
+    if (this.remoteClient) {
+      this.remoteClient.close();
+      this.remoteClient = null;
+    }
   }
 
   /**

src/overlay/branch-diff.ts

@@ -0,0 +1,144 @@
+/**
+ * Branch Diff Indexer
+ *
+ * Identifies files changed on the current feature branch relative to
+ * a base branch (e.g., main), enabling selective indexing of only the
+ * developer's changes instead of the entire codebase.
+ *
+ * Uses `git diff --name-status` against the merge-base to correctly
+ * handle diverged branches: only files the developer actually touched
+ * appear in the diff, not upstream commits merged into main since the
+ * branch point.
+ */
+
+import { execSync } from 'child_process';
+import { BranchDiffResult } from './types';
+
+/**
+ * Detects files changed on the current branch relative to a base branch.
+ *
+ * Usage:
+ * ```ts
+ * const diff = new BranchDiffIndexer('/path/to/repo');
+ * const result = diff.getChangedFiles('main');
+ * console.log(result.added, result.modified, result.deleted);
+ * ```
+ */
+export class BranchDiffIndexer {
+  private projectRoot: string;
+
+  /**
+   * @param projectRoot - Absolute path to the git repository root
+   */
+  constructor(projectRoot: string) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Get all files changed between the current HEAD and the base branch.
+   *
+   * Uses `git merge-base` to find the common ancestor, then
+   * `git diff --name-status` to categorize changes as added,
+   * modified, or deleted.
+   *
+   * @param baseBranch - Name of the base branch (e.g., 'main')
+   * @returns Categorized diff result with file lists
+   * @throws Error if not inside a git repository or the base branch doesn't exist
+   */
+  getChangedFiles(baseBranch: string): BranchDiffResult {
+    const currentBranch = this.getCurrentBranch();
+    const mergeBase = this.getMergeBase(baseBranch);
+
+    const added: string[] = [];
+    const modified: string[] = [];
+    const deleted: string[] = [];
+
+    // git diff --name-status gives lines like:
+    //   A\tpath/to/new-file.ts
+    //   M\tpath/to/changed-file.ts
+    //   D\tpath/to/removed-file.ts
+    //   R100\told-name.ts\tnew-name.ts
+    const output = this.exec(
+      `git diff --name-status ${mergeBase} HEAD`
+    ).trim();
+
+    if (!output) {
+      return { added, modified, deleted, currentBranch, baseBranch };
+    }
+
+    for (const line of output.split('\n')) {
+      const parts = line.split('\t');
+      const status = parts[0];
+      // For renames (R###), the new file path is in parts[2]
+      const filePath = status?.startsWith('R') ? parts[2] : parts[1];
+
+      if (!status || !filePath) continue;
+
+      if (status.startsWith('A')) {
+        added.push(filePath);
+      } else if (status.startsWith('M') || status.startsWith('R')) {
+        modified.push(filePath);
+      } else if (status.startsWith('D')) {
+        deleted.push(filePath);
+      }
+    }
+
+    return { added, modified, deleted, currentBranch, baseBranch };
+  }
+
+  /**
+   * Get the list of files that need to be indexed for the overlay.
+   *
+   * Returns the union of added and modified files — these are the
+   * files whose graph data differs from the base branch. Deleted
+   * files are excluded (they need to be masked, not indexed).
+   *
+   * @param baseBranch - Name of the base branch
+   * @returns Array of relative file paths to index
+   */
+  getFilesToIndex(baseBranch: string): string[] {
+    const diff = this.getChangedFiles(baseBranch);
+    return [...diff.added, ...diff.modified];
+  }
+
+  /**
+   * Get the name of the currently checked-out branch.
+   *
+   * @returns Branch name, or 'HEAD' if in detached-HEAD state
+   */
+  getCurrentBranch(): string {
+    return this.exec('git rev-parse --abbrev-ref HEAD').trim();
+  }
+
+  /**
+   * Find the merge-base (common ancestor) between the base branch and HEAD.
+   *
+   * @param baseBranch - Name of the base branch
+   * @returns Commit hash of the merge-base
+   * @throws Error if the base branch doesn't exist
+   */
+  getMergeBase(baseBranch: string): string {
+    try {
+      return this.exec(`git merge-base ${baseBranch} HEAD`).trim();
+    } catch {
+      throw new Error(
+        `Cannot find merge-base between '${baseBranch}' and HEAD. ` +
+        `Does the branch '${baseBranch}' exist?`
+      );
+    }
+  }
+
+  /**
+   * Execute a git command in the project root.
+   *
+   * @param command - Shell command to run
+   * @returns stdout as a string
+   */
+  private exec(command: string): string {
+    return execSync(command, {
+      cwd: this.projectRoot,
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  }
+}

src/overlay/index.ts

@@ -0,0 +1,12 @@
+/**
+ * Overlay Module
+ *
+ * Provides the remote graph overlay system for team-friendly code
+ * intelligence. Combines a centrally-built base graph (main branch)
+ * with local feature branch diffs for seamless, merged queries.
+ */
+
+export { RemoteGraphClient } from './remote-client';
+export { BranchDiffIndexer } from './branch-diff';
+export { OverlayQueryEngine } from './overlay-engine';
+export type { RemoteGraphConfig, BranchDiffResult } from './types';