feat: use tree-sitter symbols in generic analyzer

- extract symbol components via Tree-sitter when available, fallback to legacy parsing
- expand language/extension detection to cover more common languages
- sync pnpm lockfile for tree-sitter runtime deps
- update internal-docs submodule pointer and simplify releasing notes

Author: PatrickSys

RELEASING.md

@@ -8,23 +8,6 @@ This repo publishes `codebase-context` to npm.
 - Release Please bot opens/updates a release PR automatically
 - When you merge the release PR, it publishes to npm with provenance
 
-## Setup (one-time)
-
-**1. Configure npm Trusted Publisher:**
-- Go to https://www.npmjs.com/package/codebase-context/access
-- Add GitHub Actions trusted publisher:
-  - Organization: `PatrickSys`
-  - Repository: `codebase-context`
-  - Workflow: `release-please.yml`
-  - Environment: (leave empty)
-
-That's it! No tokens, no rotation, just OIDC.
-
-**2. Allow Release Please to work:**
-- GitHub Settings > Actions > General
-- Enable "Read and write permissions"
-- Enable "Allow GitHub Actions to create and approve pull requests"
-
 ## Releasing
 
 1. Merge PRs to master
@@ -42,6 +25,7 @@ npm view codebase-context@X.Y.Z --json | jq .dist.attestations
 ## Troubleshooting
 
 If publish fails, check:
+
 - Node 24+ in workflow (required for npm Trusted Publishers)
 - `id-token: write` permission in workflow
 - `registry-url` is set in setup-node

internal-docs

@@ -18,6 +18,7 @@ import {
 } from '../../types/index.js';
 import { createChunksFromCode } from '../../utils/chunking.js';
 import { detectLanguage } from '../../utils/language-detection.js';
+import { extractTreeSitterSymbols, type TreeSitterSymbol } from '../../utils/tree-sitter.js';
 import {
   detectWorkspaceType,
   scanWorkspacePackageJsons,
@@ -100,28 +101,58 @@ export class GenericAnalyzer implements FrameworkAnalyzer {
     let components: CodeComponent[] = [];
     let imports: ImportStatement[] = [];
     let exports: ExportStatement[] = [];
+    let treeSitterGrammar: string | undefined;
+    let usesTreeSitterSymbols = false;
 
     try {
+      const treeSitterResult = await extractTreeSitterSymbols(content, language);
+      if (treeSitterResult && treeSitterResult.symbols.length > 0) {
+        components = this.convertTreeSitterSymbolsToComponents(treeSitterResult.symbols);
+        treeSitterGrammar = treeSitterResult.grammarFile;
+        usesTreeSitterSymbols = true;
+      }
+
       if (language === 'typescript' || language === 'javascript') {
         const parsed = await this.parseJSTSFile(filePath, content, language);
-        components = parsed.components;
         imports = parsed.imports;
         exports = parsed.exports;
+
+        // Keep legacy parser as fallback if Tree-sitter produced nothing.
+        if (components.length === 0) {
+          components = parsed.components;
+          usesTreeSitterSymbols = false;
+          treeSitterGrammar = undefined;
+        }
       } else {
-        // For other languages, use basic line-based parsing
-        components = this.parseGenericFile(content);
+        // For other languages, use regex fallback if Tree-sitter produced nothing.
+        if (components.length === 0) {
+          components = this.parseGenericFile(content);
+        }
       }
     } catch (error) {
       console.warn(`Failed to parse ${filePath}:`, error);
     }
 
+    const metadata: Record<string, any> = {
+      analyzer: this.name,
+      fileSize: content.length,
+      lineCount: content.split('\n').length,
+      chunkStrategy: usesTreeSitterSymbols ? 'tree-sitter-symbol' : 'line-or-component'
+    };
+
+    if (usesTreeSitterSymbols && treeSitterGrammar) {
+      metadata.treeSitterGrammar = treeSitterGrammar;
+      metadata.symbolAware = true;
+    }
+
     // Create chunks
     const chunks = await createChunksFromCode(
       content,
       filePath,
       relativePath,
       language,
-      components
+      components,
+      metadata
     );
 
     return {
@@ -131,15 +162,27 @@ export class GenericAnalyzer implements FrameworkAnalyzer {
       imports,
       exports,
       dependencies: [],
-      metadata: {
-        analyzer: this.name,
-        fileSize: content.length,
-        lineCount: content.split('\n').length
-      },
+      metadata,
       chunks
     };
   }
 
+  private convertTreeSitterSymbolsToComponents(symbols: TreeSitterSymbol[]): CodeComponent[] {
+    return symbols.map((symbol) => ({
+      name: symbol.name,
+      type: symbol.kind,
+      componentType: symbol.kind,
+      startLine: symbol.startLine,
+      endLine: symbol.endLine,
+      metadata: {
+        extraction: 'tree-sitter',
+        nodeType: symbol.nodeType,
+        startIndex: symbol.startIndex,
+        endIndex: symbol.endIndex
+      }
+    }));
+  }
+
   async detectCodebaseMetadata(rootPath: string): Promise<CodebaseMetadata> {
     let projectName = path.basename(rootPath);
     let dependencies: Dependency[] = [];

pnpm-lock.yaml

@@ -46,12 +46,21 @@ const extensionToLanguage: Record<string, string> = {
   '.zsh': 'shellscript',
   '.ps1': 'powershell',
   '.py': 'python',
+  '.pyi': 'python',
   '.rb': 'ruby',
+  '.php': 'php',
   '.java': 'java',
+  '.kt': 'kotlin',
+  '.kts': 'kotlin',
   '.go': 'go',
   '.rs': 'rust',
+  '.cs': 'csharp',
+  '.swift': 'swift',
+  '.scala': 'scala',
   '.c': 'c',
   '.cpp': 'cpp',
+  '.cc': 'cpp',
+  '.cxx': 'cpp',
   '.h': 'c',
   '.hpp': 'cpp'
 };
@@ -121,13 +130,24 @@ const codeExtensions = new Set([
   '.mdx',
   '.graphql',
   '.gql',
+  '.toml',
+  '.xml',
   '.py',
+  '.pyi',
   '.rb',
+  '.php',
   '.java',
+  '.kt',
+  '.kts',
   '.go',
   '.rs',
+  '.cs',
+  '.swift',
+  '.scala',
   '.c',
   '.cpp',
+  '.cc',
+  '.cxx',
   '.h',
   '.hpp'
 ]);