Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions __tests__/extraction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ describe('Language Detection', () => {
expect(detectLanguage('main.py')).toBe('python');
});

it('should detect Bash and shell scripts', () => {
expect(detectLanguage('scripts/deploy.sh')).toBe('bash');
expect(detectLanguage('scripts/deploy.bash')).toBe('bash');
expect(detectLanguage('test/integration.bats')).toBe('bash');
expect(detectLanguage('bin/deploy', '#!/usr/bin/env bash\nset -euo pipefail\n')).toBe('bash');
expect(detectLanguage('bin/run', '#!/bin/sh\necho ok\n')).toBe('bash');
});

it('should detect Go files', () => {
expect(detectLanguage('main.go')).toBe('go');
});
Expand Down Expand Up @@ -184,6 +192,7 @@ class ENGINE_API UNetConnectionRepControl : public UObject
describe('Language Support', () => {
it('should report supported languages', () => {
expect(isLanguageSupported('typescript')).toBe(true);
expect(isLanguageSupported('bash')).toBe(true);
expect(isLanguageSupported('python')).toBe(true);
expect(isLanguageSupported('go')).toBe(true);
expect(isLanguageSupported('unknown')).toBe(false);
Expand All @@ -193,6 +202,7 @@ describe('Language Support', () => {
const languages = getSupportedLanguages();
expect(languages).toContain('typescript');
expect(languages).toContain('javascript');
expect(languages).toContain('bash');
expect(languages).toContain('python');
expect(languages).toContain('go');
expect(languages).toContain('rust');
Expand All @@ -208,6 +218,71 @@ describe('Language Support', () => {
});
});

describe('Bash Extraction', () => {
it('should extract shell functions, variables, command calls, and sourced scripts', () => {
const code = `#!/usr/bin/env bash
set -euo pipefail

readonly ROOT_DIR="$(cd "$(dirname "\${BASH_SOURCE[0]}")" && pwd)"
export IMAGE_TAG="latest"

source ./env.sh

log() {
printf '%s\n' "$1"
}

deploy() {
local target="\${1:-dev}"
./scripts/build.sh "$target"
bash ./scripts/migrate.sh
kubectl apply -f k8s/
log "deployed"
}

deploy "$@"
`;

const result = extractFromSource('scripts/deploy.sh', code);

expect(result.nodes.find((n) => n.kind === 'function' && n.name === 'log')).toBeDefined();
expect(result.nodes.find((n) => n.kind === 'function' && n.name === 'deploy')).toBeDefined();
expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'ROOT_DIR')).toBeDefined();
expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'IMAGE_TAG')?.isExported).toBe(true);
expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'target')).toBeDefined();

const calls = result.unresolvedReferences.filter((r) => r.referenceKind === 'calls').map((r) => r.referenceName);
expect(calls).toContain('./scripts/build.sh');
expect(calls).toContain('bash');
expect(calls).toContain('kubectl');
expect(calls).toContain('log');
expect(calls).toContain('deploy');

const imports = result.nodes.filter((n) => n.kind === 'import').map((n) => n.name);
expect(imports).toEqual(['./env.sh', './scripts/build.sh', './scripts/migrate.sh']);

const importRefs = result.unresolvedReferences.filter((r) => r.referenceKind === 'imports').map((r) => r.referenceName);
expect(importRefs).toEqual(['./env.sh', './scripts/build.sh', './scripts/migrate.sh']);
});

it('should include extensionless Bash shebang scripts in directory scans', () => {
const tempDir = createTempDir();
try {
const binDir = path.join(tempDir, 'bin');
fs.mkdirSync(binDir, { recursive: true });
fs.writeFileSync(path.join(binDir, 'deploy'), '#!/usr/bin/env bash\ndeploy() { echo ok; }\n');
fs.writeFileSync(path.join(binDir, 'notes'), 'plain text\n');

const files = scanDirectory(tempDir);

expect(files).toContain('bin/deploy');
expect(files).not.toContain('bin/notes');
} finally {
cleanupTempDir(tempDir);
}
});
});

describe('Nix Extraction', () => {
it('should distinguish Nix variable and function bindings', () => {
const code = `
Expand Down
22 changes: 22 additions & 0 deletions __tests__/resolution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,28 @@ describe('Resolution Module', () => {
expect(result).toBe('src/helpers.ts');
});

it('should resolve extensionless Bash imports to Bats scripts', () => {
const context: ResolutionContext = {
getNodesInFile: () => [],
getNodesByName: () => [],
getNodesByQualifiedName: () => [],
getNodesByKind: () => [],
fileExists: (p) => p === 'test/helpers/assertions.bats',
readFile: () => null,
getProjectRoot: () => '',
getAllFiles: () => ['test/helpers/assertions.bats'],
};

const result = resolveImportPath(
'./helpers/assertions',
'test/integration.bats',
'bash',
context
);

expect(result).toBe('test/helpers/assertions.bats');
});

it('should extract JS/TS import mappings', () => {
const content = `
import { foo } from './foo';
Expand Down
32 changes: 32 additions & 0 deletions __tests__/sync.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,38 @@ import { execFileSync } from 'child_process';
import CodeGraph from '../src/index';

describe('Sync Module', () => {
describe('Bash sync support', () => {
let testDir: string;
let cg: CodeGraph;

beforeEach(() => {
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-sync-bash-'));
cg = CodeGraph.initSync(testDir);
});

afterEach(() => {
if (cg) {
cg.destroy();
}
if (fs.existsSync(testDir)) {
fs.rmSync(testDir, { recursive: true, force: true });
}
});

it('should sync newly added extensionless Bash shebang scripts', async () => {
const binDir = path.join(testDir, 'bin');
fs.mkdirSync(binDir, { recursive: true });
fs.writeFileSync(path.join(binDir, 'deploy'), '#!/usr/bin/env bash\ndeploy() {\n echo ok\n}\n');

const result = await cg.sync();
const nodes = cg.getNodesInFile('bin/deploy');

expect(result.filesAdded).toBe(1);
expect(cg.getFile('bin/deploy')?.language).toBe('bash');
expect(nodes.some((n) => n.kind === 'function' && n.name === 'deploy')).toBe(true);
});
});

describe('Sync Functionality', () => {
let testDir: string;
let cg: CodeGraph;
Expand Down
24 changes: 24 additions & 0 deletions src/extraction/grammars.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const WASM_GRAMMAR_FILES: Record<GrammarLanguage, string> = {
tsx: 'tree-sitter-tsx.wasm',
javascript: 'tree-sitter-javascript.wasm',
jsx: 'tree-sitter-javascript.wasm',
bash: 'tree-sitter-bash.wasm',
python: 'tree-sitter-python.wasm',
go: 'tree-sitter-go.wasm',
rust: 'tree-sitter-rust.wasm',
Expand Down Expand Up @@ -71,6 +72,9 @@ export const EXTENSION_MAP: Record<string, Language> = {
'.xsjs': 'javascript',
'.xsjslib': 'javascript',
'.jsx': 'jsx',
'.sh': 'bash',
'.bash': 'bash',
'.bats': 'bash',
'.py': 'python',
'.pyw': 'python',
'.go': 'go',
Expand Down Expand Up @@ -213,6 +217,24 @@ export function isErlangAppFile(filePath: string): boolean {
return /\.app(?:\.src)?$/i.test(filePath);
}

/**
* Extensionless executable scripts can still be Bash. Keep this deliberately
* narrow so arbitrary extensionless docs/binaries are not treated as source by
* path alone; callers that have content available should pair it with
* `hasBashShebang`.
*/
export function isPotentialBashShebangPath(filePath: string): boolean {
const base = path.basename(filePath);
return base.length > 0 && !base.includes('.');
}

/** True when the first line names bash or POSIX sh as the interpreter. */
export function hasBashShebang(source: string): boolean {
const firstNewline = source.indexOf('\n');
const firstLine = (firstNewline >= 0 ? source.slice(0, firstNewline) : source).trim();
return /^#!.*(?:^|[\/\s])(?:bash|sh)(?:\s|$)/.test(firstLine);
}

/**
* Play Framework routes file: the extensionless `conf/routes` (and included
* `conf/*.routes`). No grammar — route extraction is done by the Play framework
Expand Down Expand Up @@ -377,6 +399,7 @@ export function detectLanguage(filePath: string, source?: string, overrides?: Re
// OTP `.app`/`.app.src` resource files — Erlang terms the grammar parses as
// top-level expressions (last-dot ext `.src` is too generic for the map).
if (isErlangAppFile(filePath)) return 'erlang';
if (source && isPotentialBashShebangPath(filePath) && hasBashShebang(source)) return 'bash';
const lang = (overrides && overrides[ext]) || EXTENSION_MAP[ext] || 'unknown';

// .h files could be C, C++, or Objective-C — check source content
Expand Down Expand Up @@ -510,6 +533,7 @@ export function getLanguageDisplayName(language: Language): string {
javascript: 'JavaScript',
tsx: 'TypeScript (TSX)',
jsx: 'JavaScript (JSX)',
bash: 'Bash / shell',
python: 'Python',
go: 'Go',
rust: 'Rust',
Expand Down
52 changes: 42 additions & 10 deletions src/extraction/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
import { QueryBuilder } from '../db/queries';
import { extractFromSource } from './tree-sitter';
import { ParseWorkerPool, resolveParsePoolSize } from './parse-pool';
import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages } from './grammars';
import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, hasBashShebang, isPotentialBashShebangPath } from './grammars';
import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns } from '../project-config';
import { isCodeGraphDataDir } from '../directory';
import { logDebug, logWarn } from '../errors';
Expand Down Expand Up @@ -117,6 +117,34 @@ export function hashContent(content: string): string {
return crypto.createHash('sha256').update(content).digest('hex');
}

/** Read only the shebang-sized prefix needed to classify extensionless shell scripts. */
function fileHasBashShebang(absPath: string): boolean {
let fd: number | null = null;
try {
fd = fs.openSync(absPath, 'r');
const buf = Buffer.alloc(256);
const bytes = fs.readSync(fd, buf, 0, buf.length, 0);
return hasBashShebang(buf.subarray(0, bytes).toString('utf-8'));
} catch {
return false;
} finally {
if (fd !== null) {
try { fs.closeSync(fd); } catch { /* ignore */ }
}
}
}

/** Source-file predicate with content-backed Bash shebang support. */
function isSourceFileAtPath(absPath: string, relativePath: string, overrides?: Record<string, Language>): boolean {
if (isSourceFile(relativePath, overrides)) return true;
return isPotentialBashShebangPath(relativePath) && fileHasBashShebang(absPath);
}

function detectLanguageForGrammarLoad(filePath: string, overrides?: Record<string, Language>): Language {
const lang = detectLanguage(filePath, undefined, overrides);
return lang === 'unknown' && isPotentialBashShebangPath(filePath) ? 'bash' : lang;
}

/**
* Skip files larger than this (bytes). Generated bundles, minified JS, and
* vendored blobs blow the WASM heap and the worker-recycle budget for no useful
Expand Down Expand Up @@ -424,7 +452,7 @@ function collectIncludedFiles(
if (defaults.ignores(rel)) return;
if (!include.ignores(rel)) return;
if (exclude && exclude.ignores(rel)) return;
if (!isSourceFile(rel, overrides)) return;
if (!isSourceFileAtPath(abs, rel, overrides)) return;
out.add(rel);
}
};
Expand Down Expand Up @@ -1112,7 +1140,8 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, over
}

const filePath = normalizePath(prefix + rel);
if (!isSourceFile(filePath, overrides)) continue;
const sourceFile = isSourceFileAtPath(path.join(repoDir, rel), filePath, overrides);
if (!sourceFile && !(statusCode.includes('D') && isPotentialBashShebangPath(filePath))) continue;

if (statusCode.includes('D')) {
// Deletions stay unfiltered: getChangedFiles acts on one only when the
Expand Down Expand Up @@ -1173,7 +1202,7 @@ export function scanDirectory(
const files: string[] = [];
let count = 0;
for (const filePath of gitFiles) {
if (isSourceFile(filePath, overrides)) {
if (isSourceFileAtPath(path.join(rootDir, filePath), filePath, overrides)) {
files.push(filePath);
count++;
onProgress?.(count, filePath);
Expand Down Expand Up @@ -1202,7 +1231,7 @@ export async function scanDirectoryAsync(
const files: string[] = [];
let count = 0;
for (const filePath of gitFiles) {
if (isSourceFile(filePath, overrides)) {
if (isSourceFileAtPath(path.join(rootDir, filePath), filePath, overrides)) {
files.push(filePath);
count++;
onProgress?.(count, filePath);
Expand Down Expand Up @@ -1306,7 +1335,7 @@ function scanDirectoryWalk(
walk(fullPath, active);
}
} else if (stat.isFile()) {
if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
if (!isIgnored(fullPath, false, active) && isSourceFileAtPath(fullPath, relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
Expand All @@ -1323,7 +1352,7 @@ function scanDirectoryWalk(
walk(fullPath, active);
}
} else if (entry.isFile()) {
if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
if (!isIgnored(fullPath, false, active) && isSourceFileAtPath(fullPath, relativePath, overrides)) {
files.push(relativePath);
count++;
onProgress?.(count, relativePath);
Expand Down Expand Up @@ -1523,8 +1552,11 @@ export class ExtractionOrchestrator {
});
await new Promise(resolve => setImmediate(resolve));

// Detect needed languages and load grammars in the parse worker
const neededLanguages = [...new Set(files.map((f) => detectLanguage(f, undefined, overrides)))];
// Detect needed languages and load grammars in the parse worker. Extensionless
// Bash scripts were admitted by a content-backed shebang check during scan,
// but content is not available here yet, so include the Bash grammar for
// extensionless scanned files whose path alone still reports unknown.
const neededLanguages = [...new Set(files.map((f) => detectLanguageForGrammarLoad(f, overrides)))];
// .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
neededLanguages.push('cpp');
Expand Down Expand Up @@ -2290,7 +2322,7 @@ export class ExtractionOrchestrator {
// Load only grammars needed for changed files
if (filesToIndex.length > 0) {
const overrides = loadExtensionOverrides(this.rootDir);
const neededLanguages = [...new Set(filesToIndex.map((f) => detectLanguage(f, undefined, overrides)))];
const neededLanguages = [...new Set(filesToIndex.map((f) => detectLanguageForGrammarLoad(f, overrides)))];
// .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
neededLanguages.push('cpp');
Expand Down
Loading