Skip to content

Commit 6244a9b

Browse files
committed
feat: replace regex parsers with tree-sitter AST parsing
Replace all regex-based parsers (TypeScript, Python, Rust, Java) with tree-sitter AST parsers using gotreesitter pure Go runtime. Add new parsers for C#, Ruby, PHP, Kotlin, Swift, and C/C++. - 10 language extractors with full import/export/type extraction - Pure Go, no CGO -- cross-compiles to all platforms - 205 embedded grammars via gotreesitter - Delete old regex parsers (typescript.go, python.go, rust.go, java.go) - All existing tests updated and passing
1 parent cec513c commit 6244a9b

24 files changed

Lines changed: 2102 additions & 547 deletions

DEPENDENCIES.md

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,29 +4,29 @@
44
55
```mermaid
66
graph LR
7-
classDef typescript fill:#c9d1d9,color:#0d1117,stroke:#c9d1d9
8-
classDef python fill:#b1bac4,color:#0d1117,stroke:#b1bac4
97
classDef rust fill:#a5b3bf,color:#0d1117,stroke:#a5b3bf
108
classDef go fill:#e6edf3,color:#0d1117,stroke:#e6edf3
119
classDef javascript fill:#c9d1d9,color:#0d1117,stroke:#c9d1d9
1210
classDef java fill:#8b949e,color:#0d1117,stroke:#8b949e
11+
classDef typescript fill:#c9d1d9,color:#0d1117,stroke:#c9d1d9
12+
classDef python fill:#b1bac4,color:#0d1117,stroke:#b1bac4
13+
internal_summary["internal/summary<br/>AI-powered codebase summaries"]:::go
14+
internal_detect["internal/detect<br/>Framework and tool detection"]:::go
1315
internal_engine["internal/engine<br/>Core orchestration engine"]:::go
14-
internal_parser["internal/parser<br/>Source code parsers"]:::go
15-
npm_bin["npm/bin<br/>Bin"]:::go
16-
internal_cli["internal/cli<br/>Command-line interface"]:::go
17-
internal_monorepo["internal/monorepo<br/>Monorepo detection"]:::go
18-
internal_renderer["internal/renderer<br/>Output renderers"]:::go
16+
internal_git["internal/git<br/>Git integration"]:::go
1917
cmd_stacklit["cmd/stacklit<br/>Stacklit"]:::go
20-
internal_summary["internal/summary<br/>AI-powered codebase summaries"]:::go
18+
internal_cli["internal/cli<br/>Command-line interface"]:::go
19+
internal_mcp["internal/mcp<br/>MCP server for AI agents"]:::go
20+
npm_bin["npm/bin<br/>Bin"]:::go
2121
assets["assets<br/>Static assets"]:::go
22-
internal_git["internal/git<br/>Git integration"]:::go
22+
internal_config["internal/config<br/>Configuration management"]:::go
2323
internal_graph["internal/graph<br/>Dependency graph"]:::go
24-
internal_mcp["internal/mcp<br/>MCP server for AI agents"]:::go
24+
internal_monorepo["internal/monorepo<br/>Monorepo detection"]:::go
25+
internal_renderer["internal/renderer<br/>Output renderers"]:::go
26+
npm["npm<br/>Npm"]:::go
27+
internal_parser["internal/parser<br/>Source code parsers"]:::go
2528
internal_schema["internal/schema<br/>Data schema definitions"]:::go
2629
internal_walker["internal/walker<br/>File system walker"]:::go
27-
npm["npm<br/>Npm"]:::go
28-
internal_config["internal/config<br/>Configuration management"]:::go
29-
internal_detect["internal/detect<br/>Framework and tool detection"]:::go
3030
cmd_stacklit --> internal_cli
3131
internal_cli --> internal_engine
3232
internal_cli --> internal_git

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
require (
1212
github.com/google/jsonschema-go v0.4.2 // indirect
1313
github.com/inconshreveable/mousetrap v1.1.0 // indirect
14+
github.com/odvcencio/gotreesitter v0.13.4 // indirect
1415
github.com/segmentio/asm v1.1.3 // indirect
1516
github.com/segmentio/encoding v0.5.4 // indirect
1617
github.com/spf13/pflag v1.0.9 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
1111
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
1212
github.com/modelcontextprotocol/go-sdk v1.5.0 h1:CHU0FIX9kpueNkxuYtfYQn1Z0slhFzBZuq+x6IiblIU=
1313
github.com/modelcontextprotocol/go-sdk v1.5.0/go.mod h1:gggDIhoemhWs3BGkGwd1umzEXCEMMvAnhTrnbXJKKKA=
14+
github.com/odvcencio/gotreesitter v0.13.4 h1:O/FqOlabRz1Neg6UISx0URtwuN1FQ2eGCc846KHcBbQ=
15+
github.com/odvcencio/gotreesitter v0.13.4/go.mod h1:Sx+iYJBfw5xSWkSttLSuFvguJctlH+ma1BTxZ0MPCqo=
1416
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1517
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1618
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

internal/parser/java.go

Lines changed: 0 additions & 63 deletions
This file was deleted.

internal/parser/java_test.go

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,9 @@ import (
55
"testing"
66
)
77

8-
func TestJavaParserCanParse(t *testing.T) {
9-
p := &JavaParser{}
10-
11-
cases := []struct {
12-
filename string
13-
want bool
14-
}{
15-
{"Main.java", true},
16-
{"UserService.java", true},
17-
{"main.go", false},
18-
{"app.py", false},
19-
{"index.ts", false},
20-
}
21-
22-
for _, tc := range cases {
23-
got := p.CanParse(tc.filename)
24-
if got != tc.want {
25-
t.Errorf("CanParse(%q) = %v, want %v", tc.filename, got, tc.want)
26-
}
27-
}
28-
}
298

309
func TestJavaParserParse_Imports(t *testing.T) {
31-
p := &JavaParser{}
10+
p := &TreeSitterParser{}
3211

3312
content, err := os.ReadFile("../../testdata/java-project/src/Main.java")
3413
if err != nil {
@@ -60,7 +39,7 @@ func TestJavaParserParse_Imports(t *testing.T) {
6039
}
6140

6241
func TestJavaParserParse_Exports(t *testing.T) {
63-
p := &JavaParser{}
42+
p := &TreeSitterParser{}
6443

6544
content, err := os.ReadFile("../../testdata/java-project/src/Main.java")
6645
if err != nil {
@@ -88,7 +67,7 @@ func TestJavaParserParse_Exports(t *testing.T) {
8867
}
8968

9069
func TestJavaParserParse_Entrypoint(t *testing.T) {
91-
p := &JavaParser{}
70+
p := &TreeSitterParser{}
9271

9372
content, err := os.ReadFile("../../testdata/java-project/src/Main.java")
9473
if err != nil {
@@ -106,7 +85,7 @@ func TestJavaParserParse_Entrypoint(t *testing.T) {
10685
}
10786

10887
func TestJavaParserParse_NoEntrypoint(t *testing.T) {
109-
p := &JavaParser{}
88+
p := &TreeSitterParser{}
11089

11190
content := []byte(`package com.example;
11291

internal/parser/parser.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,7 @@ var registry []Parser
3030
func init() {
3131
registry = []Parser{
3232
&GoParser{},
33-
&TypeScriptParser{},
34-
&PythonParser{},
35-
&RustParser{},
36-
&JavaParser{},
33+
&TreeSitterParser{},
3734
&GenericParser{},
3835
}
3936
}

internal/parser/python.go

Lines changed: 0 additions & 104 deletions
This file was deleted.

internal/parser/python_test.go

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,9 @@ import (
66
"testing"
77
)
88

9-
func TestPythonParserCanParse(t *testing.T) {
10-
p := &PythonParser{}
11-
12-
cases := []struct {
13-
filename string
14-
want bool
15-
}{
16-
{"app.py", true},
17-
{"__init__.py", true},
18-
{"setup.py", true},
19-
{"main.go", false},
20-
{"index.ts", false},
21-
{"README.md", false},
22-
}
23-
24-
for _, tc := range cases {
25-
got := p.CanParse(tc.filename)
26-
if got != tc.want {
27-
t.Errorf("CanParse(%q) = %v, want %v", tc.filename, got, tc.want)
28-
}
29-
}
30-
}
319

3210
func TestPythonParserParse_Imports(t *testing.T) {
33-
p := &PythonParser{}
11+
p := &TreeSitterParser{}
3412

3513
content, err := os.ReadFile("../../testdata/python-project/app.py")
3614
if err != nil {
@@ -62,7 +40,7 @@ func TestPythonParserParse_Imports(t *testing.T) {
6240
}
6341

6442
func TestPythonParserParse_Exports(t *testing.T) {
65-
p := &PythonParser{}
43+
p := &TreeSitterParser{}
6644

6745
content, err := os.ReadFile("../../testdata/python-project/app.py")
6846
if err != nil {
@@ -98,7 +76,7 @@ func TestPythonParserParse_Exports(t *testing.T) {
9876
}
9977

10078
func TestPythonParserParse_Entrypoint(t *testing.T) {
101-
p := &PythonParser{}
79+
p := &TreeSitterParser{}
10280

10381
content, err := os.ReadFile("../../testdata/python-project/app.py")
10482
if err != nil {
@@ -116,7 +94,7 @@ func TestPythonParserParse_Entrypoint(t *testing.T) {
11694
}
11795

11896
func TestPythonParserParse_NoEntrypoint(t *testing.T) {
119-
p := &PythonParser{}
97+
p := &TreeSitterParser{}
12098

12199
content := []byte(`def helper():
122100
pass
@@ -136,7 +114,7 @@ class MyClass:
136114
}
137115

138116
func TestPythonParserParse_LineCount(t *testing.T) {
139-
p := &PythonParser{}
117+
p := &TreeSitterParser{}
140118

141119
content := []byte("import os\nimport sys\n\nprint('hello')\n")
142120
info, err := p.Parse("script.py", content)

0 commit comments

Comments
 (0)