Add git indexing: orchestrator, worker, chunking, and embeddings (#2)

Author: karngyan

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Download dependencies
+        run: go mod download
+
+      - name: Build
+        run: go build ./...
+
+      - name: Test
+        run: go test ./...

cmd/api/main.go

@@ -3,6 +3,8 @@ package main
 import (
 	"github.com/gomantics/semantix/internal/api"
 	"github.com/gomantics/semantix/internal/db"
+	"github.com/gomantics/semantix/internal/domains/indexing"
+	"github.com/gomantics/semantix/internal/libs/openai"
 	"github.com/gomantics/semantix/internal/qdrant"
 	"github.com/gomantics/semantix/pkg/logger"
 	"go.uber.org/fx"
@@ -21,7 +23,9 @@ func main() {
 		fx.Invoke(
 			db.Init,
 			qdrant.Init,
+			openai.Init,
 			api.Run,
+			indexing.Run,
 		),
 		fx.WithLogger(func(l *zap.Logger) fxevent.Logger {
 			return &fxevent.ZapLogger{

cmd/dev/main.go

@@ -3,6 +3,8 @@ package main
 import (
 	"github.com/gomantics/semantix/internal/api"
 	"github.com/gomantics/semantix/internal/db"
+	"github.com/gomantics/semantix/internal/domains/indexing"
+	"github.com/gomantics/semantix/internal/libs/openai"
 	"github.com/gomantics/semantix/internal/qdrant"
 	"github.com/gomantics/semantix/pkg/logger"
 	"go.uber.org/fx"
@@ -21,7 +23,9 @@ func main() {
 		fx.Invoke(
 			db.Init,
 			qdrant.Init,
+			openai.Init,
 			api.Run,
+			indexing.Run,
 		),
 		fx.WithLogger(func(l *zap.Logger) fxevent.Logger {
 			return &fxevent.ZapLogger{

docs/upnext/phase-2-git-indexing.md docs/completed/phase-2-git-indexing.mddocs/upnext/phase-2-git-indexing.md renamed to docs/completed/phase-2-git-indexing.md

@@ -73,18 +73,19 @@ Support the major git hosting providers.
 
 ### 2.3 Tree-sitter Chunking
 
-AST-aware code chunking using chunkx or similar.
+AST-aware code chunking using [`github.com/gomantics/chunkx`](https://github.com/gomantics/chunkx) - our own Go library implementing the CAST algorithm.
 
-- [ ] **Language detection** from file extension and content
+- [ ] **Language detection** - chunkx uses file extension via `languages.*` constants
 
 - [ ] **Chunking strategy**:
   - Functions/methods as primary chunks
   - Classes/structs with their methods
   - Large functions split at logical boundaries
-  - Target: ~500 tokens per chunk
+  - Target: ~500 tokens per chunk via `chunkx.WithMaxSize(500)`
 
-- [ ] **Chunk metadata**:
+- [ ] **Chunk metadata** - map chunkx output to our internal type:
   ```go
+  // chunkx returns []chunkx.Chunk; map to:
   type Chunk struct {
       Content    string
       FilePath   string
@@ -96,14 +97,27 @@ AST-aware code chunking using chunkx or similar.
   }
   ```
 
+- [ ] **Quick example**:
+  ```go
+  import (
+      "github.com/gomantics/chunkx"
+      "github.com/gomantics/chunkx/languages"
+  )
+
+  chunker := chunkx.NewChunker()
+  chunks, err := chunker.Chunk(code,
+      chunkx.WithLanguage(languages.Go),
+      chunkx.WithMaxSize(500))
+  ```
+
 - [ ] **Language support** (priority order):
   - Go, Python, JavaScript/TypeScript
   - Java, Rust, C/C++
   - Ruby, PHP
   - Markdown, YAML, JSON (as text)
 
 **Files to create/modify:**
-- `libs/chunking/chunker.go`
+- `libs/chunking/chunker.go` - thin wrapper around chunkx
 - `domains/chunking/chunker.go` - higher-level orchestration
 
 ---
@@ -327,7 +341,7 @@ POST /v1/workspaces/:wid/repos  (status = pending)
 ## Dependencies
 
 - `github.com/go-git/go-git/v5` - Git operations
-- Tree-sitter Go bindings or `chunkx` CLI
+- `github.com/gomantics/chunkx` - AST-based code chunking (CAST algorithm, 30+ languages)
 - `github.com/sashabaranov/go-openai` - OpenAI client
 - `github.com/qdrant/go-client` - Qdrant client
 

go.mod

@@ -9,10 +9,18 @@ tool (
 )
 
 require (
+	github.com/approvals/go-approval-tests v1.5.0
+	github.com/go-git/go-git/v5 v5.17.0
+	github.com/gomantics/chunkx v0.0.3
 	github.com/jackc/pgx/v5 v5.8.0
 	github.com/labstack/echo/v4 v4.13.4
 	github.com/pressly/goose/v3 v3.27.0
 	github.com/qdrant/go-client v1.16.2
+	github.com/sashabaranov/go-openai v1.41.2
+	github.com/stretchr/testify v1.11.1
+	github.com/testcontainers/testcontainers-go v0.40.0
+	github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0
+	github.com/testcontainers/testcontainers-go/modules/qdrant v0.40.0
 	go.uber.org/fx v1.24.0
 	go.uber.org/zap v1.27.1
 	google.golang.org/grpc v1.79.1
@@ -22,47 +30,97 @@ require (
 	cel.dev/expr v0.25.1 // indirect
 	dario.cat/mergo v1.0.2 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/air-verse/air v1.64.5 // indirect
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/bep/godartsass/v2 v2.5.0 // indirect
 	github.com/bep/golibsass v1.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/cubicdaiya/gonp v1.0.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v28.5.2+incompatible // indirect
+	github.com/docker/go-connections v0.6.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ebitengine/purego v0.9.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.8.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-sql-driver/mysql v1.9.3 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gohugoio/hugo v0.149.1 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/gomantics/cfgx v0.0.7 // indirect
 	github.com/gomantics/sx v0.0.3 // indirect
 	github.com/google/cel-go v0.26.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/compress v1.18.4 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
+	github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
+	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/go-archive v0.1.0 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect
 	github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect
 	github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
 	github.com/pingcap/tidb/pkg/parser v0.0.0-20250324122243-d51e00e5bbf0 // indirect
+	github.com/pjbgf/sha1cd v0.3.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/riza-io/grpc-go v0.2.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.10 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/skeema/knownhosts v1.3.1 // indirect
+	github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/cast v1.9.2 // indirect
 	github.com/spf13/cobra v1.10.1 // indirect
@@ -71,10 +129,19 @@ require (
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/tdewolff/parse/v2 v2.8.3 // indirect
 	github.com/tetratelabs/wazero v1.9.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect
 	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
+	go.opentelemetry.io/otel v1.40.0 // indirect
+	go.opentelemetry.io/otel/metric v1.40.0 // indirect
+	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -85,10 +152,11 @@ require (
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.68.0 // indirect
 	modernc.org/mathutil v1.7.1 // indirect