From eee57f9bc44b55c95070d26d9f993db91fe1130c Mon Sep 17 00:00:00 2001 From: yuanhe Date: Wed, 8 Jul 2026 20:23:15 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20embedded=20PostgreSQL=20=E2=80=94=20zer?= =?UTF-8?q?o-config=20single-binary=20startup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When DATABASE_URL is unset and the server is in dev profile, the server automatically downloads a PostgreSQL binary (via embedded-postgres-go), initializes a data directory under ~/.parsar/postgres, runs goose migrations, and connects — no Docker, no external DB, no .env required. Co-Authored-By: Claude Opus 4.7 (1M context) --- go.mod | 4 +- go.sum | 9 +- server/cmd/server/main.go | 82 ++++++++++++++ server/internal/embeddedpg/embeddedpg.go | 138 +++++++++++++++++++++++ start.sh | 32 ++++++ 5 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 server/internal/embeddedpg/embeddedpg.go create mode 100755 start.sh diff --git a/go.mod b/go.mod index 7206a94..e7f2f79 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.5.2 github.com/bwmarrin/discordgo v0.29.0 + github.com/fergusstrange/embedded-postgres v1.34.0 github.com/go-chi/chi/v5 v5.3.1 github.com/go-chi/httprate v0.16.0 github.com/golang-jwt/jwt/v5 v5.3.1 @@ -30,10 +31,11 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/rogpeppe/go-internal v1.15.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.55.0 // indirect diff --git a/go.sum b/go.sum index 9eac5b6..5f2bf7c 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,13 @@ github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+Eg github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fergusstrange/embedded-postgres v1.34.0 h1:c6RKhPKFsLVU+Tdxsx8q0UxCHsvZZ/iShAnljRBXs6s= +github.com/fergusstrange/embedded-postgres v1.34.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8= @@ -56,6 +57,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.9.8 h1:l9Pijmb/ZMUv6zjMwdN2tP9RB4k/y25HjwZxgBZK0gQ= github.com/larksuite/oapi-sdk-go/v3 v3.9.8/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= @@ -83,6 +86,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I= github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= @@ -103,6 +108,8 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index d49f15f..1de7029 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -24,6 +24,7 @@ import ( "crypto/sha256" "crypto/tls" "crypto/x509" + "database/sql" "errors" "fmt" "net" @@ -31,11 +32,14 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "strconv" "strings" "syscall" "time" + "github.com/pressly/goose/v3" + "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" agentdaemonbinding "github.com/MiniMax-AI-Dev/parsar/server/internal/agentdaemon/binding" agentdaemongateway "github.com/MiniMax-AI-Dev/parsar/server/internal/agentdaemon/gateway" @@ -54,6 +58,7 @@ import ( connagentdaemon "github.com/MiniMax-AI-Dev/parsar/server/internal/connector/agentdaemon" "github.com/MiniMax-AI-Dev/parsar/server/internal/db" "github.com/MiniMax-AI-Dev/parsar/server/internal/db/sqlc" + "github.com/MiniMax-AI-Dev/parsar/server/internal/embeddedpg" "github.com/MiniMax-AI-Dev/parsar/server/internal/dev" gatewaypkg "github.com/MiniMax-AI-Dev/parsar/server/internal/gateway" "github.com/MiniMax-AI-Dev/parsar/server/internal/gateway/channel" @@ -208,6 +213,29 @@ func main() { } } + // Embedded PostgreSQL: when DATABASE_URL is empty and we're in dev + // profile, start an embedded PG subprocess and auto-run migrations. + var embeddedPG *embeddedpg.EmbeddedPG + if strings.TrimSpace(cfg.Database.URL) == "" && cfg.Profile() == config.ProfileDev { + log.Bg().Info("DATABASE_URL not set — starting embedded PostgreSQL", + "data_dir", filepath.Join(cfg.Server.DataDir, "postgres")) + var err error + embeddedPG, err = embeddedpg.Start(cfg.Server.DataDir) + if err != nil { + log.Bg().Error("failed to start embedded PostgreSQL", "error", err) + os.Exit(1) + } + cfg.Database.URL = embeddedpg.ConnectionURL() + log.Bg().Info("embedded PostgreSQL ready", "url", cfg.Database.URL) + + if err := runAutoMigrate(cfg.Database.URL); err != nil { + embeddedPG.Stop() //nolint:errcheck + log.Bg().Error("auto-migration failed", "error", err) + os.Exit(1) + } + log.Bg().Info("auto-migration complete") + } + pool := openPool(cfg.Database.URL) dbStore, auditIngester := buildStore(pool, cfg.Audit.OTLP.FanoutEndpoint) @@ -1009,6 +1037,7 @@ func main() { log.Bg().Error("parsar server stopped", "error", err) drainOTLPReceiver(otlpReceiver) drainAudit(auditIngester) + stopEmbeddedPG(embeddedPG) os.Exit(1) } case <-ctx.Done(): @@ -1024,6 +1053,7 @@ func main() { // ingester (flush buffer) last. drainOTLPReceiver(otlpReceiver) drainAudit(auditIngester) + stopEmbeddedPG(embeddedPG) } } @@ -1079,6 +1109,17 @@ func drainOTLPReceiver(r *otlp.Receiver) { } } +func stopEmbeddedPG(pg *embeddedpg.EmbeddedPG) { + if pg == nil { + return + } + if err := pg.Stop(); err != nil { + log.Bg().Warn("embedded PostgreSQL shutdown error", "error", err) + } else { + log.Bg().Info("embedded PostgreSQL stopped") + } +} + // runtimeHeartbeatSweeperStore is the tiny store surface needed by the // runtime liveness sweeper. Narrower than *store.Store so startup // config tests stay DB-free. @@ -1451,6 +1492,47 @@ func openPool(databaseURL string) *pgxpool.Pool { return pool } +// runAutoMigrate applies goose migrations against the given database URL. +// Used only when embedded PG is active (no external DATABASE_URL). +func runAutoMigrate(databaseURL string) error { + migrationsDir, err := resolveMigrationsDirForEmbed() + if err != nil { + return err + } + sqlDB, err := sql.Open("pgx", databaseURL) + if err != nil { + return fmt.Errorf("auto-migrate: open: %w", err) + } + defer sqlDB.Close() + + if err := sqlDB.Ping(); err != nil { + return fmt.Errorf("auto-migrate: ping: %w", err) + } + goose.SetBaseFS(nil) + goose.SetLogger(goose.NopLogger()) + if err := goose.SetDialect("postgres"); err != nil { + return err + } + return goose.UpContext(context.Background(), sqlDB, migrationsDir) +} + +// resolveMigrationsDirForEmbed looks for the migrations directory in the +// same priority as cmd/migrate: PARSAR_MIGRATIONS_DIR env, then +// ./server/migrations relative to CWD. +func resolveMigrationsDirForEmbed() (string, error) { + if explicit := os.Getenv("PARSAR_MIGRATIONS_DIR"); explicit != "" { + if info, err := os.Stat(explicit); err == nil && info.IsDir() { + return explicit, nil + } + } + candidate := filepath.Join("server", "migrations") + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate, nil + } + cwd, _ := os.Getwd() + return "", fmt.Errorf("auto-migrate: no migrations directory found (CWD=%s); set PARSAR_MIGRATIONS_DIR", cwd) +} + // buildStore constructs the *store.Store and the audit ingester from a // shared pgxpool. Returns (nil, nil) when the pool is unavailable, so // the caller can degrade gracefully without panicking. diff --git a/server/internal/embeddedpg/embeddedpg.go b/server/internal/embeddedpg/embeddedpg.go new file mode 100644 index 0000000..4bd1618 --- /dev/null +++ b/server/internal/embeddedpg/embeddedpg.go @@ -0,0 +1,138 @@ +package embeddedpg + +import ( + "database/sql" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + embeddedpostgres "github.com/fergusstrange/embedded-postgres" + _ "github.com/jackc/pgx/v5/stdlib" +) + +const ( + Port = 15432 + User = "parsar" + Password = "parsar" + Database = "parsar" +) + +type EmbeddedPG struct { + db *embeddedpostgres.EmbeddedPostgres + dataDir string + adopted bool // true when we reused an orphan PG — don't call pg.Stop() +} + +func connectionURL(dbName string) string { + return fmt.Sprintf("postgres://%s:%s@127.0.0.1:%d/%s?sslmode=disable", User, Password, Port, dbName) +} + +func ConnectionURL() string { return connectionURL(Database) } + +// Start launches an embedded PostgreSQL instance under dataDir/postgres. +// If an orphan PG from a previous crash is already listening on the port, +// it is killed first so we get a clean startup with proper lifecycle control. +func Start(dataDir string) (*EmbeddedPG, error) { + pgDir := filepath.Join(dataDir, "postgres") + if err := os.MkdirAll(pgDir, 0o700); err != nil { + return nil, fmt.Errorf("embeddedpg: mkdir %s: %w", pgDir, err) + } + + if isPortListening(Port) { + killOrphanPG(Port) + } + + runtimeDir := filepath.Join(pgDir, "runtime") + dbDataDir := filepath.Join(pgDir, "data") + + pg := embeddedpostgres.NewDatabase( + embeddedpostgres.DefaultConfig(). + Username(User). + Password(Password). + Database("postgres"). + Port(Port). + DataPath(dbDataDir). + RuntimePath(runtimeDir). + BinariesPath(filepath.Join(pgDir, "bin")), + ) + + if err := pg.Start(); err != nil { + return nil, fmt.Errorf("embeddedpg: start: %w", err) + } + + if err := ensureDatabase(); err != nil { + pg.Stop() //nolint:errcheck + return nil, err + } + + return &EmbeddedPG{db: pg, dataDir: pgDir}, nil +} + +func (e *EmbeddedPG) Stop() error { + if e.db == nil { + return nil + } + return e.db.Stop() +} + +func ensureDatabase() error { + conn, err := sql.Open("pgx", connectionURL("postgres")) + if err != nil { + return fmt.Errorf("embeddedpg: open postgres db: %w", err) + } + defer conn.Close() + + var exists bool + err = conn.QueryRow("SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)", Database).Scan(&exists) + if err != nil { + return fmt.Errorf("embeddedpg: check database: %w", err) + } + if exists { + return nil + } + + if _, err := conn.Exec("CREATE DATABASE " + Database); err != nil { + return fmt.Errorf("embeddedpg: create database: %w", err) + } + return nil +} + +func isPortListening(port uint32) bool { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 500*1e6) + if err != nil { + return false + } + conn.Close() + return true +} + +// killOrphanPG finds and kills any postgres process listening on the port. +// This handles the case where a previous `go run` was killed but left the +// PG child process alive. +func killOrphanPG(port uint32) { + out, err := exec.Command("lsof", "-ti", fmt.Sprintf("tcp:%d", port)).Output() + if err != nil || len(out) == 0 { + return + } + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + pid, err := strconv.Atoi(strings.TrimSpace(line)) + if err != nil || pid <= 1 { + continue + } + if p, err := os.FindProcess(pid); err == nil { + p.Signal(os.Interrupt) + } + } + // Brief wait for the process to exit + for range 20 { + if !isPortListening(port) { + return + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..0c3ab71 --- /dev/null +++ b/start.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# One-command Parsar server with embedded PostgreSQL + frontend. +# No Docker, no external DB, no .env file needed. +# +# Usage: ./start.sh [port] +# port defaults to 18080 + +set -e +cd "$(dirname "$0")" + +PORT="${1:-18080}" +WEB_DIR="apps/web/dist" + +# Build frontend if not already built +if [ ! -d "$WEB_DIR" ]; then + echo "Building frontend (first time)..." + (cd apps/web && pnpm install --frozen-lockfile && pnpm build) +fi + +export PARSAR_FEISHU_MOCK=true +export PARSAR_AGENT_DAEMON_OWNER_URL="http://127.0.0.1:${PORT}" +export PARSAR_ADDR=":${PORT}" +export PARSAR_PUBLIC_URL="http://127.0.0.1:${PORT}" +export PARSAR_WEB_DIST="$PWD/$WEB_DIR" + +echo "Starting Parsar on http://127.0.0.1:${PORT}" +echo " Data dir: ~/.parsar" +echo " DB: embedded PostgreSQL (auto-managed)" +echo " Web UI: $PARSAR_WEB_DIST" +echo "" + +exec go run ./server/cmd/server