From 7ff4ad026cdc3870661170a2f53398747352bb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 12 May 2026 17:08:08 +0300 Subject: [PATCH 1/3] fix(db): persist CPE in postgres and mysql (#2487) Adds cpe column to both SQL schemas, idempotent migration for existing tables, and JSON binding in InsertBatch. --- internal/db/mysql.go | 35 +++++++++++++++++++++++++++++++++-- internal/db/postgres.go | 16 +++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index 96c2f391a..c1773b93b 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -151,6 +151,9 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { -- Trace trace JSON, + -- CPE (Common Platform Enumeration) + cpe JSON, + INDEX idx_timestamp (timestamp), INDEX idx_url (url(255)), INDEX idx_host (host), @@ -163,9 +166,33 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { return fmt.Errorf("failed to create schema: %w", err) } + if err := m.ensureColumn(ctx, "cpe", "JSON"); err != nil { + return fmt.Errorf("failed to ensure cpe column: %w", err) + } + return nil } +func (m *mysqlDatabase) ensureColumn(ctx context.Context, column, definition string) error { + var count int + err := m.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`, + m.cfg.TableName, column, + ).Scan(&count) + if err != nil { + return err + } + if count > 0 { + return nil + } + _, err = m.db.ExecContext(ctx, + fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", + quoteIdentifier(m.cfg.TableName), quoteIdentifier(column), definition), + ) + return err +} + func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result) error { if len(results) == 0 { return nil @@ -193,7 +220,8 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result words, `+"`lines`"+`, header, extracts, extract_regex, chain, chain_status_codes, headless_body, screenshot_bytes, screenshot_path, screenshot_path_rel, stored_response_path, - knowledgebase, link_request, trace + knowledgebase, link_request, trace, + cpe ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -205,7 +233,8 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ? + ?, ?, ?, + ? )`, tableName) stmt, err := tx.PrepareContext(ctx, query) @@ -236,6 +265,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result kbJSON, _ := json.Marshal(r.KnowledgeBase) linkReqJSON, _ := json.Marshal(r.LinkRequest) traceJSON, _ := json.Marshal(r.Trace) + cpeJSON, _ := json.Marshal(r.CPE) _, err = stmt.ExecContext(ctx, r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, @@ -249,6 +279,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result chainJSON, chainStatusJSON, r.HeadlessBody, r.ScreenshotBytes, r.ScreenshotPath, r.ScreenshotPathRel, r.StoredResponsePath, kbJSON, linkReqJSON, traceJSON, + cpeJSON, ) if err != nil { return fmt.Errorf("failed to insert result: %w", err) diff --git a/internal/db/postgres.go b/internal/db/postgres.go index 0eca9831a..df504b1f3 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -150,15 +150,21 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { link_request JSONB, -- Trace - trace JSONB + trace JSONB, + + -- CPE (Common Platform Enumeration) + cpe JSONB ); + ALTER TABLE %s ADD COLUMN IF NOT EXISTS cpe JSONB; + CREATE INDEX IF NOT EXISTS %s ON %s(timestamp DESC); CREATE INDEX IF NOT EXISTS %s ON %s(url); CREATE INDEX IF NOT EXISTS %s ON %s(host); CREATE INDEX IF NOT EXISTS %s ON %s(status_code); CREATE INDEX IF NOT EXISTS %s ON %s USING GIN(tech); `, + tableName, tableName, idxTimestamp, tableName, idxURL, tableName, @@ -201,7 +207,8 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res words, lines, header, extracts, extract_regex, chain, chain_status_codes, headless_body, screenshot_bytes, screenshot_path, screenshot_path_rel, stored_response_path, - knowledgebase, link_request, trace + knowledgebase, link_request, trace, + cpe ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, @@ -213,7 +220,8 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, - $60, $61, $62 + $60, $61, $62, + $63 )`, tableName) stmt, err := tx.PrepareContext(ctx, query) @@ -235,6 +243,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res kbJSON, _ := json.Marshal(r.KnowledgeBase) linkReqJSON, _ := json.Marshal(r.LinkRequest) traceJSON, _ := json.Marshal(r.Trace) + cpeJSON, _ := json.Marshal(r.CPE) _, err = stmt.ExecContext(ctx, r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, @@ -248,6 +257,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res chainJSON, pq.Array(r.ChainStatusCodes), r.HeadlessBody, r.ScreenshotBytes, r.ScreenshotPath, r.ScreenshotPathRel, r.StoredResponsePath, kbJSON, linkReqJSON, traceJSON, + cpeJSON, ) if err != nil { return fmt.Errorf("failed to insert result: %w", err) From f5455605baeb25026fa568f28df5ebfb0e0f6f77 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 13 May 2026 21:05:22 +0200 Subject: [PATCH 2/3] adding comment note --- internal/db/mysql.go | 5 +++++ internal/db/postgres.go | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index c1773b93b..59432fab6 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -166,6 +166,11 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { return fmt.Errorf("failed to create schema: %w", err) } + // Back-compat for databases whose schema was created before CPE support. + // New installs already get this column via the CREATE TABLE above; this + // path only matters for in-place upgrades. + // TODO: replace these ad-hoc ensureColumn calls with a proper migration + // framework (e.g. golang-migrate / goose) once more schema changes accumulate. if err := m.ensureColumn(ctx, "cpe", "JSON"); err != nil { return fmt.Errorf("failed to ensure cpe column: %w", err) } diff --git a/internal/db/postgres.go b/internal/db/postgres.go index df504b1f3..24cbb70bf 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -156,6 +156,12 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { cpe JSONB ); + -- Back-compat for databases whose schema was created before CPE support. + -- New installs already get this column via the CREATE TABLE above; this + -- statement only matters for in-place upgrades. + -- TODO: replace these ad-hoc ALTER TABLE statements with a proper + -- migration framework (e.g. golang-migrate / goose) once more schema + -- changes accumulate. ALTER TABLE %s ADD COLUMN IF NOT EXISTS cpe JSONB; CREATE INDEX IF NOT EXISTS %s ON %s(timestamp DESC); From af3a1a0bc0f978bd402b36b48a334d62e9fed952 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 13 May 2026 21:14:50 +0200 Subject: [PATCH 3/3] fixing lint --- go.mod | 2 +- go.sum | 4 ++-- runner/options.go | 9 ++++----- runner/runner.go | 27 ++++++++++++--------------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index a043013d5..455da34a3 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.3.10 github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.107 - github.com/projectdiscovery/utils v0.10.1 + github.com/projectdiscovery/utils v0.11.0 github.com/projectdiscovery/wappalyzergo v0.2.79 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index af9c88719..3d0791933 100644 --- a/go.sum +++ b/go.sum @@ -369,8 +369,8 @@ github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiB github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZEQRGAoYGP6F5M= github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY= -github.com/projectdiscovery/utils v0.10.1 h1:9luYfL7PpN1L/cLO4bAES4+ltDaEBKOUnRiTn920XfM= -github.com/projectdiscovery/utils v0.10.1/go.mod h1:x3jGS2YIxnUYxlpB9HWBKf0k+AE83nYCGRX/YStC8G8= +github.com/projectdiscovery/utils v0.11.0 h1:CxImZSRyj9spy1wpB9HKJopr5MsIPm2r5iS8uyhAMoQ= +github.com/projectdiscovery/utils v0.11.0/go.mod h1:q2mZngH1s4WDO3knYxG7iyP1KcxoRSORJCWSpCKFc1s= github.com/projectdiscovery/wappalyzergo v0.2.79 h1:LBAd+nA+yv2Hf//q2TlODLRDkaaqzWlCaIPcwYyHZcU= github.com/projectdiscovery/wappalyzergo v0.2.79/go.mod h1:hRsnKNleH693FFJsBOD5NMUDbxw/Q94f0Oq2OV04Q6M= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/runner/options.go b/runner/options.go index 50a08e773..4d1cc71c9 100644 --- a/runner/options.go +++ b/runner/options.go @@ -801,11 +801,10 @@ func (options *Options) ValidateOptions() error { var resolvers []string for _, resolver := range options.Resolvers { if fileutil.FileExists(resolver) { - chFile, err := fileutil.ReadFile(resolver) - if err != nil { - return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver) - } - for line := range chFile { + for line, err := range fileutil.Lines(resolver) { + if err != nil { + return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver) + } line = strings.TrimSpace(line) if line != "" && strings.Contains(line, ",") { for item := range strings.SplitSeq(line, ",") { diff --git a/runner/runner.go b/runner/runner.go index 0b7d4597a..af9db5661 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -743,11 +743,10 @@ func (r *Runner) streamInput() (chan string, error) { return } } else { - fchan, err := fileutil.ReadFile(r.options.InputFile) - if err != nil { - return - } - for item := range fchan { + for item, err := range fileutil.Lines(r.options.InputFile) { + if err != nil { + return + } if r.options.SkipDedupe || r.testAndSet(item) { if !trySend(item) { return @@ -761,11 +760,10 @@ func (r *Runner) streamInput() (chan string, error) { gologger.Fatal().Msgf("No input provided: %s", err) } for _, file := range files { - fchan, err := fileutil.ReadFile(file) - if err != nil { - return - } - for item := range fchan { + for item, err := range fileutil.Lines(file) { + if err != nil { + return + } if r.options.SkipDedupe || r.testAndSet(item) { if !trySend(item) { return @@ -775,11 +773,10 @@ func (r *Runner) streamInput() (chan string, error) { } } if fileutil.HasStdin() { - fchan, err := fileutil.ReadFileWithReader(os.Stdin) - if err != nil { - return - } - for item := range fchan { + for item, err := range fileutil.LinesReader(os.Stdin) { + if err != nil { + return + } if r.options.SkipDedupe || r.testAndSet(item) { if !trySend(item) { return