From 81461d35e8f4e0d5283ebcd854b1ee35228df58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 6 Jan 2026 13:22:43 +0300 Subject: [PATCH] feat: add passive CPE and WordPress detection Add support for passive detection of CPE (Common Platform Enumeration) identifiers and WordPress plugins/themes using awesome-search-queries. CPE Detection (-cpe flag): - Matches response title, body, and favicon hash against patterns - Extracts product, vendor, and generates CPE 2.3 identifiers - Uses patterns from Shodan, FOFA, Google dorks WordPress Detection (-wp flag): - Detects plugins via /wp-content/plugins/[name]/ patterns - Detects themes via /wp-content/themes/[name]/ patterns - Validates against known plugins/themes list New CLI flags in PROBES group: - -cpe: display CPE based on awesome-search-queries - -wp, -wordpress: display WordPress plugins and themes Both are automatically included in JSON/CSV output. Closes #1975 --- README.md | 6 +- go.mod | 1 + go.sum | 2 + runner/cpe.go | 225 ++++++++++++++++++++++++++++++++++++++++++++ runner/options.go | 8 ++ runner/runner.go | 61 ++++++++++++ runner/types.go | 2 + runner/wordpress.go | 118 +++++++++++++++++++++++ 8 files changed, 421 insertions(+), 2 deletions(-) create mode 100644 runner/cpe.go create mode 100644 runner/wordpress.go diff --git a/README.md b/README.md index 978b4a16..283e6691 100644 --- a/README.md +++ b/README.md @@ -110,9 +110,11 @@ PROBES: -title display page title -bp, -body-preview display first N characters of response body (default 100) -server, -web-server display server name - -td, -tech-detect display technology in use based on wappalyzer dataset + -td, -tech-detect display technology in use based on wappalyzer dataset -cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection - -method display http request method + -cpe display CPE (Common Platform Enumeration) based on awesome-search-queries + -wp, -wordpress display WordPress plugins and themes + -method display http request method -ws, -websocket display server using websocket -ip display host ip -cname display host cname diff --git a/go.mod b/go.mod index f684ff43..1ac2fdd6 100644 --- a/go.mod +++ b/go.mod @@ -128,6 +128,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect diff --git a/go.sum b/go.sum index 114df462..0188b44c 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng= +github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.2.17 h1:Ah7KIft60ZiE6etGuX/63HiDJu0C7szhEwYTQugVorU= diff --git a/runner/cpe.go b/runner/cpe.go new file mode 100644 index 00000000..76444fa9 --- /dev/null +++ b/runner/cpe.go @@ -0,0 +1,225 @@ +package runner + +import ( + "encoding/json" + "fmt" + "strings" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type CPEInfo struct { + Product string `json:"product,omitempty"` + Vendor string `json:"vendor,omitempty"` + CPE string `json:"cpe,omitempty"` +} + +type CPEDetector struct { + titlePatterns map[string][]CPEInfo + bodyPatterns map[string][]CPEInfo + faviconPatterns map[string][]CPEInfo +} + +type rawQuery struct { + Name string `json:"name"` + Vendor json.RawMessage `json:"vendor"` + Type string `json:"type"` + Engines []rawEngine `json:"engines"` +} + +type rawEngine struct { + Platform string `json:"platform"` + Queries []string `json:"queries"` +} + +func NewCPEDetector() (*CPEDetector, error) { + data, err := awesomesearchqueries.GetQueries() + if err != nil { + return nil, fmt.Errorf("failed to load queries: %w", err) + } + + var queries []rawQuery + if err := json.Unmarshal(data, &queries); err != nil { + return nil, fmt.Errorf("failed to parse queries: %w", err) + } + + detector := &CPEDetector{ + titlePatterns: make(map[string][]CPEInfo), + bodyPatterns: make(map[string][]CPEInfo), + faviconPatterns: make(map[string][]CPEInfo), + } + + for _, q := range queries { + vendor := parseVendor(q.Vendor) + info := CPEInfo{ + Product: q.Name, + Vendor: vendor, + CPE: generateCPE(vendor, q.Name), + } + + for _, engine := range q.Engines { + for _, query := range engine.Queries { + detector.extractPattern(query, info) + } + } + } + + return detector, nil +} + +func parseVendor(raw json.RawMessage) string { + var vendorStr string + if err := json.Unmarshal(raw, &vendorStr); err == nil { + return vendorStr + } + + var vendorSlice []string + if err := json.Unmarshal(raw, &vendorSlice); err == nil && len(vendorSlice) > 0 { + return vendorSlice[0] + } + + return "" +} + +func generateCPE(vendor, product string) string { + if vendor == "" || product == "" { + return "" + } + return fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*", + strings.ToLower(strings.ReplaceAll(vendor, " ", "_")), + strings.ToLower(strings.ReplaceAll(product, " ", "_"))) +} + +func (d *CPEDetector) extractPattern(query string, info CPEInfo) { + query = strings.TrimSpace(query) + + titlePrefixes := []string{ + "http.title:", + "title=", + "title==", + "intitle:", + "title:", + "title='", + `title="`, + } + + for _, prefix := range titlePrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + pattern = strings.TrimPrefix(pattern, prefix[:len(prefix)-1]) + if pattern != "" { + pattern = strings.ToLower(pattern) + d.titlePatterns[pattern] = appendUnique(d.titlePatterns[pattern], info) + } + return + } + } + + bodyPrefixes := []string{ + "http.html:", + "body=", + "body==", + "intext:", + } + + for _, prefix := range bodyPrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + if pattern != "" { + pattern = strings.ToLower(pattern) + d.bodyPatterns[pattern] = appendUnique(d.bodyPatterns[pattern], info) + } + return + } + } + + faviconPrefixes := []string{ + "http.favicon.hash:", + "icon_hash=", + "icon_hash==", + } + + for _, prefix := range faviconPrefixes { + if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) { + pattern := extractQuotedValue(strings.TrimPrefix(query, prefix)) + if pattern != "" { + d.faviconPatterns[pattern] = appendUnique(d.faviconPatterns[pattern], info) + } + return + } + } +} + +func extractQuotedValue(s string) string { + s = strings.TrimSpace(s) + + if len(s) >= 2 { + if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') { + s = s[1 : len(s)-1] + } + } + + if idx := strings.Index(s, "\" ||"); idx > 0 { + s = s[:idx] + } + if idx := strings.Index(s, "' ||"); idx > 0 { + s = s[:idx] + } + + return strings.TrimSpace(s) +} + +func appendUnique(slice []CPEInfo, info CPEInfo) []CPEInfo { + for _, existing := range slice { + if existing.Product == info.Product && existing.Vendor == info.Vendor { + return slice + } + } + return append(slice, info) +} + +func (d *CPEDetector) Detect(title, body, faviconHash string) []CPEInfo { + seen := make(map[string]bool) + var results []CPEInfo + + titleLower := strings.ToLower(title) + bodyLower := strings.ToLower(body) + + for pattern, infos := range d.titlePatterns { + if strings.Contains(titleLower, pattern) { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + for pattern, infos := range d.bodyPatterns { + if strings.Contains(bodyLower, pattern) { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + if faviconHash != "" { + if infos, ok := d.faviconPatterns[faviconHash]; ok { + for _, info := range infos { + key := info.Product + "|" + info.Vendor + if !seen[key] { + seen[key] = true + results = append(results, info) + } + } + } + } + + return results +} diff --git a/runner/options.go b/runner/options.go index c5c98d7d..46e9bc80 100644 --- a/runner/options.go +++ b/runner/options.go @@ -85,6 +85,8 @@ type ScanOptions struct { NoFallback bool NoFallbackScheme bool TechDetect bool + CPEDetect bool + WordPress bool StoreChain bool StoreVisionReconClusters bool MaxResponseBodySizeToSave int @@ -148,6 +150,8 @@ func (s *ScanOptions) Clone() *ScanOptions { NoFallback: s.NoFallback, NoFallbackScheme: s.NoFallbackScheme, TechDetect: s.TechDetect, + CPEDetect: s.CPEDetect, + WordPress: s.WordPress, StoreChain: s.StoreChain, OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, @@ -256,6 +260,8 @@ type Options struct { NoFallback bool NoFallbackScheme bool TechDetect bool + CPEDetect bool + WordPress bool CustomFingerprintFile string TLSGrab bool protocol string @@ -387,6 +393,8 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"), + flagSet.BoolVar(&options.CPEDetect, "cpe", false, "display CPE (Common Platform Enumeration) based on awesome-search-queries"), + flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress plugins and themes"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVarP(&options.OutputWebSocket, "websocket", "ws", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), diff --git a/runner/runner.go b/runner/runner.go index b708f6a4..6182d396 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -81,6 +81,8 @@ type Runner struct { options *Options hp *httpx.HTTPX wappalyzer *wappalyzer.Wappalyze + cpeDetector *CPEDetector + wpDetector *WordPressDetector scanopts ScanOptions hm *hybrid.HybridMap excludeCdn bool @@ -133,6 +135,20 @@ func New(options *Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create wappalyzer client") } + if options.CPEDetect || options.JSONOutput || options.CSVOutput { + runner.cpeDetector, err = NewCPEDetector() + if err != nil { + gologger.Warning().Msgf("Could not create CPE detector: %s", err) + } + } + + if options.WordPress || options.JSONOutput || options.CSVOutput { + runner.wpDetector, err = NewWordPressDetector() + if err != nil { + gologger.Warning().Msgf("Could not create WordPress detector: %s", err) + } + } + if options.StoreResponseDir != "" { _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) @@ -297,6 +313,8 @@ func New(options *Options) (*Runner, error) { scanopts.NoFallback = options.NoFallback scanopts.NoFallbackScheme = options.NoFallbackScheme scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload + scanopts.CPEDetect = options.CPEDetect || options.JSONOutput || options.CSVOutput + scanopts.WordPress = options.WordPress || options.JSONOutput || options.CSVOutput scanopts.StoreChain = options.StoreChain scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave @@ -2311,6 +2329,47 @@ retry: } } + var cpeMatches []CPEInfo + if r.cpeDetector != nil { + cpeMatches = r.cpeDetector.Detect(title, string(resp.Data), faviconMMH3) + if len(cpeMatches) > 0 && r.options.CPEDetect { + for _, cpe := range cpeMatches { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Cyan(cpe.CPE).String()) + } else { + builder.WriteString(cpe.CPE) + } + builder.WriteRune(']') + } + } + } + + var wpInfo *WordPressInfo + if r.wpDetector != nil { + wpInfo = r.wpDetector.Detect(string(resp.Data)) + if wpInfo.HasData() && r.options.WordPress { + if len(wpInfo.Plugins) > 0 { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Green("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")).String()) + } else { + builder.WriteString("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")) + } + builder.WriteRune(']') + } + if len(wpInfo.Themes) > 0 { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Green("wp-themes:" + strings.Join(wpInfo.Themes, ",")).String()) + } else { + builder.WriteString("wp-themes:" + strings.Join(wpInfo.Themes, ",")) + } + builder.WriteRune(']') + } + } + } + result := Result{ Timestamp: time.Now(), Request: request, @@ -2374,6 +2433,8 @@ retry: RequestRaw: requestDump, Response: resp, FaviconData: faviconData, + CPE: cpeMatches, + WordPress: wpInfo, } if resp.BodyDomains != nil { result.Fqdns = resp.BodyDomains.Fqdns diff --git a/runner/types.go b/runner/types.go index ab013a70..9c169843 100644 --- a/runner/types.go +++ b/runner/types.go @@ -102,6 +102,8 @@ type Result struct { Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"` + CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" mapstructure:"cpe"` + WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" mapstructure:"wordpress"` } type Trace struct { diff --git a/runner/wordpress.go b/runner/wordpress.go new file mode 100644 index 00000000..20efc242 --- /dev/null +++ b/runner/wordpress.go @@ -0,0 +1,118 @@ +package runner + +import ( + "bufio" + "bytes" + "regexp" + "strings" + + awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries" +) + +type WordPressInfo struct { + Plugins []string `json:"plugins,omitempty"` + Themes []string `json:"themes,omitempty"` +} + +type WordPressDetector struct { + knownPlugins map[string]struct{} + knownThemes map[string]struct{} + pluginRegex *regexp.Regexp + themeRegex *regexp.Regexp +} + +func NewWordPressDetector() (*WordPressDetector, error) { + detector := &WordPressDetector{ + knownPlugins: make(map[string]struct{}), + knownThemes: make(map[string]struct{}), + } + + var err error + + detector.pluginRegex, err = regexp.Compile(`/wp-content/plugins/([a-zA-Z0-9_-]+)/`) + if err != nil { + return nil, err + } + + detector.themeRegex, err = regexp.Compile(`/wp-content/themes/([a-zA-Z0-9_-]+)/`) + if err != nil { + return nil, err + } + + pluginsData, err := awesomesearchqueries.GetWordPressPlugins() + if err != nil { + return nil, err + } + if err := detector.loadList(pluginsData, detector.knownPlugins); err != nil { + return nil, err + } + + themesData, err := awesomesearchqueries.GetWordPressThemes() + if err != nil { + return nil, err + } + if err := detector.loadList(themesData, detector.knownThemes); err != nil { + return nil, err + } + + return detector, nil +} + +func (d *WordPressDetector) loadList(data []byte, target map[string]struct{}) error { + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + target[line] = struct{}{} + } + } + return scanner.Err() +} + +func (d *WordPressDetector) Detect(body string) *WordPressInfo { + if body == "" { + return nil + } + + info := &WordPressInfo{} + seenPlugins := make(map[string]struct{}) + seenThemes := make(map[string]struct{}) + + if matches := d.pluginRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + for _, match := range matches { + if len(match) > 1 { + plugin := match[1] + if _, seen := seenPlugins[plugin]; !seen { + if _, known := d.knownPlugins[plugin]; known { + info.Plugins = append(info.Plugins, plugin) + seenPlugins[plugin] = struct{}{} + } + } + } + } + } + + if matches := d.themeRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 { + for _, match := range matches { + if len(match) > 1 { + theme := match[1] + if _, seen := seenThemes[theme]; !seen { + if _, known := d.knownThemes[theme]; known { + info.Themes = append(info.Themes, theme) + seenThemes[theme] = struct{}{} + } + } + } + } + } + + if len(info.Plugins) == 0 && len(info.Themes) == 0 { + return nil + } + + return info +} + +func (w *WordPressInfo) HasData() bool { + return w != nil && (len(w.Plugins) > 0 || len(w.Themes) > 0) +}