diff --git a/README.md b/README.md index 283e6691..58ba92dc 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,10 @@ Usage: Flags: INPUT: - -l, -list string input file containing list of hosts to process - -rr, -request string file containing raw request - -u, -target string[] input target host(s) to probe + -l, -list string input file containing list of hosts to process + -rr, -request string file containing raw request + -u, -target string[] input target host(s) to probe + -im, -input-mode string mode of input file (burp) PROBES: -sc, -status-code display response status-code @@ -279,6 +280,7 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http # Notes - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. +- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/inputformats/burp.go b/common/inputformats/burp.go new file mode 100644 index 00000000..8af34327 --- /dev/null +++ b/common/inputformats/burp.go @@ -0,0 +1,44 @@ +package inputformats + +import ( + "io" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/seh-msft/burpxml" +) + +// BurpFormat is a Burp Suite XML file parser +type BurpFormat struct{} + +// NewBurpFormat creates a new Burp XML file parser +func NewBurpFormat() *BurpFormat { + return &BurpFormat{} +} + +var _ Format = &BurpFormat{} + +// Name returns the name of the format +func (b *BurpFormat) Name() string { + return "burp" +} + +// Parse parses the Burp XML input and calls the provided callback +// function for each URL it discovers. +func (b *BurpFormat) Parse(input io.Reader, callback func(url string) bool) error { + items, err := burpxml.Parse(input, true) + if err != nil { + return errors.Wrap(err, "could not parse burp xml") + } + + for i, item := range items.Items { + if item.Url == "" { + gologger.Debug().Msgf("Skipping burp item %d: empty URL", i) + continue + } + if !callback(item.Url) { + break + } + } + return nil +} diff --git a/common/inputformats/burp_test.go b/common/inputformats/burp_test.go new file mode 100644 index 00000000..ad2db527 --- /dev/null +++ b/common/inputformats/burp_test.go @@ -0,0 +1,169 @@ +package inputformats + +import ( + "strings" + "testing" +) + +func TestBurpFormat_Name(t *testing.T) { + b := NewBurpFormat() + if b.Name() != "burp" { + t.Errorf("Expected name 'burp', got '%s'", b.Name()) + } +} + +func TestBurpFormat_Parse(t *testing.T) { + burpXML := ` + + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + + + + + example.com + 443 + https + + + null + + 200 + 100 + JSON + + + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return true + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 2 { + t.Errorf("Expected 2 URLs, got %d", len(urls)) + } + + expectedURLs := []string{"http://example.com/path1", "https://example.com/path2"} + if len(urls) != len(expectedURLs) { + t.Fatalf("Expected %d URLs, got %d: %v", len(expectedURLs), len(urls), urls) + } + for i, expected := range expectedURLs { + if urls[i] != expected { + t.Errorf("Expected URL %d to be '%s', got '%s'", i, expected, urls[i]) + } + } +} + +func TestBurpFormat_ParseEmpty(t *testing.T) { + burpXML := ` + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return true + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 0 { + t.Errorf("Expected 0 URLs, got %d", len(urls)) + } +} + +func TestBurpFormat_ParseStopEarly(t *testing.T) { + burpXML := ` + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + + + + example.com + 80 + http + + + null + + 200 + 100 + HTML + + + +` + + b := NewBurpFormat() + var urls []string + + err := b.Parse(strings.NewReader(burpXML), func(url string) bool { + urls = append(urls, url) + return false // stop after first + }) + + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + + if len(urls) != 1 { + t.Errorf("Expected 1 URL (stopped early), got %d", len(urls)) + } +} + +func TestBurpFormat_ParseMalformed(t *testing.T) { + malformedXML := ` + + + + +` + + b := NewBurpFormat() + err := b.Parse(strings.NewReader(malformedXML), func(url string) bool { + return true + }) + + if err == nil { + t.Error("Expected error for malformed XML, got nil") + } +} diff --git a/common/inputformats/formats.go b/common/inputformats/formats.go new file mode 100644 index 00000000..31a25a38 --- /dev/null +++ b/common/inputformats/formats.go @@ -0,0 +1,41 @@ +// TODO: This package should be abstracted out to projectdiscovery/utils +// so it can be shared between httpx, nuclei, and other tools. +package inputformats + +import ( + "io" + "strings" +) + +// Format is an interface implemented by all input formats +type Format interface { + // Name returns the name of the format + Name() string + // Parse parses the input and calls the provided callback + // function for each URL it discovers. + Parse(input io.Reader, callback func(url string) bool) error +} + +// Supported formats +var formats = []Format{ + NewBurpFormat(), +} + +// GetFormat returns the format by name +func GetFormat(name string) Format { + for _, f := range formats { + if strings.EqualFold(f.Name(), name) { + return f + } + } + return nil +} + +// SupportedFormats returns a comma-separated list of supported format names +func SupportedFormats() string { + var names []string + for _, f := range formats { + names = append(names, f.Name()) + } + return strings.Join(names, ", ") +} diff --git a/common/inputformats/formats_test.go b/common/inputformats/formats_test.go new file mode 100644 index 00000000..aefe05d2 --- /dev/null +++ b/common/inputformats/formats_test.go @@ -0,0 +1,43 @@ +package inputformats + +import ( + "strings" + "testing" +) + +func TestGetFormat(t *testing.T) { + tests := []struct { + name string + input string + wantNil bool + wantName string + }{ + {"burp lowercase", "burp", false, "burp"}, + {"burp uppercase", "BURP", false, "burp"}, + {"burp mixed case", "Burp", false, "burp"}, + {"invalid format", "invalid", true, ""}, + {"empty string", "", true, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetFormat(tt.input) + if tt.wantNil && got != nil { + t.Errorf("GetFormat(%q) = %v, want nil", tt.input, got) + } + if !tt.wantNil && got == nil { + t.Errorf("GetFormat(%q) = nil, want non-nil", tt.input) + } + if !tt.wantNil && got != nil && got.Name() != tt.wantName { + t.Errorf("GetFormat(%q).Name() = %q, want %q", tt.input, got.Name(), tt.wantName) + } + }) + } +} + +func TestSupportedFormats(t *testing.T) { + supported := SupportedFormats() + if !strings.Contains(supported, "burp") { + t.Errorf("SupportedFormats() = %q, expected to contain 'burp'", supported) + } +} diff --git a/go.mod b/go.mod index 1ac2fdd6..fdbaf490 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,8 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 + github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.2 ) @@ -128,7 +130,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect diff --git a/go.sum b/go.sum index 0188b44c..395a3a44 100644 --- a/go.sum +++ b/go.sum @@ -389,6 +389,8 @@ github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uR github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg= +github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= diff --git a/runner/options.go b/runner/options.go index 46e9bc80..a8bfeb54 100644 --- a/runner/options.go +++ b/runner/options.go @@ -23,6 +23,7 @@ import ( customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" httpxcommon "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/httpx/common/inputformats" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/networkpolicy" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" @@ -191,6 +192,7 @@ type Options struct { SocksProxy string Proxy string InputFile string + InputMode string InputTargetHost goflags.StringSlice Methods string RequestURI string @@ -375,6 +377,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"), flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVarP(&options.InputMode, "input-mode", "im", "", fmt.Sprintf("mode of input file (%s)", inputformats.SupportedFormats())), ) flagSet.CreateGroup("Probes", "Probes", @@ -677,6 +680,14 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } + if options.InputMode != "" && inputformats.GetFormat(options.InputMode) == nil { + return fmt.Errorf("invalid input mode '%s', supported formats: %s", options.InputMode, inputformats.SupportedFormats()) + } + + if options.InputMode != "" && options.InputFile == "" { + return errors.New("-im/-input-mode requires -l/-list to specify an input file") + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 2a202652..feae6136 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -35,6 +35,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/httpx/common/inputformats" "github.com/projectdiscovery/httpx/common/pagetypeclassifier" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" @@ -524,13 +525,22 @@ func (r *Runner) prepareInput() { } // check if file has been provided if fileutil.FileExists(r.options.InputFile) { - finput, err := os.Open(r.options.InputFile) - if err != nil { - gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) - } - numHosts, err = r.loadAndCloseFile(finput) - if err != nil { - gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + // check if input mode is specified for special format handling + if format := r.getInputFormat(); format != nil { + numTargets, err := r.loadFromFormat(r.options.InputFile, format) + if err != nil { + gologger.Fatal().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) + } + numHosts = numTargets + } else { + finput, err := os.Open(r.options.InputFile) + if err != nil { + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + } + numHosts, err = r.loadAndCloseFile(finput) + if err != nil { + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) + } } } else if r.options.InputFile != "" { files, err := fileutilz.ListFilesWithPattern(r.options.InputFile) @@ -624,19 +634,52 @@ func (r *Runner) testAndSet(k string) bool { return true } +// getInputFormat returns the format for the configured input mode. +// Returns nil if no input mode is configured, or logs fatal if the format is invalid. +func (r *Runner) getInputFormat() inputformats.Format { + if r.options.InputMode == "" { + return nil + } + format := inputformats.GetFormat(r.options.InputMode) + if format == nil { + gologger.Fatal().Msgf("Invalid input mode '%s'. Supported: %s\n", r.options.InputMode, inputformats.SupportedFormats()) + } + return format +} + func (r *Runner) streamInput() (chan string, error) { out := make(chan string) go func() { defer close(out) if fileutil.FileExists(r.options.InputFile) { - fchan, err := fileutil.ReadFile(r.options.InputFile) - if err != nil { - return - } - for item := range fchan { - if r.options.SkipDedupe || r.testAndSet(item) { - out <- item + // check if input mode is specified for special format handling + if format := r.getInputFormat(); format != nil { + finput, err := os.Open(r.options.InputFile) + if err != nil { + gologger.Error().Msgf("Could not open input file '%s': %s\n", r.options.InputFile, err) + return + } + defer finput.Close() + if err := format.Parse(finput, func(item string) bool { + item = strings.TrimSpace(item) + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } + return true + }); err != nil { + gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err) + return + } + } else { + fchan, err := fileutil.ReadFile(r.options.InputFile) + if err != nil { + return + } + for item := range fchan { + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } } } } else if r.options.InputFile != "" { @@ -692,6 +735,31 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { return numTargets, err } +func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) { + finput, err := os.Open(filePath) + if err != nil { + return 0, err + } + defer finput.Close() + + err = format.Parse(finput, func(target string) bool { + target = strings.TrimSpace(target) + expandedTarget, countErr := r.countTargetFromRawTarget(target) + if countErr == nil && expandedTarget > 0 { + numTargets += expandedTarget + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(countErr, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numTargets += 1 + } + } + return true + }) + return numTargets, err +} + func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) { if rawTarget == "" { return 0, nil