From b60bfa24dba5225137f87bcd2878d7a879a6d9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 12 Jan 2026 14:03:07 +0300 Subject: [PATCH 1/6] feat: add secret file authentication support (-sf flag) --- common/authprovider/authx/basic_auth.go | 31 +++ common/authprovider/authx/bearer_auth.go | 31 +++ common/authprovider/authx/cookies_auth.go | 60 ++++++ common/authprovider/authx/file.go | 242 ++++++++++++++++++++++ common/authprovider/authx/headers_auth.go | 39 ++++ common/authprovider/authx/query_auth.go | 42 ++++ common/authprovider/authx/strategy.go | 16 ++ common/authprovider/file.go | 114 ++++++++++ common/authprovider/interface.go | 53 +++++ common/authprovider/multi.go | 50 +++++ runner/options.go | 7 + runner/runner.go | 22 ++ 12 files changed, 707 insertions(+) create mode 100644 common/authprovider/authx/basic_auth.go create mode 100644 common/authprovider/authx/bearer_auth.go create mode 100644 common/authprovider/authx/cookies_auth.go create mode 100644 common/authprovider/authx/file.go create mode 100644 common/authprovider/authx/headers_auth.go create mode 100644 common/authprovider/authx/query_auth.go create mode 100644 common/authprovider/authx/strategy.go create mode 100644 common/authprovider/file.go create mode 100644 common/authprovider/interface.go create mode 100644 common/authprovider/multi.go diff --git a/common/authprovider/authx/basic_auth.go b/common/authprovider/authx/basic_auth.go new file mode 100644 index 00000000..b7579066 --- /dev/null +++ b/common/authprovider/authx/basic_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BasicAuthStrategy{} +) + +// BasicAuthStrategy is a strategy for basic auth +type BasicAuthStrategy struct { + Data *Secret +} + +// NewBasicAuthStrategy creates a new basic auth strategy +func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy { + return &BasicAuthStrategy{Data: data} +} + +// Apply applies the basic auth strategy to the request +func (s *BasicAuthStrategy) Apply(req *http.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} + +// ApplyOnRR applies the basic auth strategy to the retryable request +func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} diff --git a/common/authprovider/authx/bearer_auth.go b/common/authprovider/authx/bearer_auth.go new file mode 100644 index 00000000..edf6f439 --- /dev/null +++ b/common/authprovider/authx/bearer_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BearerTokenAuthStrategy{} +) + +// BearerTokenAuthStrategy is a strategy for bearer token auth +type BearerTokenAuthStrategy struct { + Data *Secret +} + +// NewBearerTokenAuthStrategy creates a new bearer token auth strategy +func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy { + return &BearerTokenAuthStrategy{Data: data} +} + +// Apply applies the bearer token auth strategy to the request +func (s *BearerTokenAuthStrategy) Apply(req *http.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} + +// ApplyOnRR applies the bearer token auth strategy to the retryable request +func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go new file mode 100644 index 00000000..0b94e854 --- /dev/null +++ b/common/authprovider/authx/cookies_auth.go @@ -0,0 +1,60 @@ +package authx + +import ( + "net/http" + "slices" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &CookiesAuthStrategy{} +) + +// CookiesAuthStrategy is a strategy for cookies auth +type CookiesAuthStrategy struct { + Data *Secret +} + +// NewCookiesAuthStrategy creates a new cookies auth strategy +func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy { + return &CookiesAuthStrategy{Data: data} +} + +// Apply applies the cookies auth strategy to the request +func (s *CookiesAuthStrategy) Apply(req *http.Request) { + for _, cookie := range s.Data.Cookies { + c := &http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + } + req.AddCookie(c) + } +} + +// ApplyOnRR applies the cookies auth strategy to the retryable request +func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + existingCookies := req.Cookies() + + for _, newCookie := range s.Data.Cookies { + for i, existing := range existingCookies { + if existing.Name == newCookie.Key { + existingCookies = slices.Delete(existingCookies, i, i+1) + break + } + } + } + + // Clear and reset remaining cookies + req.Header.Del("Cookie") + for _, cookie := range existingCookies { + req.AddCookie(cookie) + } + // Add new cookies + for _, cookie := range s.Data.Cookies { + req.AddCookie(&http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + }) + } +} diff --git a/common/authprovider/authx/file.go b/common/authprovider/authx/file.go new file mode 100644 index 00000000..a31fb9a5 --- /dev/null +++ b/common/authprovider/authx/file.go @@ -0,0 +1,242 @@ +package authx + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/projectdiscovery/utils/errkit" + "github.com/projectdiscovery/utils/generic" + stringsutil "github.com/projectdiscovery/utils/strings" + "gopkg.in/yaml.v3" +) + +type AuthType string + +const ( + BasicAuth AuthType = "BasicAuth" + BearerTokenAuth AuthType = "BearerToken" + HeadersAuth AuthType = "Header" + CookiesAuth AuthType = "Cookie" + QueryAuth AuthType = "Query" +) + +// SupportedAuthTypes returns the supported auth types +func SupportedAuthTypes() []string { + return []string{ + string(BasicAuth), + string(BearerTokenAuth), + string(HeadersAuth), + string(CookiesAuth), + string(QueryAuth), + } +} + +// Authx is a struct for secrets or credentials file +type Authx struct { + ID string `json:"id" yaml:"id"` + Info AuthFileInfo `json:"info" yaml:"info"` + Secrets []Secret `json:"static" yaml:"static"` +} + +type AuthFileInfo struct { + Name string `json:"name" yaml:"name"` + Author string `json:"author" yaml:"author"` + Severity string `json:"severity" yaml:"severity"` + Description string `json:"description" yaml:"description"` +} + +// Secret is a struct for secret or credential +type Secret struct { + Type string `json:"type" yaml:"type"` + Domains []string `json:"domains" yaml:"domains"` + DomainsRegex []string `json:"domains-regex" yaml:"domains-regex"` + Headers []KV `json:"headers" yaml:"headers"` // Headers preserve exact casing (useful for case-sensitive APIs) + Cookies []Cookie `json:"cookies" yaml:"cookies"` + Params []KV `json:"params" yaml:"params"` + Username string `json:"username" yaml:"username"` // can be either email or username + Password string `json:"password" yaml:"password"` + Token string `json:"token" yaml:"token"` // Bearer Auth token +} + +// GetStrategy returns the auth strategy for the secret +func (s *Secret) GetStrategy() AuthStrategy { + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + return NewBasicAuthStrategy(s) + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + return NewBearerTokenAuthStrategy(s) + case strings.EqualFold(s.Type, string(HeadersAuth)): + return NewHeadersAuthStrategy(s) + case strings.EqualFold(s.Type, string(CookiesAuth)): + return NewCookiesAuthStrategy(s) + case strings.EqualFold(s.Type, string(QueryAuth)): + return NewQueryAuthStrategy(s) + } + return nil +} + +func (s *Secret) Validate() error { + if !stringsutil.EqualFoldAny(s.Type, SupportedAuthTypes()...) { + return fmt.Errorf("invalid type: %s", s.Type) + } + if len(s.Domains) == 0 && len(s.DomainsRegex) == 0 { + return fmt.Errorf("domains or domains-regex cannot be empty") + } + if len(s.DomainsRegex) > 0 { + for _, domain := range s.DomainsRegex { + _, err := regexp.Compile(domain) + if err != nil { + return fmt.Errorf("invalid domain regex: %s", domain) + } + } + } + + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + if s.Username == "" { + return fmt.Errorf("username cannot be empty in basic auth") + } + if s.Password == "" { + return fmt.Errorf("password cannot be empty in basic auth") + } + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + if s.Token == "" { + return fmt.Errorf("token cannot be empty in bearer token auth") + } + case strings.EqualFold(s.Type, string(HeadersAuth)): + if len(s.Headers) == 0 { + return fmt.Errorf("headers cannot be empty in headers auth") + } + for _, header := range s.Headers { + if err := header.Validate(); err != nil { + return fmt.Errorf("invalid header in headersAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(CookiesAuth)): + if len(s.Cookies) == 0 { + return fmt.Errorf("cookies cannot be empty in cookies auth") + } + for _, cookie := range s.Cookies { + if cookie.Raw != "" { + if err := cookie.Parse(); err != nil { + return fmt.Errorf("invalid raw cookie in cookiesAuth: %s", err) + } + } + if err := cookie.Validate(); err != nil { + return fmt.Errorf("invalid cookie in cookiesAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(QueryAuth)): + if len(s.Params) == 0 { + return fmt.Errorf("query cannot be empty in query auth") + } + for _, query := range s.Params { + if err := query.Validate(); err != nil { + return fmt.Errorf("invalid query in queryAuth: %s", err) + } + } + default: + return fmt.Errorf("invalid type: %s", s.Type) + } + return nil +} + +type KV struct { + Key string `json:"key" yaml:"key"` // Header key (preserves exact casing) + Value string `json:"value" yaml:"value"` +} + +func (k *KV) Validate() error { + if k.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if k.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +type Cookie struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Raw string `json:"raw" yaml:"raw"` +} + +func (c *Cookie) Validate() error { + if c.Raw != "" { + return nil + } + if c.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if c.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +// Parse parses the cookie +// in raw the cookie is in format of +// Set-Cookie: =; Expires=; Path=; Domain=; Secure; HttpOnly +func (c *Cookie) Parse() error { + if c.Raw == "" { + return fmt.Errorf("raw cookie cannot be empty") + } + tmp := strings.TrimPrefix(c.Raw, "Set-Cookie: ") + slice := strings.Split(tmp, ";") + if len(slice) == 0 { + return fmt.Errorf("invalid raw cookie no ; found") + } + // first element is the cookie name and value + cookie := strings.Split(slice[0], "=") + if len(cookie) == 2 { + c.Key = cookie[0] + c.Value = cookie[1] + return nil + } + return fmt.Errorf("invalid raw cookie: %s", c.Raw) +} + +// GetAuthDataFromFile reads the auth data from file +func GetAuthDataFromFile(file string) (*Authx, error) { + ext := filepath.Ext(file) + if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") { + return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext) + } + bin, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if ext == ".yml" || ext == ".yaml" { + return GetAuthDataFromYAML(bin) + } + return GetAuthDataFromJSON(bin) +} + +// GetAuthDataFromYAML reads the auth data from yaml +func GetAuthDataFromYAML(data []byte) (*Authx, error) { + var auth Authx + err := yaml.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal yaml") + return nil, errorErr + } + return &auth, nil +} + +// GetAuthDataFromJSON reads the auth data from json +func GetAuthDataFromJSON(data []byte) (*Authx, error) { + var auth Authx + err := json.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal json") + return nil, errorErr + } + return &auth, nil +} diff --git a/common/authprovider/authx/headers_auth.go b/common/authprovider/authx/headers_auth.go new file mode 100644 index 00000000..d474f75b --- /dev/null +++ b/common/authprovider/authx/headers_auth.go @@ -0,0 +1,39 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &HeadersAuthStrategy{} +) + +// HeadersAuthStrategy is a strategy for headers auth +type HeadersAuthStrategy struct { + Data *Secret +} + +// NewHeadersAuthStrategy creates a new headers auth strategy +func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy { + return &HeadersAuthStrategy{Data: data} +} + +// Apply applies the headers auth strategy to the request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) Apply(req *http.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} + +// ApplyOnRR applies the headers auth strategy to the retryable request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} diff --git a/common/authprovider/authx/query_auth.go b/common/authprovider/authx/query_auth.go new file mode 100644 index 00000000..796d8b1f --- /dev/null +++ b/common/authprovider/authx/query_auth.go @@ -0,0 +1,42 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + _ AuthStrategy = &QueryAuthStrategy{} +) + +// QueryAuthStrategy is a strategy for query auth +type QueryAuthStrategy struct { + Data *Secret +} + +// NewQueryAuthStrategy creates a new query auth strategy +func NewQueryAuthStrategy(data *Secret) *QueryAuthStrategy { + return &QueryAuthStrategy{Data: data} +} + +// Apply applies the query auth strategy to the request +func (s *QueryAuthStrategy) Apply(req *http.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.URL.RawQuery = q.Encode() +} + +// ApplyOnRR applies the query auth strategy to the retryable request +func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.Request.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.Request.URL.RawQuery = q.Encode() +} diff --git a/common/authprovider/authx/strategy.go b/common/authprovider/authx/strategy.go new file mode 100644 index 00000000..35e42e35 --- /dev/null +++ b/common/authprovider/authx/strategy.go @@ -0,0 +1,16 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +// AuthStrategy is an interface for auth strategies +// basic auth , bearer token, headers, cookies, query +type AuthStrategy interface { + // Apply applies the strategy to the request + Apply(*http.Request) + // ApplyOnRR applies the strategy to the retryable request + ApplyOnRR(*retryablehttp.Request) +} diff --git a/common/authprovider/file.go b/common/authprovider/file.go new file mode 100644 index 00000000..465cd06c --- /dev/null +++ b/common/authprovider/file.go @@ -0,0 +1,114 @@ +package authprovider + +import ( + "net" + "net/url" + "regexp" + "strings" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + "github.com/projectdiscovery/utils/errkit" + urlutil "github.com/projectdiscovery/utils/url" +) + +// FileAuthProvider is an auth provider for file based auth +// it accepts a secrets file and returns its provider +type FileAuthProvider struct { + Path string + store *authx.Authx + compiled map[*regexp.Regexp][]authx.AuthStrategy + domains map[string][]authx.AuthStrategy +} + +// NewFileAuthProvider creates a new file based auth provider +func NewFileAuthProvider(path string) (AuthProvider, error) { + store, err := authx.GetAuthDataFromFile(path) + if err != nil { + return nil, err + } + if len(store.Secrets) == 0 { + return nil, ErrNoSecrets + } + for _, secret := range store.Secrets { + if err := secret.Validate(); err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("invalid secret in file: %s", path) + return nil, errorErr + } + } + f := &FileAuthProvider{Path: path, store: store} + f.init() + return f, nil +} + +// init initializes the file auth provider +func (f *FileAuthProvider) init() { + for _, _secret := range f.store.Secrets { + secret := _secret // allocate copy of pointer + if len(secret.DomainsRegex) > 0 { + for _, domain := range secret.DomainsRegex { + if f.compiled == nil { + f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy) + } + compiled, err := regexp.Compile(domain) + if err != nil { + continue + } + + if ss, ok := f.compiled[compiled]; ok { + f.compiled[compiled] = append(ss, secret.GetStrategy()) + } else { + f.compiled[compiled] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } + for _, domain := range secret.Domains { + if f.domains == nil { + f.domains = make(map[string][]authx.AuthStrategy) + } + domain = strings.TrimSpace(domain) + domain = strings.TrimSuffix(domain, ":80") + domain = strings.TrimSuffix(domain, ":443") + if ss, ok := f.domains[domain]; ok { + f.domains[domain] = append(ss, secret.GetStrategy()) + } else { + f.domains[domain] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } +} + +// LookupAddr looks up a given domain/address and returns appropriate auth strategy +func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy { + var strategies []authx.AuthStrategy + + if strings.Contains(addr, ":") { + // default normalization for host:port + host, port, err := net.SplitHostPort(addr) + if err == nil && (port == "80" || port == "443") { + addr = host + } + } + for domain, strategy := range f.domains { + if strings.EqualFold(domain, addr) { + strategies = append(strategies, strategy...) + } + } + for compiled, strategy := range f.compiled { + if compiled.MatchString(addr) { + strategies = append(strategies, strategy...) + } + } + + return strategies +} + +// LookupURL looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} + +// LookupURLX looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} diff --git a/common/authprovider/interface.go b/common/authprovider/interface.go new file mode 100644 index 00000000..981496fc --- /dev/null +++ b/common/authprovider/interface.go @@ -0,0 +1,53 @@ +// TODO: This package should be abstracted out to projectdiscovery/utils +// so it can be shared between httpx, nuclei, and other tools. +package authprovider + +import ( + "fmt" + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + ErrNoSecrets = fmt.Errorf("no secrets in given provider") +) + +var ( + _ AuthProvider = &FileAuthProvider{} +) + +// AuthProvider is an interface for auth providers +// It implements a data structure suitable for quick lookup and retrieval +// of auth strategies +type AuthProvider interface { + // LookupAddr looks up a given domain/address and returns appropriate auth strategy + // for it (accepted inputs are scanme.sh or scanme.sh:443) + LookupAddr(string) []authx.AuthStrategy + // LookupURL looks up a given URL and returns appropriate auth strategy + // it accepts a valid url struct and returns the auth strategy + LookupURL(*url.URL) []authx.AuthStrategy + // LookupURLX looks up a given URL and returns appropriate auth strategy + // it accepts pd url struct (i.e urlutil.URL) and returns the auth strategy + LookupURLX(*urlutil.URL) []authx.AuthStrategy +} + +// AuthProviderOptions contains options for the auth provider +type AuthProviderOptions struct { + // File based auth provider options + SecretsFiles []string +} + +// NewAuthProvider creates a new auth provider from the given options +func NewAuthProvider(options *AuthProviderOptions) (AuthProvider, error) { + var providers []AuthProvider + for _, file := range options.SecretsFiles { + provider, err := NewFileAuthProvider(file) + if err != nil { + return nil, err + } + providers = append(providers, provider) + } + return NewMultiAuthProvider(providers...), nil +} diff --git a/common/authprovider/multi.go b/common/authprovider/multi.go new file mode 100644 index 00000000..24d59f04 --- /dev/null +++ b/common/authprovider/multi.go @@ -0,0 +1,50 @@ +package authprovider + +import ( + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +// MultiAuthProvider is a convenience wrapper for multiple auth providers +// it returns the first matching auth strategy for a given domain +// if there are multiple auth strategies for a given domain, it returns the first one +type MultiAuthProvider struct { + Providers []AuthProvider +} + +// NewMultiAuthProvider creates a new multi auth provider +func NewMultiAuthProvider(providers ...AuthProvider) AuthProvider { + return &MultiAuthProvider{Providers: providers} +} + +func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupAddr(host) + if len(strategy) > 0 { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURL(u) + if strategy != nil { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURLX(u) + if strategy != nil { + return strategy + } + } + return nil +} diff --git a/runner/options.go b/runner/options.go index 46e9bc80..69c3f8bf 100644 --- a/runner/options.go +++ b/runner/options.go @@ -351,6 +351,8 @@ type Options struct { // AssetFileUpload AssetFileUpload string TeamID string + // SecretFile is the path to the secret file for authentication + SecretFile string // OnClose adds a callback function that is invoked when httpx is closed // to be exact at end of existing closures OnClose func() @@ -523,6 +525,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), + flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"), ) flagSet.CreateGroup("debug", "Debug", @@ -677,6 +680,10 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } + if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) { + return fmt.Errorf("secret file '%s' does not exist", options.SecretFile) + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 2a202652..0d0ad1e4 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -36,6 +36,7 @@ import ( "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/httpx/common/pagetypeclassifier" + "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" @@ -94,6 +95,7 @@ type Runner struct { pHashClusters []pHashCluster simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server + authProvider authprovider.AuthProvider } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -412,6 +414,16 @@ func New(options *Options) (*Runner, error) { } runner.pageTypeClassifier = pageTypeClassifier + if options.SecretFile != "" { + authProviderOpts := &authprovider.AuthProviderOptions{ + SecretsFiles: []string{options.SecretFile}, + } + runner.authProvider, err = authprovider.NewAuthProvider(authProviderOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create auth provider") + } + } + if options.HttpApiEndpoint != "" { apiServer := NewServer(options.HttpApiEndpoint, options) gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint) @@ -1705,6 +1717,16 @@ retry: } hp.SetCustomHeaders(req, hp.CustomHeaders) + + // Apply auth strategies if auth provider is configured + if r.authProvider != nil { + if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 { + for _, strategy := range strategies { + strategy.ApplyOnRR(req) + } + } + } + // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) From 62eed15d6d41cf5738c05850cd5532b02159fb97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 12 Jan 2026 14:31:40 +0300 Subject: [PATCH 2/6] fix: address CodeRabbitAI review comments - Fix slices.Delete while iterating bug in cookies_auth.go - Fix Cookie.Parse to handle '=' chars in values using SplitN - Add case-insensitive file extension handling in GetAuthDataFromFile - Remove unreachable default case in Secret.Validate - Make LookupURL/LookupURLX consistent with LookupAddr (use len>0) - Improve comments for clarity Co-Authored-By: Claude Opus 4.5 --- common/authprovider/authx/cookies_auth.go | 28 ++++++++++++----------- common/authprovider/authx/file.go | 20 ++++++++-------- common/authprovider/file.go | 4 ++-- common/authprovider/multi.go | 4 ++-- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go index 0b94e854..0d7872b2 100644 --- a/common/authprovider/authx/cookies_auth.go +++ b/common/authprovider/authx/cookies_auth.go @@ -2,7 +2,6 @@ package authx import ( "net/http" - "slices" "github.com/projectdiscovery/retryablehttp-go" ) @@ -24,30 +23,33 @@ func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy { // Apply applies the cookies auth strategy to the request func (s *CookiesAuthStrategy) Apply(req *http.Request) { for _, cookie := range s.Data.Cookies { - c := &http.Cookie{ + req.AddCookie(&http.Cookie{ Name: cookie.Key, Value: cookie.Value, - } - req.AddCookie(c) + }) } } // ApplyOnRR applies the cookies auth strategy to the retryable request func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - existingCookies := req.Cookies() + // Build a set of cookie names to replace + newCookieNames := make(map[string]struct{}, len(s.Data.Cookies)) + for _, cookie := range s.Data.Cookies { + newCookieNames[cookie.Key] = struct{}{} + } - for _, newCookie := range s.Data.Cookies { - for i, existing := range existingCookies { - if existing.Name == newCookie.Key { - existingCookies = slices.Delete(existingCookies, i, i+1) - break - } + // Filter existing cookies, keeping only those not being replaced + existingCookies := req.Cookies() + filteredCookies := make([]*http.Cookie, 0, len(existingCookies)) + for _, cookie := range existingCookies { + if _, shouldReplace := newCookieNames[cookie.Name]; !shouldReplace { + filteredCookies = append(filteredCookies, cookie) } } - // Clear and reset remaining cookies + // Clear and reset cookies req.Header.Del("Cookie") - for _, cookie := range existingCookies { + for _, cookie := range filteredCookies { req.AddCookie(cookie) } // Add new cookies diff --git a/common/authprovider/authx/file.go b/common/authprovider/authx/file.go index a31fb9a5..ca35568b 100644 --- a/common/authprovider/authx/file.go +++ b/common/authprovider/authx/file.go @@ -139,8 +139,6 @@ func (s *Secret) Validate() error { return fmt.Errorf("invalid query in queryAuth: %s", err) } } - default: - return fmt.Errorf("invalid type: %s", s.Type) } return nil } @@ -192,18 +190,22 @@ func (c *Cookie) Parse() error { return fmt.Errorf("invalid raw cookie no ; found") } // first element is the cookie name and value - cookie := strings.Split(slice[0], "=") - if len(cookie) == 2 { - c.Key = cookie[0] - c.Value = cookie[1] - return nil + // Use SplitN to preserve '=' characters in the cookie value + cookie := strings.SplitN(slice[0], "=", 2) + if len(cookie) != 2 { + return fmt.Errorf("invalid raw cookie: missing '=' in cookie name=value: %s", c.Raw) } - return fmt.Errorf("invalid raw cookie: %s", c.Raw) + c.Key = strings.TrimSpace(cookie[0]) + c.Value = strings.TrimSpace(cookie[1]) + if c.Key == "" { + return fmt.Errorf("invalid raw cookie: empty cookie name: %s", c.Raw) + } + return nil } // GetAuthDataFromFile reads the auth data from file func GetAuthDataFromFile(file string) (*Authx, error) { - ext := filepath.Ext(file) + ext := strings.ToLower(filepath.Ext(file)) if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") { return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext) } diff --git a/common/authprovider/file.go b/common/authprovider/file.go index 465cd06c..ca6d6073 100644 --- a/common/authprovider/file.go +++ b/common/authprovider/file.go @@ -44,7 +44,7 @@ func NewFileAuthProvider(path string) (AuthProvider, error) { // init initializes the file auth provider func (f *FileAuthProvider) init() { for _, _secret := range f.store.Secrets { - secret := _secret // allocate copy of pointer + secret := _secret // capture loop variable for use in GetStrategy() if len(secret.DomainsRegex) > 0 { for _, domain := range secret.DomainsRegex { if f.compiled == nil { @@ -83,7 +83,7 @@ func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy { var strategies []authx.AuthStrategy if strings.Contains(addr, ":") { - // default normalization for host:port + // strip default ports (80/443) for consistent domain matching host, port, err := net.SplitHostPort(addr) if err == nil && (port == "80" || port == "443") { addr = host diff --git a/common/authprovider/multi.go b/common/authprovider/multi.go index 24d59f04..0ef00551 100644 --- a/common/authprovider/multi.go +++ b/common/authprovider/multi.go @@ -32,7 +32,7 @@ func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy { func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { for _, provider := range m.Providers { strategy := provider.LookupURL(u) - if strategy != nil { + if len(strategy) > 0 { return strategy } } @@ -42,7 +42,7 @@ func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { for _, provider := range m.Providers { strategy := provider.LookupURLX(u) - if strategy != nil { + if len(strategy) > 0 { return strategy } } From a780d059ee5df4fa4c7d117c62a124a0fd22f650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 12 Jan 2026 14:33:44 +0300 Subject: [PATCH 3/6] test: add comprehensive tests for auth provider - Test Cookie.Parse edge cases (equals in value, spaces, empty fields) - Test Secret validation for all auth types - Test file loading with different extensions (case-insensitive) - Test all auth strategies (Apply and ApplyOnRR methods) - Test FileAuthProvider domain lookup (exact and regex) - Test MultiAuthProvider delegation Co-Authored-By: Claude Opus 4.5 --- common/authprovider/authx/file_test.go | 408 +++++++++++++++++++++ common/authprovider/authx/strategy_test.go | 215 +++++++++++ common/authprovider/provider_test.go | 256 +++++++++++++ 3 files changed, 879 insertions(+) create mode 100644 common/authprovider/authx/file_test.go create mode 100644 common/authprovider/authx/strategy_test.go create mode 100644 common/authprovider/provider_test.go diff --git a/common/authprovider/authx/file_test.go b/common/authprovider/authx/file_test.go new file mode 100644 index 00000000..b42fc59b --- /dev/null +++ b/common/authprovider/authx/file_test.go @@ -0,0 +1,408 @@ +package authx + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCookieParse(t *testing.T) { + tests := []struct { + name string + raw string + wantKey string + wantValue string + wantErr bool + }{ + { + name: "simple cookie", + raw: "session=abc123", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "cookie with Set-Cookie prefix", + raw: "Set-Cookie: session=abc123; Path=/", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "cookie with equals in value", + raw: "token=eyJhbGciOiJIUzI1NiJ9==; Path=/", + wantKey: "token", + wantValue: "eyJhbGciOiJIUzI1NiJ9==", + wantErr: false, + }, + { + name: "cookie with spaces", + raw: " session = abc123 ; Path=/", + wantKey: "session", + wantValue: "abc123", + wantErr: false, + }, + { + name: "empty raw", + raw: "", + wantErr: true, + }, + { + name: "missing equals", + raw: "sessionabc123; Path=/", + wantErr: true, + }, + { + name: "empty key", + raw: "=abc123; Path=/", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Cookie{Raw: tt.raw} + err := c.Parse() + + if (err != nil) != tt.wantErr { + t.Errorf("Cookie.Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + if c.Key != tt.wantKey { + t.Errorf("Cookie.Parse() Key = %v, want %v", c.Key, tt.wantKey) + } + if c.Value != tt.wantValue { + t.Errorf("Cookie.Parse() Value = %v, want %v", c.Value, tt.wantValue) + } + } + }) + } +} + +func TestSecretValidate(t *testing.T) { + tests := []struct { + name string + secret Secret + wantErr bool + }{ + { + name: "valid basic auth", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Username: "user", + Password: "pass", + }, + wantErr: false, + }, + { + name: "valid bearer token", + secret: Secret{ + Type: "BearerToken", + Domains: []string{"example.com"}, + Token: "abc123", + }, + wantErr: false, + }, + { + name: "valid header auth", + secret: Secret{ + Type: "Header", + Domains: []string{"example.com"}, + Headers: []KV{{Key: "X-API-Key", Value: "secret"}}, + }, + wantErr: false, + }, + { + name: "valid cookie auth", + secret: Secret{ + Type: "Cookie", + Domains: []string{"example.com"}, + Cookies: []Cookie{{Key: "session", Value: "abc123"}}, + }, + wantErr: false, + }, + { + name: "valid query auth", + secret: Secret{ + Type: "Query", + Domains: []string{"example.com"}, + Params: []KV{{Key: "api_key", Value: "secret"}}, + }, + wantErr: false, + }, + { + name: "invalid type", + secret: Secret{ + Type: "InvalidType", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "missing domains", + secret: Secret{ + Type: "BasicAuth", + Username: "user", + Password: "pass", + }, + wantErr: true, + }, + { + name: "basic auth missing username", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Password: "pass", + }, + wantErr: true, + }, + { + name: "basic auth missing password", + secret: Secret{ + Type: "BasicAuth", + Domains: []string{"example.com"}, + Username: "user", + }, + wantErr: true, + }, + { + name: "bearer auth missing token", + secret: Secret{ + Type: "BearerToken", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "header auth missing headers", + secret: Secret{ + Type: "Header", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "cookie auth missing cookies", + secret: Secret{ + Type: "Cookie", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "query auth missing params", + secret: Secret{ + Type: "Query", + Domains: []string{"example.com"}, + }, + wantErr: true, + }, + { + name: "valid domain regex", + secret: Secret{ + Type: "BearerToken", + DomainsRegex: []string{".*\\.example\\.com"}, + Token: "abc123", + }, + wantErr: false, + }, + { + name: "invalid domain regex", + secret: Secret{ + Type: "BearerToken", + DomainsRegex: []string{"[invalid"}, + Token: "abc123", + }, + wantErr: true, + }, + { + name: "case insensitive type", + secret: Secret{ + Type: "basicauth", + Domains: []string{"example.com"}, + Username: "user", + Password: "pass", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.secret.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Secret.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGetAuthDataFromFile(t *testing.T) { + // Create temp directory for test files + tmpDir := t.TempDir() + + yamlContent := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + username: user + password: pass +` + + jsonContent := `{ + "id": "test", + "info": {"name": "test"}, + "static": [ + { + "type": "BasicAuth", + "domains": ["example.com"], + "username": "user", + "password": "pass" + } + ] +}` + + tests := []struct { + name string + filename string + content string + wantErr bool + }{ + { + name: "yaml file lowercase", + filename: "secrets.yaml", + content: yamlContent, + wantErr: false, + }, + { + name: "yml file lowercase", + filename: "secrets.yml", + content: yamlContent, + wantErr: false, + }, + { + name: "json file lowercase", + filename: "secrets.json", + content: jsonContent, + wantErr: false, + }, + { + name: "yaml file uppercase", + filename: "secrets.YAML", + content: yamlContent, + wantErr: false, + }, + { + name: "json file uppercase", + filename: "secrets.JSON", + content: jsonContent, + wantErr: false, + }, + { + name: "invalid extension", + filename: "secrets.txt", + content: yamlContent, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create test file + filePath := filepath.Join(tmpDir, tt.filename) + err := os.WriteFile(filePath, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + _, err = GetAuthDataFromFile(filePath) + if (err != nil) != tt.wantErr { + t.Errorf("GetAuthDataFromFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestSecretGetStrategy(t *testing.T) { + tests := []struct { + name string + secret Secret + wantType string + wantNil bool + }{ + { + name: "basic auth strategy", + secret: Secret{ + Type: "BasicAuth", + Username: "user", + Password: "pass", + }, + wantType: "*authx.BasicAuthStrategy", + wantNil: false, + }, + { + name: "bearer token strategy", + secret: Secret{ + Type: "BearerToken", + Token: "abc123", + }, + wantType: "*authx.BearerTokenAuthStrategy", + wantNil: false, + }, + { + name: "header strategy", + secret: Secret{ + Type: "Header", + Headers: []KV{{Key: "X-API-Key", Value: "secret"}}, + }, + wantType: "*authx.HeadersAuthStrategy", + wantNil: false, + }, + { + name: "cookie strategy", + secret: Secret{ + Type: "Cookie", + Cookies: []Cookie{{Key: "session", Value: "abc123"}}, + }, + wantType: "*authx.CookiesAuthStrategy", + wantNil: false, + }, + { + name: "query strategy", + secret: Secret{ + Type: "Query", + Params: []KV{{Key: "api_key", Value: "secret"}}, + }, + wantType: "*authx.QueryAuthStrategy", + wantNil: false, + }, + { + name: "unknown type returns nil", + secret: Secret{ + Type: "UnknownType", + }, + wantNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := tt.secret.GetStrategy() + if tt.wantNil { + if strategy != nil { + t.Errorf("GetStrategy() = %T, want nil", strategy) + } + } else { + if strategy == nil { + t.Errorf("GetStrategy() = nil, want %s", tt.wantType) + } + } + }) + } +} diff --git a/common/authprovider/authx/strategy_test.go b/common/authprovider/authx/strategy_test.go new file mode 100644 index 00000000..3beeadd0 --- /dev/null +++ b/common/authprovider/authx/strategy_test.go @@ -0,0 +1,215 @@ +package authx + +import ( + "net/http" + "testing" + + "github.com/projectdiscovery/retryablehttp-go" +) + +func TestBasicAuthStrategy(t *testing.T) { + secret := &Secret{ + Username: "user", + Password: "pass", + } + strategy := NewBasicAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + user, pass, ok := req.BasicAuth() + if !ok { + t.Error("Basic auth not set") + } + if user != "user" { + t.Errorf("Username = %v, want user", user) + } + if pass != "pass" { + t.Errorf("Password = %v, want pass", pass) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + user, pass, ok := req.BasicAuth() + if !ok { + t.Error("Basic auth not set") + } + if user != "user" { + t.Errorf("Username = %v, want user", user) + } + if pass != "pass" { + t.Errorf("Password = %v, want pass", pass) + } + }) +} + +func TestBearerTokenAuthStrategy(t *testing.T) { + secret := &Secret{ + Token: "mytoken123", + } + strategy := NewBearerTokenAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + auth := req.Header.Get("Authorization") + expected := "Bearer mytoken123" + if auth != expected { + t.Errorf("Authorization = %v, want %v", auth, expected) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + auth := req.Header.Get("Authorization") + expected := "Bearer mytoken123" + if auth != expected { + t.Errorf("Authorization = %v, want %v", auth, expected) + } + }) +} + +func TestHeadersAuthStrategy(t *testing.T) { + // Headers strategy preserves exact casing, so use exact key names + secret := &Secret{ + Headers: []KV{ + {Key: "X-API-Key", Value: "secret123"}, + {Key: "X-Custom", Value: "value"}, + }, + } + strategy := NewHeadersAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + // Use direct map access since headers preserve exact casing + if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { + t.Errorf("X-API-Key = %v, want [secret123]", got) + } + if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" { + t.Errorf("X-Custom = %v, want [value]", got) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + strategy.ApplyOnRR(req) + + // Use direct map access since headers preserve exact casing + if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { + t.Errorf("X-API-Key = %v, want [secret123]", got) + } + if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" { + t.Errorf("X-Custom = %v, want [value]", got) + } + }) +} + +func TestCookiesAuthStrategy(t *testing.T) { + secret := &Secret{ + Cookies: []Cookie{ + {Key: "session", Value: "abc123"}, + {Key: "auth", Value: "xyz789"}, + }, + } + strategy := NewCookiesAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) + + cookies := req.Cookies() + if len(cookies) != 2 { + t.Errorf("Expected 2 cookies, got %d", len(cookies)) + } + + found := make(map[string]string) + for _, c := range cookies { + found[c.Name] = c.Value + } + if found["session"] != "abc123" { + t.Errorf("session cookie = %v, want abc123", found["session"]) + } + if found["auth"] != "xyz789" { + t.Errorf("auth cookie = %v, want xyz789", found["auth"]) + } + }) + + t.Run("ApplyOnRR replaces existing cookies", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + // Add existing cookie that should be replaced + req.AddCookie(&http.Cookie{Name: "session", Value: "old_value"}) + // Add existing cookie that should be kept + req.AddCookie(&http.Cookie{Name: "other", Value: "keep_me"}) + + strategy.ApplyOnRR(req) + + cookies := req.Cookies() + found := make(map[string]string) + for _, c := range cookies { + found[c.Name] = c.Value + } + + // New cookie values should override old ones + if found["session"] != "abc123" { + t.Errorf("session cookie = %v, want abc123", found["session"]) + } + if found["auth"] != "xyz789" { + t.Errorf("auth cookie = %v, want xyz789", found["auth"]) + } + // Existing non-replaced cookie should be preserved + if found["other"] != "keep_me" { + t.Errorf("other cookie = %v, want keep_me", found["other"]) + } + }) +} + +func TestQueryAuthStrategy(t *testing.T) { + secret := &Secret{ + Params: []KV{ + {Key: "api_key", Value: "secret123"}, + {Key: "token", Value: "abc"}, + }, + } + strategy := NewQueryAuthStrategy(secret) + + t.Run("Apply", func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com/path?existing=value", nil) + strategy.Apply(req) + + query := req.URL.Query() + if got := query.Get("api_key"); got != "secret123" { + t.Errorf("api_key = %v, want secret123", got) + } + if got := query.Get("token"); got != "abc" { + t.Errorf("token = %v, want abc", got) + } + if got := query.Get("existing"); got != "value" { + t.Errorf("existing = %v, want value", got) + } + }) + + t.Run("ApplyOnRR", func(t *testing.T) { + req, _ := retryablehttp.NewRequest("GET", "http://example.com/path?existing=value", nil) + strategy.ApplyOnRR(req) + + query := req.Request.URL.Query() + if got := query.Get("api_key"); got != "secret123" { + t.Errorf("api_key = %v, want secret123", got) + } + if got := query.Get("token"); got != "abc" { + t.Errorf("token = %v, want abc", got) + } + if got := query.Get("existing"); got != "value" { + t.Errorf("existing = %v, want value", got) + } + }) +} diff --git a/common/authprovider/provider_test.go b/common/authprovider/provider_test.go new file mode 100644 index 00000000..a29cde51 --- /dev/null +++ b/common/authprovider/provider_test.go @@ -0,0 +1,256 @@ +package authprovider + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + urlutil "github.com/projectdiscovery/utils/url" +) + +func createTestSecretsFile(t *testing.T, content string) string { + t.Helper() + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "secrets.yaml") + err := os.WriteFile(filePath, []byte(content), 0644) + if err != nil { + t.Fatalf("Failed to create test secrets file: %v", err) + } + return filePath +} + +func TestFileAuthProviderLookupAddr(t *testing.T) { + content := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + - api.example.com:443 + username: user + password: pass + - type: BearerToken + domains-regex: + - ".*\\.test\\.com" + token: regextoken +` + filePath := createTestSecretsFile(t, content) + provider, err := NewFileAuthProvider(filePath) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + tests := []struct { + name string + addr string + wantCount int + }{ + { + name: "exact match", + addr: "example.com", + wantCount: 1, + }, + { + name: "exact match case insensitive", + addr: "EXAMPLE.COM", + wantCount: 1, + }, + { + name: "with port 443 normalized", + addr: "example.com:443", + wantCount: 1, + }, + { + name: "with port 80 normalized", + addr: "example.com:80", + wantCount: 1, + }, + { + name: "subdomain exact match", + addr: "api.example.com", + wantCount: 1, + }, + { + name: "regex match", + addr: "foo.test.com", + wantCount: 1, + }, + { + name: "regex match subdomain", + addr: "bar.baz.test.com", + wantCount: 1, + }, + { + name: "no match", + addr: "unknown.com", + wantCount: 0, + }, + { + name: "non-standard port not normalized", + addr: "example.com:8080", + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategies := provider.LookupAddr(tt.addr) + if len(strategies) != tt.wantCount { + t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount) + } + }) + } +} + +func TestFileAuthProviderLookupURL(t *testing.T) { + content := `id: test +info: + name: test +static: + - type: BasicAuth + domains: + - example.com + username: user + password: pass +` + filePath := createTestSecretsFile(t, content) + provider, err := NewFileAuthProvider(filePath) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + t.Run("LookupURL", func(t *testing.T) { + u, _ := url.Parse("https://example.com/path") + strategies := provider.LookupURL(u) + if len(strategies) != 1 { + t.Errorf("LookupURL() returned %d strategies, want 1", len(strategies)) + } + }) + + t.Run("LookupURLX", func(t *testing.T) { + u, _ := urlutil.Parse("https://example.com/path") + strategies := provider.LookupURLX(u) + if len(strategies) != 1 { + t.Errorf("LookupURLX() returned %d strategies, want 1", len(strategies)) + } + }) +} + +func TestMultiAuthProvider(t *testing.T) { + content1 := `id: test1 +info: + name: test1 +static: + - type: BasicAuth + domains: + - first.com + username: user1 + password: pass1 +` + content2 := `id: test2 +info: + name: test2 +static: + - type: BearerToken + domains: + - second.com + token: token2 +` + filePath1 := createTestSecretsFile(t, content1) + provider1, err := NewFileAuthProvider(filePath1) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + // Create second file in different temp dir + tmpDir2 := t.TempDir() + filePath2 := filepath.Join(tmpDir2, "secrets2.yaml") + err = os.WriteFile(filePath2, []byte(content2), 0644) + if err != nil { + t.Fatalf("Failed to create test secrets file: %v", err) + } + provider2, err := NewFileAuthProvider(filePath2) + if err != nil { + t.Fatalf("NewFileAuthProvider() error = %v", err) + } + + multi := NewMultiAuthProvider(provider1, provider2) + + tests := []struct { + name string + addr string + wantCount int + }{ + { + name: "match first provider", + addr: "first.com", + wantCount: 1, + }, + { + name: "match second provider", + addr: "second.com", + wantCount: 1, + }, + { + name: "no match", + addr: "third.com", + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategies := multi.LookupAddr(tt.addr) + if len(strategies) != tt.wantCount { + t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount) + } + }) + } +} + +func TestNewFileAuthProviderErrors(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + }{ + { + name: "empty secrets", + content: `id: test`, + wantErr: true, + }, + { + name: "invalid secret type", + content: `id: test +static: + - type: InvalidType + domains: + - example.com +`, + wantErr: true, + }, + { + name: "missing required field", + content: `id: test +static: + - type: BasicAuth + domains: + - example.com + username: user +`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filePath := createTestSecretsFile(t, tt.content) + _, err := NewFileAuthProvider(filePath) + if (err != nil) != tt.wantErr { + t.Errorf("NewFileAuthProvider() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From a4ae407118c9fe4fa9d8e9bab7f0df4eaa978a27 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 12 Jan 2026 15:40:40 +0400 Subject: [PATCH 4/6] lint --- common/authprovider/authx/strategy_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/authprovider/authx/strategy_test.go b/common/authprovider/authx/strategy_test.go index 3beeadd0..01a8a9f5 100644 --- a/common/authprovider/authx/strategy_test.go +++ b/common/authprovider/authx/strategy_test.go @@ -91,6 +91,7 @@ func TestHeadersAuthStrategy(t *testing.T) { strategy.Apply(req) // Use direct map access since headers preserve exact casing + //nolint if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { t.Errorf("X-API-Key = %v, want [secret123]", got) } @@ -104,6 +105,7 @@ func TestHeadersAuthStrategy(t *testing.T) { strategy.ApplyOnRR(req) // Use direct map access since headers preserve exact casing + //nolint if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" { t.Errorf("X-API-Key = %v, want [secret123]", got) } From c125352ff341664e1b5627fe35ca29faecbf7e80 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 12 Jan 2026 15:41:59 +0400 Subject: [PATCH 5/6] mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fdbaf490..0158d5ad 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -177,5 +178,4 @@ require ( golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.39.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) From 93633d2e82f94ff9f9dd8dea539cc5a55f7077f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 12 Jan 2026 15:38:45 +0300 Subject: [PATCH 6/6] docs: add secret file authentication documentation --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 58ba92dc..98dce07a 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ CONFIGURATIONS: -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization -no-stdin Disable Stdin processing -hae, -http-api-endpoint string experimental http api endpoint + -sf, -secret-file string path to secret file for authentication DEBUG: -health-check, -hc run diagnostic check up @@ -284,6 +285,24 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) +- Secret files can be used for domain-based authentication via `-sf secrets.yaml`. Supported auth types: `BasicAuth`, `BearerToken`, `Header`, `Cookie`, `Query`. Example: + ```yaml + id: example-auth + info: + name: Example Auth Config + static: + - type: Header + domains: + - api.example.com + headers: + - key: X-API-Key + value: secret-key-here + - type: BasicAuth + domains-regex: + - ".*\\.internal\\.com$" + username: admin + password: secret + ``` - The following flags should be used for specific use cases instead of running them as default with other probes: - `-ports` - `-path`