feat: support auto wildcard detection across multiple domains

.gitignore

@@ -1,3 +1,4 @@
+dnsx
 cmd/dnsx/dnsx
 .DS_Store
 dist/

go.mod

@@ -9,18 +9,19 @@ require (
 	github.com/miekg/dns v1.1.62
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/asnmap v1.1.1
-	github.com/projectdiscovery/cdncheck v1.2.13
+	github.com/projectdiscovery/cdncheck v1.1.36
 	github.com/projectdiscovery/clistats v0.1.1
 	github.com/projectdiscovery/goconfig v0.0.1
 	github.com/projectdiscovery/goflags v0.1.74
-	github.com/projectdiscovery/gologger v1.1.62
-	github.com/projectdiscovery/hmap v0.0.98
-	github.com/projectdiscovery/mapcidr v1.1.97
-	github.com/projectdiscovery/ratelimit v0.0.82
-	github.com/projectdiscovery/retryabledns v1.0.111
-	github.com/projectdiscovery/utils v0.7.3
+	github.com/projectdiscovery/gologger v1.1.56
+	github.com/projectdiscovery/hmap v0.0.90
+	github.com/projectdiscovery/mapcidr v1.1.34
+	github.com/projectdiscovery/ratelimit v0.0.81
+	github.com/projectdiscovery/retryabledns v1.0.102
+	github.com/projectdiscovery/utils v0.4.20
 	github.com/rs/xid v1.5.0
 	github.com/stretchr/testify v1.11.1
+	golang.org/x/net v0.47.0
 )
 
 require (
@@ -82,12 +83,12 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/fastdialer v0.4.19 // indirect
+	github.com/projectdiscovery/fastdialer v0.4.1 // indirect
 	github.com/projectdiscovery/freeport v0.0.7 // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/networkpolicy v0.1.31 // indirect
-	github.com/projectdiscovery/retryablehttp-go v1.1.0 // indirect
-	github.com/refraction-networking/utls v1.7.1 // indirect
+	github.com/projectdiscovery/networkpolicy v0.1.1 // indirect
+	github.com/projectdiscovery/retryablehttp-go v1.0.111 // indirect
+	github.com/refraction-networking/utls v1.6.7 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
@@ -98,16 +99,16 @@ require (
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.3.0 // indirect
-	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	github.com/ulikunitz/xz v0.5.15 // indirect
-	github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
+	github.com/weppos/publicsuffix-go v0.30.1 // indirect
 	github.com/yuin/goldmark v1.7.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
@@ -118,9 +119,8 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
+	golang.org/x/exp v0.0.0-20230420155640-133eef4313cb // indirect
 	golang.org/x/mod v0.29.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
 	golang.org/x/sync v0.18.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect

internal/runner/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	TraceMaxRecursion     int
 	WildcardThreshold     int
 	WildcardDomain        string
+	AutoWildcard          bool
 	ShowStatistics        bool
 	rcodes                map[int]struct{}
 	RCode                 string
@@ -189,6 +190,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"),
 		flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"),
 		flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"),
+		flagSet.BoolVarP(&options.AutoWildcard, "auto-wildcard", "aw", false, "enable automatic wildcard filtering"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"),
 	)
 

internal/runner/runner.go

@@ -115,7 +115,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	// If no option is specified or wildcard filter has been requested use query type A
-	if len(questionTypes) == 0 || options.WildcardDomain != "" {
+	if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard {
 		options.A = true
 		questionTypes = append(questionTypes, dns.TypeA)
 	}
@@ -467,7 +467,7 @@ func (r *Runner) run() error {
 	close(r.outputchan)
 	r.wgoutputworker.Wait()
 
-	if r.options.WildcardDomain != "" {
+	if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 		gologger.Print().Msgf("Starting to filter wildcard subdomains\n")
 		ipDomain := make(map[string]map[string]struct{})
 		listIPs := []string{}
@@ -524,7 +524,11 @@ func (r *Runner) run() error {
 		numRemovedSubdomains := 0
 		for _, A := range listIPs {
 			for host := range ipDomain[A] {
-				if host == r.options.WildcardDomain {
+				wildcardDomain := r.options.WildcardDomain
+				if r.options.AutoWildcard {
+					wildcardDomain = extractDomain(host)
+				}
+				if host == wildcardDomain {
 					if _, ok := seen[host]; !ok {
 						seen[host] = struct{}{}
 						_ = r.lookupAndOutput(host)
@@ -731,7 +735,7 @@ func (r *Runner) worker() {
 			}
 		}
 		// if wildcard filtering just store the data
-		if r.options.WildcardDomain != "" {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 			if err := r.storeDNSData(dnsData.DNSData); err != nil {
 				gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err)
 			}

internal/runner/util.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	fileutil "github.com/projectdiscovery/utils/file"
+	"golang.org/x/net/publicsuffix"
 )
 
 const (
@@ -43,12 +44,28 @@ func isURL(toTest string) bool {
 }
 
 func extractDomain(URL string) string {
-	u, err := url.Parse(URL)
-	if err != nil {
-		return ""
+	var host string
+	if !strings.Contains(URL, "://") && !isURL(URL) {
+		host = strings.TrimSuffix(URL, ".")
+	} else {
+		u, err := url.Parse(URL)
+		if err != nil {
+			return ""
+		}
+		host = strings.TrimSuffix(u.Hostname(), ".")
 	}
 
-	return u.Hostname()
+	// Use public suffix list for accurate eTLD+1 extraction
+	domain, err := publicsuffix.EffectiveTLDPlusOne(host)
+	if err != nil {
+		// fallback to last two parts if publicsuffix fails
+		parts := strings.Split(host, ".")
+		if len(parts) >= 2 {
+			return strings.Join(parts[len(parts)-2:], ".")
+		}
+		return host
+	}
+	return domain
 }
 
 func prepareResolver(resolver string) string {

internal/runner/util_test.go

@@ -0,0 +1,26 @@
+package runner
+
+import (
+	"testing"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractDomain(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"example.com", "example.com"},
+		{"sub.example.com", "example.com"},
+		{"foo.bar.co.uk", "bar.co.uk"},
+		{"api.service.com.au", "service.com.au"},
+		{"foo.example.com.", "example.com"},
+		{"https://sub.example.com/path", "example.com"},
+		{"http://foo.bar.co.uk:8080", "bar.co.uk"},
+	}
+
+	for _, tc := range tests {
+		got := extractDomain(tc.input)
+		require.Equal(t, tc.expected, got, "input: %s", tc.input)
+	}
+}

internal/runner/wildcard.go

@@ -19,19 +19,30 @@ func (r *Runner) IsWildcard(host string) bool {
 		orig[A] = struct{}{}
 	}
 
-	subdomainPart := strings.TrimSuffix(host, "."+r.options.WildcardDomain)
+	var wildcardDomain string
+	if r.options.AutoWildcard {
+		wildcardDomain = extractDomain(host)
+	} else {
+		wildcardDomain = r.options.WildcardDomain
+	}
+
+	if wildcardDomain == "" {
+		return false
+	}
+
+	subdomainPart := strings.TrimSuffix(host, "."+wildcardDomain)
 	subdomainTokens := strings.Split(subdomainPart, ".")
 
 	// Build an array by preallocating a slice of a length
 	// and create the wildcard generation prefix.
 	// We use a rand prefix at the beginning like %rand%.domain.tld
 	// A permutation is generated for each level of the subdomain.
-	var hosts []string
-	hosts = append(hosts, r.options.WildcardDomain)
+	hosts := make([]string, 0, len(subdomainTokens)+1)
+	hosts = append(hosts, wildcardDomain)
 
 	if len(subdomainTokens) > 0 {
 		for i := 1; i < len(subdomainTokens); i++ {
-			newhost := strings.Join(subdomainTokens[i:], ".") + "." + r.options.WildcardDomain
+			newhost := strings.Join(subdomainTokens[i:], ".") + "." + wildcardDomain
 			hosts = append(hosts, newhost)
 		}
 	}