spidering

Author: CodeMonkeyCybersecurity

cmd/root.go

@@ -172,9 +172,19 @@ func GetLogger() *logger.Logger {
 func runIntelligentDiscovery(target string) error {
 	log.Infow("Starting intelligent discovery", "target", target)
 
-	// Create discovery engine
+	// Create discovery engine with enhanced features
 	discoveryConfig := discovery.DefaultDiscoveryConfig()
-	discoveryEngine := discovery.NewEngine(discoveryConfig, log.WithComponent("discovery"))
+	discoveryConfig.MaxDepth = 5
+	discoveryConfig.MaxAssets = 10000
+	discoveryConfig.EnableDNS = true
+	discoveryConfig.EnableCertLog = true
+	discoveryConfig.EnableSearch = true
+	discoveryConfig.EnablePortScan = true
+	discoveryConfig.EnableWebCrawl = true
+	discoveryConfig.EnableTechStack = true
+	discoveryConfig.Timeout = 60 * time.Minute
+	
+	discoveryEngine := discovery.NewEngineWithConfig(discoveryConfig, log.WithComponent("discovery"), cfg)
 
 	// Start discovery
 	session, err := discoveryEngine.StartDiscovery(target)
@@ -2278,16 +2288,17 @@ func runMainDiscovery(cmd *cobra.Command, args []string, log *logger.Logger, db
 	// Start comprehensive discovery
 	fmt.Println("🔍 Starting comprehensive asset discovery and scanning...")
 
-	// Create discovery config
+	// Create discovery config with all features enabled
 	discoveryConfig := discovery.DefaultDiscoveryConfig()
-	discoveryConfig.MaxDepth = 3
-	discoveryConfig.MaxAssets = 1000
+	discoveryConfig.MaxDepth = 5          // Increased for recursive discovery
+	discoveryConfig.MaxAssets = 10000     // Increased for comprehensive spidering
 	discoveryConfig.EnableDNS = true
 	discoveryConfig.EnableCertLog = true
-	discoveryConfig.EnableSearch = true
+	discoveryConfig.EnableSearch = true   // Search engines enabled
 	discoveryConfig.EnablePortScan = true
-	discoveryConfig.EnableWebCrawl = true
+	discoveryConfig.EnableWebCrawl = true // Web spidering enabled
 	discoveryConfig.EnableTechStack = true
+	discoveryConfig.Timeout = 60 * time.Minute // Increased timeout for thorough discovery
 
 	// Create scope validator if we have programs configured
 	var scopeValidator *discovery.ScopeValidator
@@ -2298,9 +2309,12 @@ func runMainDiscovery(cmd *cobra.Command, args []string, log *logger.Logger, db
 		fmt.Printf("⚠️  Scope validation disabled - all discovered assets will be processed\n\n")
 	}
 
-	// Initialize discovery engine with scope validation
+	// Initialize discovery engine with enhanced discovery and scope validation
 	engine := discovery.NewEngineWithScopeValidator(discoveryConfig, log, scopeValidator)
 
+	// Register enhanced discovery module
+	engine.RegisterModule(discovery.NewEnhancedDiscovery(discoveryConfig, log, cfg))
+	
 	// Start discovery with the target
 	session, err := engine.StartDiscovery(target)
 	if err != nil {

internal/config/config.go

@@ -5,13 +5,16 @@ import (
 )
 
 type Config struct {
-	Logger    LoggerConfig    `mapstructure:"logger"`
-	Database  DatabaseConfig  `mapstructure:"database"`
-	Redis     RedisConfig     `mapstructure:"redis"`
-	Worker    WorkerConfig    `mapstructure:"worker"`
-	Telemetry TelemetryConfig `mapstructure:"telemetry"`
-	Security  SecurityConfig  `mapstructure:"security"`
-	Tools     ToolsConfig     `mapstructure:"tools"`
+	Logger       LoggerConfig    `mapstructure:"logger"`
+	Database     DatabaseConfig  `mapstructure:"database"`
+	Redis        RedisConfig     `mapstructure:"redis"`
+	Worker       WorkerConfig    `mapstructure:"worker"`
+	Telemetry    TelemetryConfig `mapstructure:"telemetry"`
+	Security     SecurityConfig  `mapstructure:"security"`
+	Tools        ToolsConfig     `mapstructure:"tools"`
+	ShodanAPIKey string          `mapstructure:"shodan_api_key"`
+	CensysAPIKey string          `mapstructure:"censys_api_key"`
+	CensysSecret string          `mapstructure:"censys_secret"`
 }
 
 type LoggerConfig struct {

internal/discovery/engine.go

@@ -40,6 +40,16 @@ func NewEngine(discoveryConfig *DiscoveryConfig, structLog *logger.Logger) *Engi
 	return NewEngineWithScopeValidator(discoveryConfig, structLog, nil)
 }
 
+// NewEngineWithConfig creates a new discovery engine with full config
+func NewEngineWithConfig(discoveryConfig *DiscoveryConfig, structLog *logger.Logger, cfg *config.Config) *Engine {
+	engine := NewEngineWithScopeValidator(discoveryConfig, structLog, nil)
+	
+	// Register enhanced discovery module with config
+	engine.RegisterModule(NewEnhancedDiscovery(discoveryConfig, structLog, cfg))
+	
+	return engine
+}
+
 // NewEngineWithScopeValidator creates a new discovery engine with scope validation
 func NewEngineWithScopeValidator(discoveryConfig *DiscoveryConfig, structLog *logger.Logger, scopeValidator *ScopeValidator) *Engine {
 	if discoveryConfig == nil {
@@ -69,7 +79,7 @@ func NewEngineWithScopeValidator(discoveryConfig *DiscoveryConfig, structLog *lo
 	}
 
 	// Register default modules
-	// Context-aware discovery runs first with highest priority
+	// Context-aware discovery
 	engine.RegisterModule(NewContextAwareDiscovery(discoveryConfig, structLog))
 	engine.RegisterModule(NewDomainDiscovery(discoveryConfig, structLog))
 	engine.RegisterModule(NewNetworkDiscovery(discoveryConfig, structLog))