shadowhunter_crawler.py

#!/usr/bin/env python3
"""
ShadowHunter - Dark Web Credential Intelligence Platform
Module: Async Web Crawler (Inspired by Photon)
Author: Fevra
Version: 0.1.0

High-performance async web crawler for:
- Deep link extraction and mapping
- JavaScript file analysis for secrets
- API endpoint discovery
- Email/credential harvesting
- Subdomain enumeration
- Metadata extraction
- Form detection and analysis
"""

import asyncio
import aiohttp
import re
import hashlib
from datetime import datetime, timezone
from typing import List, Dict, Set, Optional, Any, Tuple
from dataclasses import dataclass, field
from urllib.parse import urljoin, urlparse, parse_qs
from pathlib import Path
from collections import defaultdict
import json
from enum import Enum

# Import logger
try:
    from shadowhunter_logger import get_logger, LogLevel
except ImportError:
    import logging
    def get_logger(name, **kwargs):
        return logging.getLogger(name)

# Initialize module logger
logger = get_logger("Crawler", log_level=LogLevel.DEBUG)


# ============================================================================
# REGEX PATTERNS FOR EXTRACTION
# ============================================================================

class ExtractionPatterns:
    """Regex patterns for extracting various data types from web content."""
    
    # URL patterns
    URL = re.compile(
        r'(?:href|src|action|data-url|data-src)=["\']([^"\']+)["\']',
        re.IGNORECASE
    )
    
    JS_URL = re.compile(
        r'(?:url\s*[:\(]|fetch\s*\(|XMLHttpRequest.*open\s*\([^,]+,)\s*["\']([^"\']+)["\']',
        re.IGNORECASE
    )
    
    # Email patterns
    EMAIL = re.compile(
        r'[\w\.-]+@[\w\.-]+\.\w{2,}',
        re.IGNORECASE
    )
    
    # Phone patterns (international formats)
    PHONE = re.compile(
        r'(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}|'
        r'\+[0-9]{1,3}[-.\s]?[0-9]{4,14}',
        re.IGNORECASE
    )
    
    # Social media handles
    TWITTER = re.compile(r'(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})', re.IGNORECASE)
    LINKEDIN = re.compile(r'linkedin\.com/(?:in|company)/([A-Za-z0-9_-]+)', re.IGNORECASE)
    GITHUB = re.compile(r'github\.com/([A-Za-z0-9_-]+)', re.IGNORECASE)
    
    # API endpoints
    API_ENDPOINT = re.compile(
        r'["\'](?:/api/|/v[0-9]+/|/rest/)([^"\']+)["\']',
        re.IGNORECASE
    )
    
    # Secrets and keys
    AWS_KEY = re.compile(r'(?:AKIA|A3T|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}')
    AWS_SECRET = re.compile(r'["\'][A-Za-z0-9/+=]{40}["\']')
    GOOGLE_API = re.compile(r'AIza[0-9A-Za-z_-]{35}')
    GITHUB_TOKEN = re.compile(r'gh[pousr]_[A-Za-z0-9_]{36,}')
    JWT_TOKEN = re.compile(r'eyJ[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*')
    PRIVATE_KEY = re.compile(r'-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----')
    GENERIC_SECRET = re.compile(
        r'(?:password|passwd|secret|token|api_key|apikey|auth|credential)["\'\s]*[:=]\s*["\']([^"\']{8,})["\']',
        re.IGNORECASE
    )
    
    # JavaScript files
    JS_FILE = re.compile(r'["\']([^"\']*\.js(?:\?[^"\']*)?)["\']', re.IGNORECASE)
    
    # Form fields
    FORM = re.compile(r'<form[^>]*>(.*?)</form>', re.IGNORECASE | re.DOTALL)
    INPUT = re.compile(
        r'<input[^>]*(?:name|id)=["\']([^"\']+)["\'][^>]*>',
        re.IGNORECASE
    )
    
    # Comments (may contain sensitive info)
    HTML_COMMENT = re.compile(r'<!--(.*?)-->', re.DOTALL)
    JS_COMMENT = re.compile(r'/\*[\s\S]*?\*/|//.*?(?:\n|$)')
    
    # Subdomains from content
    SUBDOMAIN = re.compile(r'[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9.]+')


# ============================================================================
# DATA MODELS
# ============================================================================

class CrawlStatus(str, Enum):
    """Page crawl status."""
    SUCCESS = "success"
    FAILED = "failed"
    TIMEOUT = "timeout"
    BLOCKED = "blocked"
    REDIRECT = "redirect"


@dataclass
class ExtractedData:
    """Data extracted from a single page."""
    urls: List[str] = field(default_factory=list)
    emails: List[str] = field(default_factory=list)
    phones: List[str] = field(default_factory=list)
    api_endpoints: List[str] = field(default_factory=list)
    js_files: List[str] = field(default_factory=list)
    forms: List[Dict[str, Any]] = field(default_factory=list)
    secrets: List[Dict[str, str]] = field(default_factory=list)
    social_profiles: List[Dict[str, str]] = field(default_factory=list)
    comments: List[str] = field(default_factory=list)
    subdomains: List[str] = field(default_factory=list)
    metadata: Dict[str, Any] = field(default_factory=dict)
    
    def merge(self, other: 'ExtractedData') -> 'ExtractedData':
        """Merge another ExtractedData into this one."""
        self.urls.extend(other.urls)
        self.emails.extend(other.emails)
        self.phones.extend(other.phones)
        self.api_endpoints.extend(other.api_endpoints)
        self.js_files.extend(other.js_files)
        self.forms.extend(other.forms)
        self.secrets.extend(other.secrets)
        self.social_profiles.extend(other.social_profiles)
        self.comments.extend(other.comments)
        self.subdomains.extend(other.subdomains)
        return self


@dataclass
class CrawlResult:
    """Result from crawling a single URL."""
    url: str
    status: CrawlStatus
    status_code: Optional[int] = None
    content_type: Optional[str] = None
    content_length: Optional[int] = None
    response_time_ms: float = 0
    extracted: ExtractedData = field(default_factory=ExtractedData)
    error: Optional[str] = None
    timestamp: str = ""
    
    def __post_init__(self):
        if not self.timestamp:
            self.timestamp = datetime.now(timezone.utc).isoformat()


@dataclass
class CrawlReport:
    """Complete crawl report for a domain."""
    domain: str
    seed_url: str
    started_at: str = ""
    completed_at: str = ""
    pages_crawled: int = 0
    pages_failed: int = 0
    
    # Aggregated data
    all_urls: Set[str] = field(default_factory=set)
    internal_urls: Set[str] = field(default_factory=set)
    external_urls: Set[str] = field(default_factory=set)
    emails: Set[str] = field(default_factory=set)
    phones: Set[str] = field(default_factory=set)
    api_endpoints: Set[str] = field(default_factory=set)
    js_files: Set[str] = field(default_factory=set)
    subdomains: Set[str] = field(default_factory=set)
    secrets: List[Dict[str, str]] = field(default_factory=list)
    social_profiles: List[Dict[str, str]] = field(default_factory=list)
    forms: List[Dict[str, Any]] = field(default_factory=list)
    
    # Page results
    results: List[CrawlResult] = field(default_factory=list)
    errors: List[str] = field(default_factory=list)
    
    def __post_init__(self):
        if not self.started_at:
            self.started_at = datetime.now(timezone.utc).isoformat()
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        return {
            "domain": self.domain,
            "seed_url": self.seed_url,
            "started_at": self.started_at,
            "completed_at": self.completed_at,
            "summary": {
                "pages_crawled": self.pages_crawled,
                "pages_failed": self.pages_failed,
                "total_urls": len(self.all_urls),
                "internal_urls": len(self.internal_urls),
                "external_urls": len(self.external_urls),
                "emails_found": len(self.emails),
                "phones_found": len(self.phones),
                "api_endpoints": len(self.api_endpoints),
                "js_files": len(self.js_files),
                "subdomains": len(self.subdomains),
                "secrets_found": len(self.secrets),
                "forms_found": len(self.forms)
            },
            "data": {
                "emails": list(self.emails),
                "phones": list(self.phones),
                "api_endpoints": list(self.api_endpoints),
                "js_files": list(self.js_files),
                "subdomains": list(self.subdomains),
                "secrets": self.secrets,
                "social_profiles": self.social_profiles,
                "forms": self.forms
            },
            "errors": self.errors
        }


# ============================================================================
# DATA EXTRACTOR
# ============================================================================

class DataExtractor:
    """
    Extracts various data types from HTML and JavaScript content.
    
    Features:
    - Email and phone extraction
    - API endpoint discovery
    - Secret detection (API keys, tokens)
    - Form analysis
    - Social media profile extraction
    - Comment mining
    """
    
    def __init__(self, base_url: str):
        """
        Initialize extractor.
        
        Args:
            base_url: Base URL for resolving relative URLs
        """
        self.base_url = base_url
        self.parsed_base = urlparse(base_url)
        self.domain = self.parsed_base.netloc
    
    def extract_all(self, content: str, content_type: str = "text/html") -> ExtractedData:
        """
        Extract all data from content.
        
        Args:
            content: Page content
            content_type: MIME type of content
        
        Returns:
            ExtractedData with all findings
        """
        data = ExtractedData()
        
        # Extract URLs
        data.urls = self._extract_urls(content)
        
        # Extract emails
        data.emails = self._extract_emails(content)
        
        # Extract phones
        data.phones = self._extract_phones(content)
        
        # Extract API endpoints
        data.api_endpoints = self._extract_api_endpoints(content)
        
        # Extract JS files
        data.js_files = self._extract_js_files(content)
        
        # Extract forms
        data.forms = self._extract_forms(content)
        
        # Extract secrets
        data.secrets = self._extract_secrets(content)
        
        # Extract social profiles
        data.social_profiles = self._extract_social_profiles(content)
        
        # Extract comments
        data.comments = self._extract_comments(content)
        
        # Extract subdomains
        data.subdomains = self._extract_subdomains(content)
        
        return data
    
    def _resolve_url(self, url: str) -> str:
        """Resolve relative URL to absolute."""
        if url.startswith(('http://', 'https://', '//')):
            if url.startswith('//'):
                return f"{self.parsed_base.scheme}:{url}"
            return url
        return urljoin(self.base_url, url)
    
    def _extract_urls(self, content: str) -> List[str]:
        """Extract all URLs from content."""
        urls = set()
        
        # HTML attributes
        for match in ExtractionPatterns.URL.finditer(content):
            url = match.group(1)
            if url and not url.startswith(('#', 'javascript:', 'mailto:', 'tel:')):
                urls.add(self._resolve_url(url))
        
        # JavaScript URLs
        for match in ExtractionPatterns.JS_URL.finditer(content):
            url = match.group(1)
            if url and url.startswith(('/', 'http')):
                urls.add(self._resolve_url(url))
        
        return list(urls)
    
    def _extract_emails(self, content: str) -> List[str]:
        """Extract email addresses."""
        emails = set()
        for match in ExtractionPatterns.EMAIL.finditer(content):
            email = match.group().lower()
            # Filter out common false positives
            if not any(fp in email for fp in ['example.com', 'test.com', '.png', '.jpg', '.gif']):
                emails.add(email)
        return list(emails)
    
    def _extract_phones(self, content: str) -> List[str]:
        """Extract phone numbers."""
        phones = set()
        for match in ExtractionPatterns.PHONE.finditer(content):
            phone = re.sub(r'[^\d+]', '', match.group())
            if len(phone) >= 10:
                phones.add(phone)
        return list(phones)
    
    def _extract_api_endpoints(self, content: str) -> List[str]:
        """Extract API endpoints."""
        endpoints = set()
        for match in ExtractionPatterns.API_ENDPOINT.finditer(content):
            endpoint = match.group(1)
            endpoints.add(endpoint)
        return list(endpoints)
    
    def _extract_js_files(self, content: str) -> List[str]:
        """Extract JavaScript file URLs."""
        js_files = set()
        for match in ExtractionPatterns.JS_FILE.finditer(content):
            js_url = match.group(1)
            if js_url:
                js_files.add(self._resolve_url(js_url))
        return list(js_files)
    
    def _extract_forms(self, content: str) -> List[Dict[str, Any]]:
        """Extract and analyze HTML forms."""
        forms = []
        for form_match in ExtractionPatterns.FORM.finditer(content):
            form_html = form_match.group(0)
            
            # Extract action URL
            action_match = re.search(r'action=["\']([^"\']+)["\']', form_html, re.I)
            action = action_match.group(1) if action_match else ""
            
            # Extract method
            method_match = re.search(r'method=["\']([^"\']+)["\']', form_html, re.I)
            method = method_match.group(1).upper() if method_match else "GET"
            
            # Extract input fields
            inputs = []
            for input_match in ExtractionPatterns.INPUT.finditer(form_html):
                inputs.append(input_match.group(1))
            
            # Identify form type
            form_type = "unknown"
            if any(f in form_html.lower() for f in ['password', 'login', 'signin']):
                form_type = "login"
            elif any(f in form_html.lower() for f in ['search', 'query']):
                form_type = "search"
            elif any(f in form_html.lower() for f in ['email', 'subscribe', 'newsletter']):
                form_type = "newsletter"
            elif any(f in form_html.lower() for f in ['register', 'signup', 'create']):
                form_type = "registration"
            
            forms.append({
                "action": self._resolve_url(action) if action else "",
                "method": method,
                "inputs": inputs,
                "type": form_type
            })
        
        return forms
    
    def _extract_secrets(self, content: str) -> List[Dict[str, str]]:
        """Extract potential secrets and API keys."""
        secrets = []
        
        # AWS keys
        for match in ExtractionPatterns.AWS_KEY.finditer(content):
            secrets.append({"type": "aws_access_key", "value": match.group()[:12] + "****"})
        
        # Google API keys
        for match in ExtractionPatterns.GOOGLE_API.finditer(content):
            secrets.append({"type": "google_api_key", "value": match.group()[:12] + "****"})
        
        # GitHub tokens
        for match in ExtractionPatterns.GITHUB_TOKEN.finditer(content):
            secrets.append({"type": "github_token", "value": match.group()[:12] + "****"})
        
        # JWT tokens
        for match in ExtractionPatterns.JWT_TOKEN.finditer(content):
            secrets.append({"type": "jwt_token", "value": match.group()[:20] + "..."})
        
        # Private keys
        for match in ExtractionPatterns.PRIVATE_KEY.finditer(content):
            secrets.append({"type": "private_key", "value": "PRIVATE KEY DETECTED"})
        
        # Generic secrets
        for match in ExtractionPatterns.GENERIC_SECRET.finditer(content):
            value = match.group(1)
            if len(value) >= 8:
                secrets.append({"type": "generic_secret", "value": value[:8] + "****"})
        
        return secrets
    
    def _extract_social_profiles(self, content: str) -> List[Dict[str, str]]:
        """Extract social media profiles."""
        profiles = []
        
        for match in ExtractionPatterns.TWITTER.finditer(content):
            profiles.append({"platform": "twitter", "handle": match.group(1)})
        
        for match in ExtractionPatterns.LINKEDIN.finditer(content):
            profiles.append({"platform": "linkedin", "handle": match.group(1)})
        
        for match in ExtractionPatterns.GITHUB.finditer(content):
            profiles.append({"platform": "github", "handle": match.group(1)})
        
        return profiles
    
    def _extract_comments(self, content: str) -> List[str]:
        """Extract HTML and JS comments that might contain sensitive info."""
        comments = []
        
        # HTML comments
        for match in ExtractionPatterns.HTML_COMMENT.finditer(content):
            comment = match.group(1).strip()
            # Only keep comments with potentially interesting content
            if any(kw in comment.lower() for kw in 
                   ['todo', 'fixme', 'bug', 'hack', 'password', 'secret', 
                    'api', 'key', 'token', 'debug', 'test', 'admin']):
                comments.append(comment[:200])  # Truncate long comments
        
        return comments
    
    def _extract_subdomains(self, content: str) -> List[str]:
        """Extract subdomains related to target domain."""
        subdomains = set()
        base_domain = self.domain.replace('www.', '')
        
        for match in ExtractionPatterns.SUBDOMAIN.finditer(content):
            potential = match.group().lower()
            if base_domain in potential and potential != base_domain:
                # Extract just the subdomain prefix
                if potential.endswith(base_domain):
                    subdomains.add(potential)
        
        return list(subdomains)


# ============================================================================
# ASYNC WEB CRAWLER
# ============================================================================

class ShadowCrawler:
    """
    High-performance async web crawler for OSINT.
    
    Features:
    - Concurrent crawling with configurable limits
    - Depth-limited exploration
    - Duplicate URL detection
    - Rate limiting and politeness
    - JavaScript file analysis
    - Comprehensive data extraction
    
    Usage:
        crawler = ShadowCrawler(max_depth=3, max_pages=100)
        report = await crawler.crawl("https://example.com")
    """
    
    # Default headers to appear as legitimate browser
    DEFAULT_HEADERS = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate",
        "DNT": "1",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1"
    }
    
    def __init__(
        self,
        max_depth: int = 3,
        max_pages: int = 100,
        max_concurrent: int = 10,
        timeout: int = 30,
        delay: float = 0.5,
        headers: Optional[Dict[str, str]] = None,
        exclude_patterns: Optional[List[str]] = None
    ):
        """
        Initialize crawler.
        
        Args:
            max_depth: Maximum crawl depth from seed URL
            max_pages: Maximum pages to crawl
            max_concurrent: Maximum concurrent requests
            timeout: Request timeout in seconds
            delay: Delay between requests (politeness)
            headers: Custom HTTP headers
            exclude_patterns: URL patterns to exclude
        """
        self.max_depth = max_depth
        self.max_pages = max_pages
        self.max_concurrent = max_concurrent
        self.timeout = timeout
        self.delay = delay
        self.headers = headers or self.DEFAULT_HEADERS
        self.exclude_patterns = exclude_patterns or [
            r'\.(?:jpg|jpeg|png|gif|ico|css|woff|woff2|ttf|eot|svg)(?:\?|$)',
            r'\?(?:utm_|ref=|share=|print=)',
            r'(?:logout|signout|unsubscribe)',
        ]
        
        # State
        self.visited: Set[str] = set()
        self.queue: asyncio.Queue = asyncio.Queue()
        self.session: Optional[aiohttp.ClientSession] = None
        self.semaphore: Optional[asyncio.Semaphore] = None
        
        logger.info("ShadowCrawler initialized", {
            "max_depth": max_depth,
            "max_pages": max_pages,
            "max_concurrent": max_concurrent
        })
    
    def _normalize_url(self, url: str) -> str:
        """Normalize URL for deduplication."""
        parsed = urlparse(url)
        # Remove fragments and trailing slashes
        normalized = f"{parsed.scheme}://{parsed.netloc}{parsed.path.rstrip('/')}"
        if parsed.query:
            # Sort query parameters for consistency
            params = sorted(parse_qs(parsed.query).items())
            query = "&".join(f"{k}={v[0]}" for k, v in params if v)
            if query:
                normalized += f"?{query}"
        return normalized
    
    def _is_valid_url(self, url: str, base_domain: str) -> bool:
        """Check if URL should be crawled."""
        try:
            parsed = urlparse(url)
            
            # Must be HTTP(S)
            if parsed.scheme not in ('http', 'https'):
                return False
            
            # Must be same domain or subdomain
            if not parsed.netloc.endswith(base_domain.replace('www.', '')):
                return False
            
            # Check exclude patterns
            for pattern in self.exclude_patterns:
                if re.search(pattern, url, re.I):
                    return False
            
            return True
            
        except Exception:
            return False
    
    async def _fetch_page(
        self,
        url: str,
        depth: int
    ) -> CrawlResult:
        """
        Fetch single page and extract data.
        
        Args:
            url: URL to fetch
            depth: Current crawl depth
        
        Returns:
            CrawlResult with extracted data
        """
        start_time = datetime.now()
        
        try:
            async with self.semaphore:
                async with self.session.get(
                    url,
                    timeout=aiohttp.ClientTimeout(total=self.timeout),
                    allow_redirects=True
                ) as response:
                    
                    # Calculate response time
                    response_time = (datetime.now() - start_time).total_seconds() * 1000
                    
                    # Get content
                    content_type = response.headers.get('Content-Type', '')
                    
                    # Only process HTML and JS content
                    if 'text/html' in content_type or 'javascript' in content_type:
                        content = await response.text()
                        content_length = len(content)
                    else:
                        content = ""
                        content_length = 0
                    
                    # Extract data
                    extractor = DataExtractor(url)
                    extracted = extractor.extract_all(content, content_type)
                    
                    logger.http(f"Crawled: {url}", {
                        "status": response.status,
                        "response_time_ms": round(response_time, 2),
                        "urls_found": len(extracted.urls),
                        "depth": depth
                    })
                    
                    return CrawlResult(
                        url=url,
                        status=CrawlStatus.SUCCESS,
                        status_code=response.status,
                        content_type=content_type,
                        content_length=content_length,
                        response_time_ms=response_time,
                        extracted=extracted
                    )
                    
        except asyncio.TimeoutError:
            logger.warn(f"Timeout: {url}")
            return CrawlResult(
                url=url,
                status=CrawlStatus.TIMEOUT,
                error="Request timeout"
            )
            
        except aiohttp.ClientError as e:
            logger.warn(f"Client error: {url} - {e}")
            return CrawlResult(
                url=url,
                status=CrawlStatus.FAILED,
                error=str(e)
            )
            
        except Exception as e:
            logger.error(f"Crawl error: {url} - {e}")
            return CrawlResult(
                url=url,
                status=CrawlStatus.FAILED,
                error=str(e)
            )
    
    async def _worker(
        self,
        report: CrawlReport,
        base_domain: str
    ):
        """
        Worker coroutine that processes URLs from queue.
        
        Args:
            report: Crawl report to populate
            base_domain: Base domain for filtering
        """
        while True:
            try:
                # Get next URL and depth from queue
                url, depth = await asyncio.wait_for(
                    self.queue.get(),
                    timeout=5.0
                )
                
                # Skip if already visited or max pages reached
                normalized_url = self._normalize_url(url)
                if normalized_url in self.visited:
                    self.queue.task_done()
                    continue
                
                if len(self.visited) >= self.max_pages:
                    self.queue.task_done()
                    continue
                
                # Mark as visited
                self.visited.add(normalized_url)
                
                # Fetch and process page
                result = await self._fetch_page(url, depth)
                report.results.append(result)
                
                if result.status == CrawlStatus.SUCCESS:
                    report.pages_crawled += 1
                    
                    # Process extracted data
                    self._update_report(report, result, url, base_domain)
                    
                    # Add discovered URLs to queue (if within depth limit)
                    if depth < self.max_depth:
                        for discovered_url in result.extracted.urls:
                            if self._is_valid_url(discovered_url, base_domain):
                                norm_discovered = self._normalize_url(discovered_url)
                                if norm_discovered not in self.visited:
                                    await self.queue.put((discovered_url, depth + 1))
                else:
                    report.pages_failed += 1
                    if result.error:
                        report.errors.append(f"{url}: {result.error}")
                
                # Politeness delay
                await asyncio.sleep(self.delay)
                
                self.queue.task_done()
                
            except asyncio.TimeoutError:
                # Queue is empty
                break
            except Exception as e:
                logger.error(f"Worker error: {e}")
                break
    
    def _update_report(
        self,
        report: CrawlReport,
        result: CrawlResult,
        url: str,
        base_domain: str
    ):
        """Update report with extracted data."""
        extracted = result.extracted
        
        # Categorize URLs
        for discovered_url in extracted.urls:
            report.all_urls.add(discovered_url)
            parsed = urlparse(discovered_url)
            if base_domain in parsed.netloc:
                report.internal_urls.add(discovered_url)
            else:
                report.external_urls.add(discovered_url)
        
        # Aggregate other data
        report.emails.update(extracted.emails)
        report.phones.update(extracted.phones)
        report.api_endpoints.update(extracted.api_endpoints)
        report.js_files.update(extracted.js_files)
        report.subdomains.update(extracted.subdomains)
        report.secrets.extend(extracted.secrets)
        report.social_profiles.extend(extracted.social_profiles)
        report.forms.extend(extracted.forms)
    
    async def crawl(self, seed_url: str) -> CrawlReport:
        """
        Crawl website starting from seed URL.
        
        Args:
            seed_url: Starting URL to crawl
        
        Returns:
            Complete crawl report
        """
        # Parse seed URL
        parsed = urlparse(seed_url)
        domain = parsed.netloc
        base_domain = domain.replace('www.', '')
        
        # Initialize report
        report = CrawlReport(
            domain=domain,
            seed_url=seed_url
        )
        
        logger.scan_started("web_crawl", [seed_url])
        logger.info(f"Starting crawl: {seed_url}", {
            "max_depth": self.max_depth,
            "max_pages": self.max_pages
        })
        
        start_time = datetime.now()
        
        # Initialize state
        self.visited = set()
        self.queue = asyncio.Queue()
        self.semaphore = asyncio.Semaphore(self.max_concurrent)
        
        # Create session
        connector = aiohttp.TCPConnector(limit=self.max_concurrent, ssl=False)
        self.session = aiohttp.ClientSession(
            headers=self.headers,
            connector=connector
        )
        
        try:
            # Seed the queue
            await self.queue.put((seed_url, 0))
            
            # Start workers
            workers = [
                asyncio.create_task(self._worker(report, base_domain))
                for _ in range(self.max_concurrent)
            ]
            
            # Wait for queue to empty
            await self.queue.join()
            
            # Cancel workers
            for worker in workers:
                worker.cancel()
            
        finally:
            await self.session.close()
        
        # Finalize report
        report.completed_at = datetime.now(timezone.utc).isoformat()
        duration = (datetime.now() - start_time).total_seconds()
        
        logger.scan_completed(
            "web_crawl",
            duration,
            report.pages_crawled,
            report.pages_failed
        )
        
        # Log findings summary
        logger.info("Crawl completed", {
            "pages_crawled": report.pages_crawled,
            "urls_found": len(report.all_urls),
            "emails_found": len(report.emails),
            "secrets_found": len(report.secrets),
            "duration_seconds": round(duration, 2)
        })
        
        # Alert on significant findings
        if report.secrets:
            logger.threat_detected(
                threat_type="exposed_secrets",
                severity="CRITICAL" if len(report.secrets) > 5 else "HIGH",
                domain=domain,
                source="Web Crawler",
                details={
                    "secrets_count": len(report.secrets),
                    "secret_types": list(set(s['type'] for s in report.secrets))
                }
            )
        
        return report
    
    def export_report(
        self,
        report: CrawlReport,
        output_path: Path,
        format: str = "json"
    ):
        """
        Export crawl report.
        
        Args:
            report: Crawl report to export
            output_path: Output file path
            format: Export format (json, markdown)
        """
        output_path = Path(output_path)
        
        if format == "json":
            with open(output_path, 'w') as f:
                json.dump(report.to_dict(), f, indent=2, default=str)
        
        elif format == "markdown":
            md = self._generate_markdown(report)
            with open(output_path, 'w') as f:
                f.write(md)
        
        logger.info(f"Report exported: {output_path}", {"format": format})
    
    def _generate_markdown(self, report: CrawlReport) -> str:
        """Generate markdown report."""
        lines = [
            f"# ShadowHunter Crawl Report",
            f"",
            f"**Target:** {report.domain}",
            f"**Seed URL:** {report.seed_url}",
            f"**Started:** {report.started_at}",
            f"**Completed:** {report.completed_at}",
            f"",
            f"## Summary",
            f"",
            f"| Metric | Value |",
            f"|--------|-------|",
            f"| Pages Crawled | {report.pages_crawled} |",
            f"| Pages Failed | {report.pages_failed} |",
            f"| Total URLs | {len(report.all_urls)} |",
            f"| Internal URLs | {len(report.internal_urls)} |",
            f"| External URLs | {len(report.external_urls)} |",
            f"| Emails Found | {len(report.emails)} |",
            f"| API Endpoints | {len(report.api_endpoints)} |",
            f"| Secrets Found | {len(report.secrets)} |",
            f"",
        ]
        
        # Emails
        if report.emails:
            lines.append("## Emails")
            lines.append("")
            for email in sorted(report.emails):
                lines.append(f"- {email}")
            lines.append("")
        
        # Secrets
        if report.secrets:
            lines.append("## ⚠️ Secrets Detected")
            lines.append("")
            for secret in report.secrets:
                lines.append(f"- **{secret['type']}**: `{secret['value']}`")
            lines.append("")
        
        # API Endpoints
        if report.api_endpoints:
            lines.append("## API Endpoints")
            lines.append("")
            for endpoint in sorted(report.api_endpoints):
                lines.append(f"- `{endpoint}`")
            lines.append("")
        
        # Subdomains
        if report.subdomains:
            lines.append("## Subdomains")
            lines.append("")
            for subdomain in sorted(report.subdomains):
                lines.append(f"- {subdomain}")
            lines.append("")
        
        return "\n".join(lines)


# ============================================================================
# ONION LINK MAPPER - Consolidated from Darkweb Monitor
# ============================================================================

@dataclass
class OnionLink:
    """
    Represents a discovered .onion link.
    
    Tracks where the link was found and its relationship
    to other dark web sites for network mapping.
    """
    url: str
    domain: str
    source_url: str  # Where it was found
    source_domain: str
    anchor_text: Optional[str] = None
    context: Optional[str] = None  # Surrounding text
    discovered_at: str = ""
    verified: bool = False
    
    def __post_init__(self):
        if not self.discovered_at:
            self.discovered_at = datetime.now(timezone.utc).isoformat()
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary."""
        return {
            "url": self.url,
            "domain": self.domain,
            "source_url": self.source_url,
            "source_domain": self.source_domain,
            "anchor_text": self.anchor_text,
            "context": self.context,
            "discovered_at": self.discovered_at,
            "verified": self.verified
        }


@dataclass
class OnionNetwork:
    """
    Network map of .onion site relationships.
    
    Tracks which sites link to each other to build
    a graph of dark web interconnections.
    """
    sites: Dict[str, Dict[str, Any]] = field(default_factory=dict)
    edges: List[Tuple[str, str]] = field(default_factory=list)
    links: List[OnionLink] = field(default_factory=list)
    
    def add_link(self, link: OnionLink):
        """Add a discovered link to the network."""
        # Add source site
        if link.source_domain not in self.sites:
            self.sites[link.source_domain] = {
                "domain": link.source_domain,
                "outgoing": [],
                "incoming": [],
                "first_seen": link.discovered_at
            }
        
        # Add target site
        if link.domain not in self.sites:
            self.sites[link.domain] = {
                "domain": link.domain,
                "outgoing": [],
                "incoming": [],
                "first_seen": link.discovered_at
            }
        
        # Add edge
        self.sites[link.source_domain]["outgoing"].append(link.domain)
        self.sites[link.domain]["incoming"].append(link.source_domain)
        self.edges.append((link.source_domain, link.domain))
        self.links.append(link)
    
    def get_statistics(self) -> Dict[str, Any]:
        """Get network statistics."""
        if not self.sites:
            return {"total_sites": 0, "total_edges": 0}
        
        # Find most connected sites
        most_outgoing = max(
            self.sites.items(),
            key=lambda x: len(x[1]["outgoing"]),
            default=(None, {"outgoing": []})
        )
        
        most_incoming = max(
            self.sites.items(),
            key=lambda x: len(x[1]["incoming"]),
            default=(None, {"incoming": []})
        )
        
        return {
            "total_sites": len(self.sites),
            "total_edges": len(self.edges),
            "unique_edges": len(set(self.edges)),
            "most_linked_to": {
                "domain": most_incoming[0],
                "count": len(most_incoming[1]["incoming"])
            } if most_incoming[0] else None,
            "most_links_out": {
                "domain": most_outgoing[0],
                "count": len(most_outgoing[1]["outgoing"])
            } if most_outgoing[0] else None
        }
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        return {
            "statistics": self.get_statistics(),
            "sites": self.sites,
            "edges": [{"source": s, "target": t} for s, t in self.edges],
            "links": [l.to_dict() for l in self.links[:100]]  # Limit for export
        }


class OnionLinkMapper:
    """
    Extracts and maps .onion links from crawled content.
    
    Provides:
    - .onion URL extraction from HTML/text
    - Anchor text and context capture
    - Network relationship mapping
    - Link verification preparation
    
    Usage:
        mapper = OnionLinkMapper()
        links = mapper.extract_from_html(html, source_url)
        mapper.add_to_network(links)
        network = mapper.get_network()
    """
    
    # Patterns for .onion URL extraction
    ONION_PATTERNS = [
        # Full URLs with http(s)
        re.compile(
            r'https?://[a-z2-7]{16,56}\.onion(?:/[^\s"\'<>]*)?',
            re.IGNORECASE
        ),
        # Bare domains
        re.compile(
            r'(?<![a-z0-9])[a-z2-7]{16,56}\.onion(?:/[^\s"\'<>]*)?',
            re.IGNORECASE
        ),
        # In href attributes
        re.compile(
            r'href=["\']([^"\']*?[a-z2-7]{16,56}\.onion[^"\']*)["\']',
            re.IGNORECASE
        ),
    ]
    
    # V3 onion address pattern (56 chars)
    ONION_V3_PATTERN = re.compile(r'[a-z2-7]{56}\.onion', re.IGNORECASE)
    
    # V2 onion address pattern (16 chars - deprecated but still exists)
    ONION_V2_PATTERN = re.compile(r'[a-z2-7]{16}\.onion', re.IGNORECASE)
    
    def __init__(self):
        """Initialize the onion link mapper."""
        self.network = OnionNetwork()
        self._seen_links: Set[str] = set()
        
        logger.info("OnionLinkMapper initialized")
    
    def extract_domain(self, url: str) -> str:
        """Extract .onion domain from URL."""
        # Find the onion address in the URL
        match = self.ONION_V3_PATTERN.search(url)
        if match:
            return match.group()
        
        match = self.ONION_V2_PATTERN.search(url)
        if match:
            return match.group()
        
        return ""
    
    def extract_from_html(
        self,
        html: str,
        source_url: str,
        extract_context: bool = True
    ) -> List[OnionLink]:
        """
        Extract .onion links from HTML content.
        
        Args:
            html: HTML content to parse
            source_url: URL of the page being parsed
            extract_context: Whether to extract surrounding context
        
        Returns:
            List of OnionLink objects
        """
        links = []
        source_domain = self.extract_domain(source_url)
        
        # Track unique links in this extraction
        seen_in_page: Set[str] = set()
        
        # Extract from href attributes with anchor text
        href_pattern = re.compile(
            r'<a[^>]*href=["\']([^"\']*?([a-z2-7]{16,56}\.onion)[^"\']*)["\'][^>]*>(.*?)</a>',
            re.IGNORECASE | re.DOTALL
        )
        
        for match in href_pattern.finditer(html):
            full_url = match.group(1)
            domain = match.group(2)
            anchor_text = re.sub(r'<[^>]+>', '', match.group(3)).strip()
            
            if domain in seen_in_page:
                continue
            
            seen_in_page.add(domain)
            
            # Normalize URL
            if not full_url.startswith('http'):
                full_url = f"http://{full_url}"
            
            # Extract context if requested
            context = None
            if extract_context:
                context = self._extract_context(html, match.start(), match.end())
            
            link = OnionLink(
                url=full_url,
                domain=domain.lower(),
                source_url=source_url,
                source_domain=source_domain,
                anchor_text=anchor_text[:200] if anchor_text else None,
                context=context
            )
            
            links.append(link)
        
        # Also find bare onion addresses in text
        for pattern in [self.ONION_V3_PATTERN, self.ONION_V2_PATTERN]:
            for match in pattern.finditer(html):
                domain = match.group().lower()
                
                if domain in seen_in_page:
                    continue
                
                seen_in_page.add(domain)
                
                # Skip if it's self-reference
                if domain == source_domain:
                    continue
                
                context = None
                if extract_context:
                    context = self._extract_context(html, match.start(), match.end())
                
                link = OnionLink(
                    url=f"http://{domain}",
                    domain=domain,
                    source_url=source_url,
                    source_domain=source_domain,
                    anchor_text=None,
                    context=context
                )
                
                links.append(link)
        
        logger.verbose(f"Extracted {len(links)} onion links from {source_url[:50]}")
        return links
    
    def _extract_context(
        self,
        text: str,
        start: int,
        end: int,
        window: int = 100
    ) -> str:
        """Extract surrounding context for a match."""
        context_start = max(0, start - window)
        context_end = min(len(text), end + window)
        
        context = text[context_start:context_end]
        
        # Clean HTML tags and normalize whitespace
        context = re.sub(r'<[^>]+>', ' ', context)
        context = ' '.join(context.split())
        
        return context[:300]  # Limit length
    
    def add_to_network(self, links: List[OnionLink]):
        """
        Add extracted links to the network map.
        
        Args:
            links: List of OnionLink objects to add
        """
        for link in links:
            # Skip duplicates
            link_key = f"{link.source_domain}->{link.domain}"
            if link_key in self._seen_links:
                continue
            
            self._seen_links.add(link_key)
            self.network.add_link(link)
        
        logger.verbose(f"Network now contains {len(self.network.sites)} sites")
    
    def get_network(self) -> OnionNetwork:
        """Get the current network map."""
        return self.network
    
    def get_links_from_domain(self, domain: str) -> List[OnionLink]:
        """Get all links originating from a domain."""
        return [l for l in self.network.links if l.source_domain == domain]
    
    def get_links_to_domain(self, domain: str) -> List[OnionLink]:
        """Get all links pointing to a domain."""
        return [l for l in self.network.links if l.domain == domain]
    
    def export_for_visualization(self) -> Dict[str, Any]:
        """
        Export network data for visualization tools.
        
        Returns data suitable for tools like D3.js, Gephi, etc.
        """
        nodes = []
        for domain, data in self.network.sites.items():
            nodes.append({
                "id": domain,
                "label": domain[:20] + "..." if len(domain) > 20 else domain,
                "inDegree": len(data["incoming"]),
                "outDegree": len(data["outgoing"]),
                "size": len(data["incoming"]) + len(data["outgoing"])
            })
        
        edges = []
        for i, (source, target) in enumerate(set(self.network.edges)):
            edges.append({
                "id": f"e{i}",
                "source": source,
                "target": target
            })
        
        return {
            "nodes": nodes,
            "edges": edges,
            "statistics": self.network.get_statistics()
        }
    
    def clear(self):
        """Clear the network map."""
        self.network = OnionNetwork()
        self._seen_links.clear()


# Add OnionLinkMapper integration to DataExtractor
def extract_onion_links(content: str, base_url: str) -> List[str]:
    """
    Standalone function to extract .onion URLs from content.
    
    Args:
        content: HTML/text content
        base_url: Source URL
    
    Returns:
        List of .onion URLs
    """
    mapper = OnionLinkMapper()
    links = mapper.extract_from_html(content, base_url, extract_context=False)
    return [link.url for link in links]


# Extend CrawlReport with onion mapping
@dataclass
class OnionCrawlReport(CrawlReport):
    """
    Extended crawl report with onion network mapping.
    
    Adds dark web-specific data to standard crawl report.
    """
    onion_links: List[OnionLink] = field(default_factory=list)
    onion_network: Optional[OnionNetwork] = None
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary including onion data."""
        base_dict = super().to_dict()
        
        base_dict["onion_data"] = {
            "onion_links_found": len(self.onion_links),
            "unique_onion_domains": len(set(l.domain for l in self.onion_links)),
            "network": self.onion_network.to_dict() if self.onion_network else None
        }
        
        return base_dict


# ============================================================================
# DEMO & TESTING
# ============================================================================

async def demo_crawler():
    """Demonstrate crawler capabilities."""
    print("=" * 80)
    print("ShadowHunter Web Crawler Demo")
    print("=" * 80)
    
    # Create crawler with limited scope for demo
    crawler = ShadowCrawler(
        max_depth=2,
        max_pages=20,
        max_concurrent=5,
        delay=0.1  # Fast for demo
    )
    
    # Demo URL (use httpbin for safe testing)
    demo_url = "https://httpbin.org"
    
    print(f"\n🕷️ Starting crawl of: {demo_url}")
    print("-" * 60)
    
    try:
        report = await crawler.crawl(demo_url)
        
        print(f"\n✅ Crawl completed!")
        print(f"   Pages crawled: {report.pages_crawled}")
        print(f"   Pages failed: {report.pages_failed}")
        print(f"   URLs discovered: {len(report.all_urls)}")
        print(f"   Internal URLs: {len(report.internal_urls)}")
        print(f"   External URLs: {len(report.external_urls)}")
        print(f"   Emails found: {len(report.emails)}")
        print(f"   API endpoints: {len(report.api_endpoints)}")
        print(f"   Secrets found: {len(report.secrets)}")
        
        # Show sample data
        if report.api_endpoints:
            print(f"\n📍 Sample API Endpoints:")
            for endpoint in list(report.api_endpoints)[:5]:
                print(f"   - {endpoint}")
        
        if report.emails:
            print(f"\n📧 Emails Found:")
            for email in list(report.emails)[:5]:
                print(f"   - {email}")
        
    except Exception as e:
        print(f"\n❌ Error during crawl: {e}")
    
    print("\n" + "=" * 80)
    print("✅ Demo completed!")
    print("=" * 80)


async def demo_onion_mapper():
    """Demonstrate onion link mapping capabilities."""
    print("=" * 80)
    print("ShadowHunter Onion Link Mapper Demo")
    print("=" * 80)
    
    mapper = OnionLinkMapper()
    
    # Example HTML with onion links
    sample_html = """
    <html>
    <body>
        <h1>Dark Web Directory</h1>
        <ul>
            <li><a href="http://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion">
                DuckDuckGo (Privacy Search)
            </a></li>
            <li><a href="http://nytimesn7cgmftshazwhfgzm37qxb44r64ytbb2dj3x62d2lljsciiyd.onion">
                NY Times
            </a></li>
            <li><a href="http://secrdrop5wyphb5x3iuzn2e76wbapxzewavfhmzmn4tgc4vqdyj4mdqd.onion">
                SecureDrop
            </a></li>
        </ul>
        
        <p>Also check out these sites:</p>
        <p>protonmailrmez3lotccipshtkleegetolb73fuirgj7r4o4vfu7ozyd.onion for email</p>
        <p>facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion for social</p>
    </body>
    </html>
    """
    
    source_url = "http://hiddenwiki2c3bkd7r6.onion/directory"
    
    print(f"\n🔍 Extracting onion links from: {source_url}")
    print("-" * 60)
    
    links = mapper.extract_from_html(sample_html, source_url)
    
    print(f"\n📋 Found {len(links)} onion links:")
    for i, link in enumerate(links, 1):
        print(f"\n   {i}. {link.domain}")
        if link.anchor_text:
            print(f"      Anchor: {link.anchor_text}")
        if link.context:
            print(f"      Context: {link.context[:60]}...")
    
    # Add to network
    mapper.add_to_network(links)
    
    # Add another page for relationship demonstration
    sample_html_2 = """
    <a href="http://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion">DDG</a>
    <a href="http://hiddenwiki2c3bkd7r6.onion">Back to Wiki</a>
    """
    
    source_url_2 = "http://nytimesn7cgmftshazwhfgzm37qxb44r64ytbb2dj3x62d2lljsciiyd.onion"
    
    links_2 = mapper.extract_from_html(sample_html_2, source_url_2)
    mapper.add_to_network(links_2)
    
    # Show network statistics
    network = mapper.get_network()
    stats = network.get_statistics()
    
    print(f"\n🌐 Network Statistics:")
    print(f"   Total sites: {stats['total_sites']}")
    print(f"   Total edges: {stats['total_edges']}")
    print(f"   Unique edges: {stats['unique_edges']}")
    
    if stats.get('most_linked_to'):
        print(f"   Most linked to: {stats['most_linked_to']['domain'][:40]}... ({stats['most_linked_to']['count']} links)")
    
    if stats.get('most_links_out'):
        print(f"   Most links out: {stats['most_links_out']['domain'][:40]}... ({stats['most_links_out']['count']} links)")
    
    # Export for visualization
    viz_data = mapper.export_for_visualization()
    print(f"\n📊 Visualization Export:")
    print(f"   Nodes: {len(viz_data['nodes'])}")
    print(f"   Edges: {len(viz_data['edges'])}")
    
    print("\n" + "=" * 80)
    print("✅ Onion Mapper demo complete!")
    print("=" * 80)


if __name__ == "__main__":
    import sys
    
    if len(sys.argv) > 1 and sys.argv[1] == "onion":
        asyncio.run(demo_onion_mapper())
    else:
        asyncio.run(demo_crawler())