shadowhunter_scheduler.py

#!/usr/bin/env python3
"""
ShadowHunter - Dark Web Credential Intelligence Platform
Module: Scheduled Site Monitoring & Change Detection
Author: Fevra
Version: 0.1.0

Consolidated from Darkweb Monitor research - provides:
- Scheduled .onion site monitoring with configurable intervals
- Content change detection using content hashing
- Keyword monitoring with contextual alerts
- Content snapshots for forensic analysis
- Database-backed monitoring history
- Integration with ShadowHunter alert system

References:
- darkweb_monitor_script.py patterns
- Raspberry Pi Dark Web Monitor research
"""

import asyncio
import hashlib
import json
import re
import sqlite3
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Optional, Any, Callable, Set
from dataclasses import dataclass, field
from pathlib import Path
from enum import Enum
from collections import defaultdict
import threading

# Import ShadowHunter modules
try:
    from shadowhunter_logger import get_logger, LogLevel
    from shadowhunter_tor import TorConnection, TorScraper
except ImportError:
    import logging
    def get_logger(name, **kwargs):
        return logging.getLogger(name)
    TorConnection = None
    TorScraper = None

# Initialize module logger
logger = get_logger("Scheduler", log_level=LogLevel.DEBUG)


# ============================================================================
# CONFIGURATION & ENUMS
# ============================================================================

class MonitorStatus(str, Enum):
    """Site monitoring status."""
    ONLINE = "online"
    OFFLINE = "offline"
    ERROR = "error"
    TIMEOUT = "timeout"
    CHANGED = "changed"


class ChangeType(str, Enum):
    """Types of detected changes."""
    CONTENT = "content"
    KEYWORD = "keyword"
    STRUCTURE = "structure"
    NEW_LINKS = "new_links"
    STATUS = "status"


class CheckInterval(Enum):
    """Predefined check intervals in seconds."""
    RAPID = 300       # 5 minutes - for critical sites
    FREQUENT = 900    # 15 minutes
    STANDARD = 3600   # 1 hour - default
    RELAXED = 7200    # 2 hours
    DAILY = 86400     # 24 hours


# ============================================================================
# DATA MODELS
# ============================================================================

@dataclass
class MonitoredSite:
    """
    Configuration for a monitored .onion site.
    
    Attributes:
        id: Unique site identifier
        url: Full .onion URL
        name: Human-readable site name
        category: Site category (marketplace, forum, paste, etc.)
        keywords: List of keywords to monitor for
        check_interval: Seconds between checks
        active: Whether monitoring is enabled
        priority: Priority level (1=highest)
        last_checked: Last check timestamp
        last_status: Last known status
        metadata: Additional site metadata
    """
    id: int
    url: str
    name: str
    category: str = "unknown"
    keywords: List[str] = field(default_factory=list)
    check_interval: int = 3600
    active: bool = True
    priority: int = 5
    last_checked: Optional[datetime] = None
    last_status: MonitorStatus = MonitorStatus.OFFLINE
    metadata: Dict[str, Any] = field(default_factory=dict)
    
    def is_due_for_check(self) -> bool:
        """Check if site is due for monitoring."""
        if not self.last_checked:
            return True
        
        elapsed = (datetime.now(timezone.utc) - self.last_checked).total_seconds()
        return elapsed >= self.check_interval
    
    def get_domain(self) -> str:
        """Extract .onion domain from URL."""
        from urllib.parse import urlparse
        parsed = urlparse(self.url)
        return parsed.netloc


@dataclass
class ContentSnapshot:
    """
    Content snapshot for change tracking.
    
    Stores both the hash and preview for efficient comparison
    while maintaining ability to review actual content changes.
    """
    site_id: int
    timestamp: datetime
    content_hash: str
    content_preview: str  # First 1000 chars
    title: str
    word_count: int
    link_count: int
    links_hash: str  # Hash of all extracted links
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class ChangeDetection:
    """
    Detected change on a monitored site.
    
    Captures what changed, when, and provides context
    for alert generation and analysis.
    """
    site_id: int
    site_name: str
    site_url: str
    change_type: ChangeType
    timestamp: datetime
    severity: str  # CRITICAL, HIGH, MEDIUM, LOW
    description: str
    old_value: Optional[str] = None
    new_value: Optional[str] = None
    context: str = ""
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class KeywordMatch:
    """
    Keyword match detection.
    
    Captures where and how a keyword was found
    with surrounding context for analysis.
    """
    site_id: int
    site_name: str
    keyword: str
    context: str  # Surrounding text
    match_count: int
    timestamp: datetime
    url: str


@dataclass
class CheckResult:
    """
    Result from a single site check.
    
    Contains all information about a monitoring check
    including status, timing, and extracted data.
    """
    site: MonitoredSite
    status: MonitorStatus
    timestamp: datetime
    response_time_ms: float
    content_hash: Optional[str] = None
    title: Optional[str] = None
    content_preview: Optional[str] = None
    word_count: int = 0
    links: List[str] = field(default_factory=list)
    changes: List[ChangeDetection] = field(default_factory=list)
    keyword_matches: List[KeywordMatch] = field(default_factory=list)
    error: Optional[str] = None


# ============================================================================
# DATABASE MANAGER
# ============================================================================

class SchedulerDatabase:
    """
    SQLite database manager for scheduled monitoring.
    
    Provides persistent storage for:
    - Monitored sites configuration
    - Check history and results
    - Content snapshots for change detection
    - Keyword match alerts
    - Activity logging
    """
    
    def __init__(self, db_path: str = "shadowhunter_scheduler.db"):
        """
        Initialize database connection.
        
        Args:
            db_path: Path to SQLite database file
        """
        self.db_path = Path(db_path)
        self._lock = threading.Lock()
        self._init_database()
        
        logger.info("Scheduler database initialized", {
            "db_path": str(self.db_path)
        })
    
    def _get_connection(self) -> sqlite3.Connection:
        """Get database connection with row factory."""
        conn = sqlite3.connect(str(self.db_path))
        conn.row_factory = sqlite3.Row
        return conn
    
    def _init_database(self):
        """Initialize database schema."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            # Monitored sites table
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS monitored_sites (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    url TEXT UNIQUE NOT NULL,
                    name TEXT NOT NULL,
                    category TEXT DEFAULT 'unknown',
                    keywords TEXT DEFAULT '[]',
                    check_interval INTEGER DEFAULT 3600,
                    priority INTEGER DEFAULT 5,
                    active INTEGER DEFAULT 1,
                    last_checked TIMESTAMP,
                    last_status TEXT DEFAULT 'offline',
                    last_content_hash TEXT,
                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    metadata TEXT DEFAULT '{}'
                )
            ''')
            
            # Site checks history
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS site_checks (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    site_id INTEGER NOT NULL,
                    check_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    status TEXT NOT NULL,
                    response_time_ms REAL,
                    content_hash TEXT,
                    title TEXT,
                    word_count INTEGER DEFAULT 0,
                    link_count INTEGER DEFAULT 0,
                    error TEXT,
                    FOREIGN KEY (site_id) REFERENCES monitored_sites(id)
                )
            ''')
            
            # Content snapshots for forensics
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS content_snapshots (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    site_id INTEGER NOT NULL,
                    snapshot_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    content_hash TEXT NOT NULL,
                    content_preview TEXT,
                    title TEXT,
                    word_count INTEGER,
                    link_count INTEGER,
                    links_hash TEXT,
                    metadata TEXT DEFAULT '{}',
                    FOREIGN KEY (site_id) REFERENCES monitored_sites(id)
                )
            ''')
            
            # Change detections
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS change_detections (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    site_id INTEGER NOT NULL,
                    detected_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    change_type TEXT NOT NULL,
                    severity TEXT NOT NULL,
                    description TEXT,
                    old_value TEXT,
                    new_value TEXT,
                    context TEXT,
                    alerted INTEGER DEFAULT 0,
                    FOREIGN KEY (site_id) REFERENCES monitored_sites(id)
                )
            ''')
            
            # Keyword alerts
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS keyword_alerts (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    site_id INTEGER NOT NULL,
                    keyword TEXT NOT NULL,
                    context TEXT,
                    match_count INTEGER DEFAULT 1,
                    detected_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    alerted INTEGER DEFAULT 0,
                    FOREIGN KEY (site_id) REFERENCES monitored_sites(id)
                )
            ''')
            
            # Create indices for performance
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_checks_site ON site_checks(site_id)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_checks_time ON site_checks(check_time)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_snapshots_site ON content_snapshots(site_id)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_changes_site ON change_detections(site_id)')
            cursor.execute('CREATE INDEX IF NOT EXISTS idx_keywords_site ON keyword_alerts(site_id)')
            
            conn.commit()
            conn.close()
    
    def add_site(
        self,
        url: str,
        name: str,
        category: str = "unknown",
        keywords: List[str] = None,
        check_interval: int = 3600,
        priority: int = 5,
        metadata: Dict[str, Any] = None
    ) -> Optional[int]:
        """
        Add a new site to monitoring.
        
        Args:
            url: .onion URL to monitor
            name: Human-readable name
            category: Site category
            keywords: Keywords to monitor
            check_interval: Check frequency in seconds
            priority: Priority level (1=highest)
            metadata: Additional metadata
        
        Returns:
            Site ID if successful, None if already exists
        """
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            try:
                cursor.execute('''
                    INSERT INTO monitored_sites 
                    (url, name, category, keywords, check_interval, priority, metadata)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                ''', (
                    url,
                    name,
                    category,
                    json.dumps(keywords or []),
                    check_interval,
                    priority,
                    json.dumps(metadata or {})
                ))
                
                conn.commit()
                site_id = cursor.lastrowid
                
                logger.info(f"Added site to monitoring: {name}", {
                    "site_id": site_id,
                    "url": url[:50],
                    "category": category
                })
                
                return site_id
                
            except sqlite3.IntegrityError:
                logger.warn(f"Site already exists: {url[:50]}")
                return None
            finally:
                conn.close()
    
    def get_active_sites(self) -> List[MonitoredSite]:
        """Get all active monitored sites."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            cursor.execute('''
                SELECT * FROM monitored_sites WHERE active = 1
                ORDER BY priority ASC, last_checked ASC
            ''')
            
            sites = []
            for row in cursor.fetchall():
                sites.append(MonitoredSite(
                    id=row['id'],
                    url=row['url'],
                    name=row['name'],
                    category=row['category'],
                    keywords=json.loads(row['keywords']) if row['keywords'] else [],
                    check_interval=row['check_interval'],
                    active=bool(row['active']),
                    priority=row['priority'],
                    last_checked=datetime.fromisoformat(row['last_checked']) if row['last_checked'] else None,
                    last_status=MonitorStatus(row['last_status']) if row['last_status'] else MonitorStatus.OFFLINE,
                    metadata=json.loads(row['metadata']) if row['metadata'] else {}
                ))
            
            conn.close()
            return sites
    
    def get_sites_due_for_check(self) -> List[MonitoredSite]:
        """Get sites that are due for checking based on their intervals."""
        sites = self.get_active_sites()
        return [site for site in sites if site.is_due_for_check()]
    
    def get_last_snapshot(self, site_id: int) -> Optional[ContentSnapshot]:
        """Get the last content snapshot for a site."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            cursor.execute('''
                SELECT * FROM content_snapshots
                WHERE site_id = ?
                ORDER BY snapshot_time DESC
                LIMIT 1
            ''', (site_id,))
            
            row = cursor.fetchone()
            conn.close()
            
            if row:
                return ContentSnapshot(
                    site_id=row['site_id'],
                    timestamp=datetime.fromisoformat(row['snapshot_time']),
                    content_hash=row['content_hash'],
                    content_preview=row['content_preview'],
                    title=row['title'],
                    word_count=row['word_count'],
                    link_count=row['link_count'],
                    links_hash=row['links_hash'],
                    metadata=json.loads(row['metadata']) if row['metadata'] else {}
                )
            return None
    
    def save_check_result(self, result: CheckResult):
        """Save check result to database."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            # Save check record
            cursor.execute('''
                INSERT INTO site_checks 
                (site_id, check_time, status, response_time_ms, content_hash, 
                 title, word_count, link_count, error)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
            ''', (
                result.site.id,
                result.timestamp.isoformat(),
                result.status.value,
                result.response_time_ms,
                result.content_hash,
                result.title,
                result.word_count,
                len(result.links),
                result.error
            ))
            
            # Update site last checked
            cursor.execute('''
                UPDATE monitored_sites 
                SET last_checked = ?, last_status = ?, last_content_hash = ?
                WHERE id = ?
            ''', (
                result.timestamp.isoformat(),
                result.status.value,
                result.content_hash,
                result.site.id
            ))
            
            # Save snapshot if we have content
            if result.content_hash and result.content_preview:
                links_hash = hashlib.sha256(
                    json.dumps(sorted(result.links)).encode()
                ).hexdigest()[:16]
                
                cursor.execute('''
                    INSERT INTO content_snapshots 
                    (site_id, content_hash, content_preview, title, 
                     word_count, link_count, links_hash)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                ''', (
                    result.site.id,
                    result.content_hash,
                    result.content_preview,
                    result.title,
                    result.word_count,
                    len(result.links),
                    links_hash
                ))
            
            # Save change detections
            for change in result.changes:
                cursor.execute('''
                    INSERT INTO change_detections 
                    (site_id, change_type, severity, description, 
                     old_value, new_value, context)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                ''', (
                    result.site.id,
                    change.change_type.value,
                    change.severity,
                    change.description,
                    change.old_value,
                    change.new_value,
                    change.context
                ))
            
            # Save keyword matches
            for match in result.keyword_matches:
                cursor.execute('''
                    INSERT INTO keyword_alerts 
                    (site_id, keyword, context, match_count)
                    VALUES (?, ?, ?, ?)
                ''', (
                    result.site.id,
                    match.keyword,
                    match.context,
                    match.match_count
                ))
            
            conn.commit()
            conn.close()
    
    def get_recent_changes(
        self, 
        hours: int = 24,
        severity: Optional[str] = None
    ) -> List[ChangeDetection]:
        """Get recent change detections."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            query = '''
                SELECT cd.*, ms.name as site_name, ms.url as site_url
                FROM change_detections cd
                JOIN monitored_sites ms ON cd.site_id = ms.id
                WHERE cd.detected_time >= datetime('now', ?)
            '''
            params = [f'-{hours} hours']
            
            if severity:
                query += ' AND cd.severity = ?'
                params.append(severity)
            
            query += ' ORDER BY cd.detected_time DESC'
            
            cursor.execute(query, params)
            
            changes = []
            for row in cursor.fetchall():
                changes.append(ChangeDetection(
                    site_id=row['site_id'],
                    site_name=row['site_name'],
                    site_url=row['site_url'],
                    change_type=ChangeType(row['change_type']),
                    timestamp=datetime.fromisoformat(row['detected_time']),
                    severity=row['severity'],
                    description=row['description'],
                    old_value=row['old_value'],
                    new_value=row['new_value'],
                    context=row['context']
                ))
            
            conn.close()
            return changes
    
    def get_statistics(self) -> Dict[str, Any]:
        """Get monitoring statistics."""
        with self._lock:
            conn = self._get_connection()
            cursor = conn.cursor()
            
            stats = {}
            
            # Total sites
            cursor.execute('SELECT COUNT(*) FROM monitored_sites')
            stats['total_sites'] = cursor.fetchone()[0]
            
            # Active sites
            cursor.execute('SELECT COUNT(*) FROM monitored_sites WHERE active = 1')
            stats['active_sites'] = cursor.fetchone()[0]
            
            # Online sites (last check)
            cursor.execute('''
                SELECT COUNT(*) FROM monitored_sites 
                WHERE active = 1 AND last_status = 'online'
            ''')
            stats['online_sites'] = cursor.fetchone()[0]
            
            # Checks in last 24h
            cursor.execute('''
                SELECT COUNT(*) FROM site_checks 
                WHERE check_time >= datetime('now', '-24 hours')
            ''')
            stats['checks_24h'] = cursor.fetchone()[0]
            
            # Changes in last 24h
            cursor.execute('''
                SELECT COUNT(*) FROM change_detections 
                WHERE detected_time >= datetime('now', '-24 hours')
            ''')
            stats['changes_24h'] = cursor.fetchone()[0]
            
            # Keyword matches in last 24h
            cursor.execute('''
                SELECT COUNT(*) FROM keyword_alerts 
                WHERE detected_time >= datetime('now', '-24 hours')
            ''')
            stats['keyword_matches_24h'] = cursor.fetchone()[0]
            
            conn.close()
            return stats


# ============================================================================
# CONTENT ANALYZER
# ============================================================================

class ContentAnalyzer:
    """
    Analyzes content for changes and keyword matches.
    
    Provides:
    - Content hashing for change detection
    - Keyword extraction with context
    - Link extraction and comparison
    - Structural change detection
    """
    
    def __init__(self):
        """Initialize content analyzer."""
        self.link_pattern = re.compile(
            r'href=["\']([^"\']+)["\']',
            re.IGNORECASE
        )
        self.onion_pattern = re.compile(
            r'[a-z2-7]{16,56}\.onion',
            re.IGNORECASE
        )
    
    def compute_hash(self, content: str) -> str:
        """
        Compute content hash for change detection.
        
        Args:
            content: Raw text content
        
        Returns:
            SHA-256 hash of normalized content
        """
        # Normalize whitespace for consistent hashing
        normalized = ' '.join(content.split())
        return hashlib.sha256(normalized.encode('utf-8')).hexdigest()
    
    def extract_text(self, html: str) -> str:
        """
        Extract clean text from HTML.
        
        Args:
            html: Raw HTML content
        
        Returns:
            Cleaned text content
        """
        from bs4 import BeautifulSoup
        
        soup = BeautifulSoup(html, 'html.parser')
        
        # Remove script and style elements
        for element in soup(['script', 'style', 'meta', 'link']):
            element.decompose()
        
        text = soup.get_text(separator=' ', strip=True)
        return text
    
    def extract_title(self, html: str) -> str:
        """Extract page title from HTML."""
        from bs4 import BeautifulSoup
        
        soup = BeautifulSoup(html, 'html.parser')
        if soup.title and soup.title.string:
            return soup.title.string.strip()
        return "No title"
    
    def extract_links(self, html: str, base_url: str) -> List[str]:
        """
        Extract all links from HTML.
        
        Args:
            html: Raw HTML content
            base_url: Base URL for resolving relative links
        
        Returns:
            List of absolute URLs
        """
        from urllib.parse import urljoin, urlparse
        
        links = set()
        
        # Extract from href attributes
        for match in self.link_pattern.finditer(html):
            href = match.group(1)
            if href and not href.startswith(('#', 'javascript:', 'mailto:')):
                absolute = urljoin(base_url, href)
                links.add(absolute)
        
        # Extract .onion addresses
        for match in self.onion_pattern.finditer(html):
            onion = match.group()
            if not onion.startswith('http'):
                onion = f"http://{onion}"
            links.add(onion)
        
        return list(links)
    
    def find_keywords(
        self,
        content: str,
        keywords: List[str],
        context_chars: int = 100
    ) -> List[KeywordMatch]:
        """
        Find keyword matches with context.
        
        Args:
            content: Text content to search
            keywords: Keywords to find
            context_chars: Characters of context to extract
        
        Returns:
            List of keyword matches
        """
        matches = []
        content_lower = content.lower()
        
        for keyword in keywords:
            keyword_lower = keyword.lower()
            
            if keyword_lower not in content_lower:
                continue
            
            # Find all occurrences with context
            pattern = re.compile(
                f'.{{0,{context_chars}}}{re.escape(keyword_lower)}.{{0,{context_chars}}}',
                re.IGNORECASE | re.DOTALL
            )
            
            found = pattern.findall(content)
            
            if found:
                # Take first 3 matches for context
                context = ' ... '.join(found[:3])
                
                matches.append(KeywordMatch(
                    site_id=0,  # Set by caller
                    site_name="",  # Set by caller
                    keyword=keyword,
                    context=context[:500],
                    match_count=len(found),
                    timestamp=datetime.now(timezone.utc),
                    url=""  # Set by caller
                ))
        
        return matches
    
    def detect_changes(
        self,
        current_snapshot: ContentSnapshot,
        previous_snapshot: Optional[ContentSnapshot]
    ) -> List[ChangeDetection]:
        """
        Detect changes between snapshots.
        
        Args:
            current_snapshot: Current content snapshot
            previous_snapshot: Previous content snapshot
        
        Returns:
            List of detected changes
        """
        changes = []
        
        if not previous_snapshot:
            # First snapshot - no changes to detect
            return changes
        
        # Content hash change
        if current_snapshot.content_hash != previous_snapshot.content_hash:
            # Calculate word count difference
            word_diff = current_snapshot.word_count - previous_snapshot.word_count
            
            # Determine severity based on change magnitude
            change_pct = abs(word_diff) / max(previous_snapshot.word_count, 1) * 100
            
            if change_pct > 50:
                severity = "CRITICAL"
            elif change_pct > 20:
                severity = "HIGH"
            elif change_pct > 5:
                severity = "MEDIUM"
            else:
                severity = "LOW"
            
            changes.append(ChangeDetection(
                site_id=current_snapshot.site_id,
                site_name="",
                site_url="",
                change_type=ChangeType.CONTENT,
                timestamp=current_snapshot.timestamp,
                severity=severity,
                description=f"Content changed ({word_diff:+d} words, {change_pct:.1f}% change)",
                old_value=f"{previous_snapshot.word_count} words",
                new_value=f"{current_snapshot.word_count} words"
            ))
        
        # Title change
        if current_snapshot.title != previous_snapshot.title:
            changes.append(ChangeDetection(
                site_id=current_snapshot.site_id,
                site_name="",
                site_url="",
                change_type=ChangeType.STRUCTURE,
                timestamp=current_snapshot.timestamp,
                severity="MEDIUM",
                description="Page title changed",
                old_value=previous_snapshot.title,
                new_value=current_snapshot.title
            ))
        
        # Links change
        if current_snapshot.links_hash != previous_snapshot.links_hash:
            link_diff = current_snapshot.link_count - previous_snapshot.link_count
            
            changes.append(ChangeDetection(
                site_id=current_snapshot.site_id,
                site_name="",
                site_url="",
                change_type=ChangeType.NEW_LINKS,
                timestamp=current_snapshot.timestamp,
                severity="MEDIUM" if abs(link_diff) > 5 else "LOW",
                description=f"Links changed ({link_diff:+d} links)",
                old_value=f"{previous_snapshot.link_count} links",
                new_value=f"{current_snapshot.link_count} links"
            ))
        
        return changes


# ============================================================================
# SITE MONITOR
# ============================================================================

class SiteMonitor:
    """
    Monitors .onion sites through Tor.
    
    Provides:
    - Async site checking via Tor
    - Content extraction and hashing
    - Change detection
    - Keyword monitoring
    """
    
    def __init__(
        self,
        tor_connection: Optional[TorConnection] = None,
        timeout: int = 30
    ):
        """
        Initialize site monitor.
        
        Args:
            tor_connection: Existing Tor connection (creates new if None)
            timeout: Request timeout in seconds
        """
        self.tor = tor_connection
        self.timeout = timeout
        self.scraper: Optional[TorScraper] = None
        self.analyzer = ContentAnalyzer()
        
        logger.info("Site monitor initialized", {"timeout": timeout})
    
    async def initialize(self) -> bool:
        """Initialize Tor connection."""
        if self.tor is None:
            self.tor = TorConnection()
        
        if await self.tor.connect():
            self.scraper = TorScraper(self.tor)
            await self.scraper.create_session()
            logger.info("Site monitor Tor connection established")
            return True
        
        logger.error("Failed to establish Tor connection")
        return False
    
    async def check_site(
        self,
        site: MonitoredSite,
        previous_snapshot: Optional[ContentSnapshot] = None
    ) -> CheckResult:
        """
        Check a single site.
        
        Args:
            site: Site to check
            previous_snapshot: Previous content snapshot for comparison
        
        Returns:
            Check result with status, content, and changes
        """
        start_time = datetime.now(timezone.utc)
        
        logger.info(f"Checking site: {site.name}", {
            "site_id": site.id,
            "url": site.url[:50]
        })
        
        try:
            # Fetch page
            fetch_start = datetime.now()
            html = await self.scraper.fetch(site.url, timeout=self.timeout)
            response_time = (datetime.now() - fetch_start).total_seconds() * 1000
            
            if html is None:
                return CheckResult(
                    site=site,
                    status=MonitorStatus.OFFLINE,
                    timestamp=start_time,
                    response_time_ms=response_time,
                    error="Failed to fetch page"
                )
            
            # Extract content
            text = self.analyzer.extract_text(html)
            title = self.analyzer.extract_title(html)
            links = self.analyzer.extract_links(html, site.url)
            content_hash = self.analyzer.compute_hash(text)
            word_count = len(text.split())
            
            # Create current snapshot
            links_hash = hashlib.sha256(
                json.dumps(sorted(links)).encode()
            ).hexdigest()[:16]
            
            current_snapshot = ContentSnapshot(
                site_id=site.id,
                timestamp=start_time,
                content_hash=content_hash,
                content_preview=text[:1000],
                title=title,
                word_count=word_count,
                link_count=len(links),
                links_hash=links_hash
            )
            
            # Detect changes
            changes = self.analyzer.detect_changes(current_snapshot, previous_snapshot)
            
            # Update change metadata
            for change in changes:
                change.site_name = site.name
                change.site_url = site.url
            
            # Find keyword matches
            keyword_matches = []
            if site.keywords:
                keyword_matches = self.analyzer.find_keywords(text, site.keywords)
                for match in keyword_matches:
                    match.site_id = site.id
                    match.site_name = site.name
                    match.url = site.url
            
            # Determine status
            status = MonitorStatus.ONLINE
            if changes:
                status = MonitorStatus.CHANGED
            
            logger.info(f"Site check complete: {site.name}", {
                "status": status.value,
                "response_time_ms": round(response_time, 2),
                "word_count": word_count,
                "link_count": len(links),
                "changes": len(changes),
                "keyword_matches": len(keyword_matches)
            })
            
            return CheckResult(
                site=site,
                status=status,
                timestamp=start_time,
                response_time_ms=response_time,
                content_hash=content_hash,
                title=title,
                content_preview=text[:1000],
                word_count=word_count,
                links=links,
                changes=changes,
                keyword_matches=keyword_matches
            )
            
        except asyncio.TimeoutError:
            return CheckResult(
                site=site,
                status=MonitorStatus.TIMEOUT,
                timestamp=start_time,
                response_time_ms=self.timeout * 1000,
                error="Request timeout"
            )
            
        except Exception as e:
            logger.error(f"Error checking site {site.name}: {e}")
            return CheckResult(
                site=site,
                status=MonitorStatus.ERROR,
                timestamp=start_time,
                response_time_ms=0,
                error=str(e)
            )
    
    async def close(self):
        """Close monitor connections."""
        if self.scraper:
            await self.scraper.close()
        if self.tor:
            self.tor.disconnect()


# ============================================================================
# SCHEDULER ENGINE
# ============================================================================

class SchedulerEngine:
    """
    Main scheduling engine for continuous monitoring.
    
    Provides:
    - Priority-based scheduling
    - Configurable check intervals
    - Automatic retry on failures
    - Integration with alert system
    """
    
    def __init__(
        self,
        db: SchedulerDatabase,
        monitor: SiteMonitor,
        alert_callback: Optional[Callable[[ChangeDetection], None]] = None,
        keyword_callback: Optional[Callable[[KeywordMatch], None]] = None
    ):
        """
        Initialize scheduler.
        
        Args:
            db: Database manager
            monitor: Site monitor
            alert_callback: Callback for change alerts
            keyword_callback: Callback for keyword matches
        """
        self.db = db
        self.monitor = monitor
        self.alert_callback = alert_callback
        self.keyword_callback = keyword_callback
        
        self._running = False
        self._check_count = 0
        
        logger.info("Scheduler engine initialized")
    
    async def run_check_cycle(self) -> List[CheckResult]:
        """
        Run a single check cycle on all due sites.
        
        Returns:
            List of check results
        """
        sites = self.db.get_sites_due_for_check()
        
        if not sites:
            logger.verbose("No sites due for checking")
            return []
        
        logger.info(f"Starting check cycle", {
            "sites_to_check": len(sites)
        })
        
        results = []
        
        for site in sites:
            # Get previous snapshot
            previous = self.db.get_last_snapshot(site.id)
            
            # Check site
            result = await self.monitor.check_site(site, previous)
            results.append(result)
            
            # Save result
            self.db.save_check_result(result)
            
            # Process changes
            for change in result.changes:
                logger.warn(f"Change detected: {change.description}", {
                    "site": site.name,
                    "change_type": change.change_type.value,
                    "severity": change.severity
                })
                
                if self.alert_callback:
                    self.alert_callback(change)
            
            # Process keyword matches
            for match in result.keyword_matches:
                logger.warn(f"Keyword match: '{match.keyword}'", {
                    "site": site.name,
                    "match_count": match.match_count
                })
                
                if self.keyword_callback:
                    self.keyword_callback(match)
            
            # Small delay between checks
            await asyncio.sleep(2)
            
            self._check_count += 1
        
        logger.info(f"Check cycle complete", {
            "sites_checked": len(results),
            "changes_found": sum(len(r.changes) for r in results),
            "keywords_found": sum(len(r.keyword_matches) for r in results)
        })
        
        return results
    
    async def run_continuous(
        self,
        check_interval: int = 60,
        max_cycles: Optional[int] = None
    ):
        """
        Run continuous monitoring.
        
        Args:
            check_interval: Seconds between check cycles
            max_cycles: Maximum cycles to run (None for infinite)
        """
        self._running = True
        cycles = 0
        
        logger.info("Starting continuous monitoring", {
            "check_interval": check_interval,
            "max_cycles": max_cycles
        })
        
        while self._running:
            try:
                await self.run_check_cycle()
                cycles += 1
                
                if max_cycles and cycles >= max_cycles:
                    logger.info(f"Completed {cycles} cycles, stopping")
                    break
                
                # Wait for next cycle
                logger.verbose(f"Next check cycle in {check_interval} seconds")
                await asyncio.sleep(check_interval)
                
            except KeyboardInterrupt:
                logger.info("Monitoring stopped by user")
                break
            except Exception as e:
                logger.error(f"Error in monitoring cycle: {e}")
                await asyncio.sleep(30)  # Wait on error
        
        self._running = False
    
    def stop(self):
        """Stop continuous monitoring."""
        self._running = False
        logger.info("Scheduler stop requested")
    
    def get_statistics(self) -> Dict[str, Any]:
        """Get scheduler statistics."""
        stats = self.db.get_statistics()
        stats['total_checks_this_session'] = self._check_count
        stats['is_running'] = self._running
        return stats


# ============================================================================
# CONVENIENCE FUNCTIONS
# ============================================================================

async def create_scheduler(
    db_path: str = "shadowhunter_scheduler.db",
    tor_config: Optional[Dict[str, Any]] = None
) -> SchedulerEngine:
    """
    Create and initialize a scheduler engine.
    
    Args:
        db_path: Path to database file
        tor_config: Tor connection configuration
    
    Returns:
        Initialized scheduler engine
    """
    db = SchedulerDatabase(db_path)
    
    tor = TorConnection(
        tor_proxy_host=tor_config.get('proxy_host', '127.0.0.1') if tor_config else '127.0.0.1',
        tor_proxy_port=tor_config.get('proxy_port', 9050) if tor_config else 9050,
        tor_control_port=tor_config.get('control_port', 9051) if tor_config else 9051
    )
    
    monitor = SiteMonitor(tor)
    
    if not await monitor.initialize():
        raise ConnectionError("Failed to initialize Tor connection")
    
    return SchedulerEngine(db, monitor)


# ============================================================================
# DEMO & TESTING
# ============================================================================

async def demo_scheduler():
    """Demonstrate scheduler capabilities."""
    print("=" * 80)
    print("ShadowHunter Scheduler Demo")
    print("=" * 80)
    
    # Initialize database
    db = SchedulerDatabase("demo_scheduler.db")
    
    # Add example sites
    example_sites = [
        {
            'url': 'http://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion',
            'name': 'DuckDuckGo Onion',
            'category': 'search',
            'keywords': ['privacy', 'search'],
            'check_interval': 3600
        },
        {
            'url': 'http://nytimesn7cgmftshazwhfgzm37qxb44r64ytbb2dj3x62d2lljsciiyd.onion',
            'name': 'NY Times Onion',
            'category': 'news',
            'keywords': ['breaking', 'security'],
            'check_interval': 1800
        }
    ]
    
    print("\n📌 Adding example sites to monitor...")
    for site in example_sites:
        site_id = db.add_site(**site)
        if site_id:
            print(f"   ✓ Added: {site['name']}")
        else:
            print(f"   • Already exists: {site['name']}")
    
    # Show statistics
    stats = db.get_statistics()
    print(f"\n📊 Scheduler Statistics:")
    print(f"   Total sites: {stats['total_sites']}")
    print(f"   Active sites: {stats['active_sites']}")
    print(f"   Checks (24h): {stats['checks_24h']}")
    print(f"   Changes (24h): {stats['changes_24h']}")
    
    # Show sites due for check
    due_sites = db.get_sites_due_for_check()
    print(f"\n🔄 Sites due for check: {len(due_sites)}")
    for site in due_sites:
        print(f"   • {site.name} ({site.check_interval}s interval)")
    
    print("\n" + "=" * 80)
    print("✅ Demo complete!")
    print("To run actual monitoring, initialize Tor and use SchedulerEngine")
    print("=" * 80)


if __name__ == "__main__":
    asyncio.run(demo_scheduler())