scraper.py

"""Scraper layer — pulls data from RSS, web pages, and APIs without any LLM."""

import hashlib
import json
import re
from collections import Counter
from datetime import datetime
from urllib.parse import urlparse

import feedparser
import requests
from bs4 import BeautifulSoup

import db


# Common stop words to filter out during keyword extraction
STOP_WORDS = set("""
a about above after again against all am an and any are aren't as at be because
been before being below between both but by can't cannot could couldn't did didn't
do does doesn't doing don't down during each few for from further get got had hadn't
has hasn't have haven't having he he'd he'll he's her here here's hers herself him
himself his how how's i i'd i'll i'm i've if in into is isn't it it's its itself
let's me more most mustn't my myself no nor not of off on once only or other ought
our ours ourselves out over own same shan't she she'd she'll she's should shouldn't
so some such than that that's the their theirs them themselves then there there's
these they they'd they'll they're they've this those through to too under until up
very was wasn't we we'd we'll we're we've were weren't what what's when when's where
where's which while who who's whom why why's will with won't would wouldn't you you'd
you'll you're you've your yours yourself yourselves just also like using used use
new one two first get got can will may
""".split())


def extract_text(html):
    """Extract clean text from HTML."""
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
        tag.decompose()
    text = soup.get_text(separator=" ", strip=True)
    text = re.sub(r'\s+', ' ', text)
    return text


def extract_keywords(text, top_n=15):
    """Extract keywords using simple TF scoring — no LLM needed."""
    words = re.findall(r'[a-z]{3,}', text.lower())
    words = [w for w in words if w not in STOP_WORDS and len(w) > 2]
    counts = Counter(words)
    return [word for word, _ in counts.most_common(top_n)]


def content_hash(text):
    """Generate hash for deduplication."""
    return hashlib.sha256(text.strip().lower().encode()).hexdigest()[:16]


def scrape_rss(source):
    """Scrape an RSS feed source."""
    url = source["url"]
    feed = feedparser.parse(url)
    added = 0

    for entry in feed.entries:
        title = entry.get("title", "Untitled")
        link = entry.get("link", "")
        summary = entry.get("summary", "")
        author = entry.get("author", "")

        # Get full content if available
        content = ""
        if hasattr(entry, "content"):
            content = entry.content[0].get("value", "")

        text = extract_text(content or summary)
        if len(text) < 50:
            continue

        h = content_hash(text)
        keywords = extract_keywords(text)

        doc_id = db.insert_document(
            source_id=source["id"],
            title=title,
            content=text,
            url=link,
            author=author,
            keywords=keywords,
            doc_hash=h
        )
        if doc_id:
            added += 1

    return added


def scrape_web(source):
    """Scrape a web page."""
    url = source["url"]
    config = json.loads(source["config"]) if source["config"] else {}

    headers = {
        "User-Agent": "KnowledgeEngine/1.0 (research bot)"
    }

    try:
        resp = requests.get(url, headers=headers, timeout=15)
        resp.raise_for_status()
    except requests.RequestException as e:
        print(f"  Error fetching {url}: {e}")
        return 0

    soup = BeautifulSoup(resp.text, "html.parser")
    title = soup.title.string if soup.title else urlparse(url).netloc

    # If selector specified in config, use it
    selector = config.get("selector")
    if selector:
        elements = soup.select(selector)
        text = " ".join(extract_text(str(el)) for el in elements)
    else:
        text = extract_text(resp.text)

    if len(text) < 50:
        return 0

    h = content_hash(text)
    keywords = extract_keywords(text)

    doc_id = db.insert_document(
        source_id=source["id"],
        title=title,
        content=text,
        url=url,
        keywords=keywords,
        doc_hash=h
    )
    return 1 if doc_id else 0


def scrape_source(source):
    """Route to the right scraper based on source type."""
    source_type = source["type"]
    if source_type == "rss":
        return scrape_rss(source)
    elif source_type == "web":
        return scrape_web(source)
    else:
        print(f"  Unknown source type: {source_type}")
        return 0


def scrape_all():
    """Scrape all enabled sources."""
    sources = db.get_sources(enabled_only=True)
    total = 0
    for source in sources:
        print(f"  Scraping: {source['name']} ({source['type']})")
        added = scrape_source(source)
        total += added
        print(f"    Added {added} documents")

        # Update last_scraped
        conn = db.get_conn()
        conn.execute(
            "UPDATE sources SET last_scraped = datetime('now') WHERE id = ?",
            (source["id"],)
        )
        conn.commit()
        conn.close()

    return total