refine.py

#!/usr/bin/env python3
"""
Knowledge Engine Refinement Agent
Runs on a schedule to:
1. Scrape all sources for new content
2. Analyze query patterns to find gaps
3. Prune low-relevance documents
4. Extract trending keywords and suggest new sources
5. Apply relevance decay to stale documents
6. Generate a refinement report
"""

import json
import os
import sys
import sqlite3
from collections import Counter
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(__file__))

import db
import scraper

REPORT_DIR = os.path.join(os.path.dirname(__file__), "reports")
os.makedirs(REPORT_DIR, exist_ok=True)


def analyze_query_patterns():
    """Find what users search for most and identify gaps."""
    conn = db.get_conn()

    # Get all queries
    queries = conn.execute(
        "SELECT query, result_count, created_at FROM query_log ORDER BY created_at DESC LIMIT 200"
    ).fetchall()

    # Find zero-result queries (gaps in our knowledge)
    gaps = [q["query"] for q in queries if q["result_count"] == 0]

    # Find most common query terms
    all_terms = []
    for q in queries:
        all_terms.extend(q["query"].lower().split())
    term_freq = Counter(all_terms).most_common(20)

    # Find low-result queries (weak coverage)
    weak = [q["query"] for q in queries if 0 < q["result_count"] <= 2]

    conn.close()
    return {
        "total_queries": len(queries),
        "gaps": list(set(gaps)),
        "weak_coverage": list(set(weak)),
        "top_terms": term_freq,
    }


def analyze_feedback_trends():
    """Understand what content users find valuable."""
    conn = db.get_conn()

    # Documents with most negative feedback
    bad_docs = conn.execute("""
        SELECT d.id, d.title, d.relevance_score, d.feedback_count, d.positive_feedback
        FROM documents d
        WHERE d.feedback_count > 0
          AND d.relevance_score < 0.3
        ORDER BY d.relevance_score ASC
        LIMIT 20
    """).fetchall()

    # Documents with most positive feedback
    good_docs = conn.execute("""
        SELECT d.id, d.title, d.relevance_score, d.feedback_count, d.positive_feedback,
               d.keywords
        FROM documents d
        WHERE d.feedback_count > 0
          AND d.relevance_score > 0.7
        ORDER BY d.relevance_score DESC
        LIMIT 20
    """).fetchall()

    # Extract keywords from highly-rated docs to understand what's valued
    valued_keywords = []
    for doc in good_docs:
        kw = json.loads(doc["keywords"]) if doc["keywords"] else []
        valued_keywords.extend(kw)
    valued_keyword_freq = Counter(valued_keywords).most_common(15)

    conn.close()
    return {
        "low_quality_docs": [(d["id"], d["title"], d["relevance_score"]) for d in bad_docs],
        "high_quality_docs": [(d["id"], d["title"], d["relevance_score"]) for d in good_docs],
        "valued_keywords": valued_keyword_freq,
    }


def prune_low_quality():
    """Remove documents that have been consistently rated as irrelevant."""
    conn = db.get_conn()

    # Delete documents with very low relevance AND enough feedback to be confident
    pruned = conn.execute("""
        DELETE FROM documents
        WHERE relevance_score < 0.15
          AND feedback_count >= 3
        RETURNING id, title
    """).fetchall()

    conn.commit()
    conn.close()
    return [(p["id"], p["title"]) for p in pruned]


def apply_decay():
    """Drift old unfeedback'd documents toward neutral over time."""
    conn = db.get_conn()
    decay_rate = 0.005

    # Only decay documents older than 7 days with no recent feedback
    conn.execute("""
        UPDATE documents
        SET relevance_score = relevance_score + (0.5 - relevance_score) * ?
        WHERE feedback_count > 0
          AND updated_at < datetime('now', '-7 days')
    """, (decay_rate,))

    affected = conn.execute("SELECT changes()").fetchone()[0]
    conn.commit()
    conn.close()
    return affected


def analyze_source_quality():
    """Rate each source by the quality of documents it produces."""
    conn = db.get_conn()

    sources = conn.execute("""
        SELECT s.id, s.name, s.type,
               COUNT(d.id) as doc_count,
               COALESCE(AVG(d.relevance_score), 0.5) as avg_relevance,
               COALESCE(SUM(d.feedback_count), 0) as total_feedback
        FROM sources s
        LEFT JOIN documents d ON d.source_id = s.id
        GROUP BY s.id
        ORDER BY avg_relevance DESC
    """).fetchall()

    conn.close()
    return [dict(s) for s in sources]


def deduplicate():
    """Find and remove near-duplicate content."""
    conn = db.get_conn()

    # Find exact hash duplicates (shouldn't exist due to UNIQUE constraint, but safety check)
    dupes = conn.execute("""
        SELECT hash, COUNT(*) as cnt
        FROM documents
        WHERE hash IS NOT NULL
        GROUP BY hash
        HAVING cnt > 1
    """).fetchall()

    removed = 0
    for d in dupes:
        # Keep the one with highest relevance, delete the rest
        to_delete = conn.execute("""
            SELECT id FROM documents
            WHERE hash = ?
            ORDER BY relevance_score DESC
            LIMIT -1 OFFSET 1
        """, (d["hash"],)).fetchall()

        for row in to_delete:
            conn.execute("DELETE FROM documents WHERE id = ?", (row["id"],))
            removed += 1

    conn.commit()
    conn.close()
    return removed


def generate_report(scrape_results, query_analysis, feedback_analysis, pruned, decayed, source_quality, deduped):
    """Generate a refinement report."""
    now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    report_path = os.path.join(REPORT_DIR, f"refinement_{now}.json")

    report = {
        "timestamp": now,
        "actions": {
            "documents_scraped": scrape_results,
            "documents_pruned": len(pruned),
            "documents_decayed": decayed,
            "duplicates_removed": deduped,
        },
        "analysis": {
            "query_patterns": query_analysis,
            "feedback_trends": feedback_analysis,
            "source_quality": source_quality,
        },
        "recommendations": []
    }

    # Generate recommendations
    if query_analysis["gaps"]:
        report["recommendations"].append(
            f"Knowledge gaps detected for: {', '.join(query_analysis['gaps'][:5])}. "
            "Consider adding sources covering these topics."
        )

    for src in source_quality:
        if src["avg_relevance"] < 0.3 and src["total_feedback"] > 5:
            report["recommendations"].append(
                f"Source '{src['name']}' has low avg relevance ({src['avg_relevance']:.2f}). "
                "Consider disabling or replacing."
            )

    if feedback_analysis["valued_keywords"]:
        top_kw = [kw for kw, _ in feedback_analysis["valued_keywords"][:5]]
        report["recommendations"].append(
            f"Users value content about: {', '.join(top_kw)}. "
            "Prioritize sources covering these topics."
        )

    with open(report_path, "w") as f:
        json.dump(report, f, indent=2, default=str)

    return report_path, report


def run_refinement():
    """Main refinement pipeline."""
    db.init_db()
    print(f"{'='*60}")
    print(f"  Knowledge Engine Refinement")
    print(f"  {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}\n")

    # 1. Scrape new content
    print("[1/6] Scraping sources...")
    scraped = scraper.scrape_all()
    print(f"      Added {scraped} new documents\n")

    # 2. Analyze queries
    print("[2/6] Analyzing query patterns...")
    query_analysis = analyze_query_patterns()
    print(f"      {query_analysis['total_queries']} queries analyzed")
    if query_analysis["gaps"]:
        print(f"      Gaps: {', '.join(query_analysis['gaps'][:3])}")
    print()

    # 3. Analyze feedback
    print("[3/6] Analyzing feedback trends...")
    feedback_analysis = analyze_feedback_trends()
    print(f"      {len(feedback_analysis['high_quality_docs'])} high-quality docs")
    print(f"      {len(feedback_analysis['low_quality_docs'])} low-quality docs")
    print()

    # 4. Prune low quality
    print("[4/6] Pruning low-quality documents...")
    pruned = prune_low_quality()
    print(f"      Pruned {len(pruned)} documents\n")

    # 5. Apply decay
    print("[5/6] Applying relevance decay...")
    decayed = apply_decay()
    print(f"      Decayed {decayed} documents\n")

    # 6. Deduplicate
    print("[6/6] Deduplicating...")
    deduped = deduplicate()
    print(f"      Removed {deduped} duplicates\n")

    # Source quality
    source_quality = analyze_source_quality()

    # Generate report
    report_path, report = generate_report(
        scraped, query_analysis, feedback_analysis, pruned, decayed, source_quality, deduped
    )

    print(f"{'='*60}")
    print(f"  Refinement Complete")
    print(f"  Report: {report_path}")
    if report["recommendations"]:
        print(f"\n  Recommendations:")
        for r in report["recommendations"]:
            print(f"    → {r}")
    print(f"{'='*60}")

    # Print stats
    stats = db.get_stats()
    print(f"\n  DB: {stats['documents']} docs | {stats['sources']} sources | "
          f"avg relevance: {stats['avg_relevance']:.3f}")


if __name__ == "__main__":
    run_refinement()