Lore_Evidence_Graph_System.py

import os
import json
import re
from datetime import datetime

base_dir = r"C:\hades\Hecton8\Docs\Lore\AppliedContent"
nodes_dir = os.path.join(base_dir, "EvidenceNodes")
out_md = os.path.join(base_dir, "DEEP_LORE_EVIDENCE_GRAPH.md")
out_json = os.path.join(base_dir, "runtime_evidence_graph.json")
out_html = os.path.join(base_dir, "DEEP_LORE_EVIDENCE_GRAPH_INTERACTIVE.html")

print("Initializing Advanced Deep Lore Evidence Graph Compiler v3.0.0...")

REQUIRED_FIELDS = ["node_id", "author", "subject_tags", "title", "surface", "text"]

evidence_graph = {
    "version": "3.0.0-DeepLore-Advanced",
    "compiled_at": datetime.now().isoformat(),
    "nodes": [],
    "subjects": {},
    "implicit_links": [],
    "statistics": {
        "total_nodes": 0,
        "total_tags": 0,
        "validation_errors": 0
    }
}

# 1. Read and Validate all nodes
all_node_ids = set()
for filename in os.listdir(nodes_dir):
    if filename.endswith(".json"):
        filepath = os.path.join(nodes_dir, filename)
        try:
            with open(filepath, "r", encoding="utf-8") as f:
                node = json.load(f)
                
                # Validation
                missing_fields = [f for f in REQUIRED_FIELDS if f not in node]
                if missing_fields:
                    print(f"[ERROR] Node {filename} is missing required fields: {missing_fields}")
                    evidence_graph["statistics"]["validation_errors"] += 1
                    continue
                
                evidence_graph["nodes"].append(node)
                all_node_ids.add(node["node_id"])
        except json.JSONDecodeError as e:
            print(f"[ERROR] Failed to parse JSON in {filename}: {e}")
            evidence_graph["statistics"]["validation_errors"] += 1

# Sort nodes for deterministic output
evidence_graph["nodes"].sort(key=lambda x: x["node_id"])

# 2. Build cross-reference graph by tags
for node in evidence_graph["nodes"]:
    for tag in node.get("subject_tags", []):
        if tag not in evidence_graph["subjects"]:
            evidence_graph["subjects"][tag] = []
        evidence_graph["subjects"][tag].append(node["node_id"])

# 3. Detect Implicit Links (mentions of other Node IDs in the text)
for node in evidence_graph["nodes"]:
    text = node["text"]
    for other_id in all_node_ids:
        if other_id != node["node_id"] and other_id in text:
            evidence_graph["implicit_links"].append({
                "source": node["node_id"],
                "target": other_id,
                "type": "explicit_mention"
            })

evidence_graph["statistics"]["total_nodes"] = len(evidence_graph["nodes"])
evidence_graph["statistics"]["total_tags"] = len(evidence_graph["subjects"])

# 4. Export Runtime JSON
with open(out_json, "w", encoding="utf-8") as f:
    json.dump(evidence_graph, f, indent=4)

# 5. Generate Advanced Markdown Graph Wiki
with open(out_md, "w", encoding="utf-8") as f:
    f.write("# HECTON-8 Deep Lore Evidence Graph\n\n")
    f.write("> [!IMPORTANT]\n")
    f.write("> This document is generated by the Advanced Lore Evidence Graph System v3.0.0. It maps the interconnections of canonical deep-lore evidence recovered by Marauders.\n\n")
    
    # Statistics
    f.write("## Compilation Statistics\n")
    f.write(f"- **Total Nodes Compiled:** {evidence_graph['statistics']['total_nodes']}\n")
    f.write(f"- **Unique Tags:** {evidence_graph['statistics']['total_tags']}\n")
    f.write(f"- **Implicit Links Detected:** {len(evidence_graph['implicit_links'])}\n")
    f.write(f"- **Validation Errors:** {evidence_graph['statistics']['validation_errors']}\n\n")

    # Mermaid Graph
    f.write("## Entity Relationship Graph\n\n")
    f.write("```mermaid\n")
    f.write("graph TD\n")
    for node in evidence_graph["nodes"]:
        f.write(f"    {node['node_id']}['{node['title']}']\n")
    for tag, node_ids in evidence_graph["subjects"].items():
        tag_id = "TAG_" + re.sub(r'[^a-zA-Z0-9]', '_', tag)
        f.write(f"    {tag_id}((\"{tag}\"))\n")
        for nid in node_ids:
            f.write(f"    {nid} --> {tag_id}\n")
    for link in evidence_graph["implicit_links"]:
        f.write(f"    {link['source']} -. mentions .-> {link['target']}\n")
    f.write("```\n\n")
    
    f.write("## 1. Subject Linkages\n\n")
    for subject, node_ids in sorted(evidence_graph["subjects"].items()):
        f.write(f"### Tag: {subject}\n")
        for nid in node_ids:
            title = next((n["title"] for n in evidence_graph["nodes"] if n["node_id"] == nid), "Unknown")
            author = next((n["author"] for n in evidence_graph["nodes"] if n["node_id"] == nid), "Unknown")
            f.write(f"- **[{nid}](#{nid.lower()}-{title.lower().replace(' ', '-')})**: *{title}* (Author: {author})\n")
        f.write("\n")
        
    f.write("---\n\n## 2. Core Evidence Nodes (Raw Transcripts)\n\n")
    for node in evidence_graph["nodes"]:
        anchor = f"{node['node_id'].lower()}-{node['title'].lower().replace(' ', '-')}"
        f.write(f"### {node['node_id']} - {node['title']}\n")
        f.write(f"- **Author**: {node['author']}\n")
        f.write(f"- **Surface**: {node['surface']}\n")
        f.write(f"- **Tags**: {', '.join(node['subject_tags'])}\n\n")
        f.write(f"> {node['text']}\n\n")
        f.write("---\n\n")

# 6. Generate Interactive HTML Version
html_content = f"""<!DOCTYPE html>
<html>
<head>
    <title>HECTON-8 Lore Evidence Graph</title>
    <style>
        body {{ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; background-color: #121212; color: #e0e0e0; margin: 40px; }}
        h1, h2, h3 {{ color: #ffffff; border-bottom: 1px solid #333; padding-bottom: 5px; }}
        .node-card {{ background-color: #1e1e1e; border: 1px solid #333; padding: 20px; margin-bottom: 20px; border-radius: 8px; }}
        .tag-pill {{ display: inline-block; background-color: #007acc; color: white; padding: 3px 8px; border-radius: 12px; font-size: 0.8em; margin-right: 5px; }}
        .stat-box {{ display: flex; gap: 20px; margin-bottom: 30px; }}
        .stat {{ background: #252526; padding: 15px; border-radius: 8px; text-align: center; flex: 1; }}
        .stat span {{ display: block; font-size: 2em; color: #4fc1ff; }}
    </style>
</head>
<body>
    <h1>HECTON-8 Deep Lore Evidence System</h1>
    <div class="stat-box">
        <div class="stat">Nodes Compiled<span>{evidence_graph['statistics']['total_nodes']}</span></div>
        <div class="stat">Unique Tags<span>{evidence_graph['statistics']['total_tags']}</span></div>
        <div class="stat">Implicit Links<span>{len(evidence_graph['implicit_links'])}</span></div>
    </div>
"""
for node in evidence_graph['nodes']:
    tags_html = "".join([f"<span class='tag-pill'>{t}</span>" for t in node['subject_tags']])
    html_content += f"""
    <div class="node-card" id="{node['node_id']}">
        <h3>{node['node_id']}: {node['title']}</h3>
        <p><strong>Author:</strong> {node['author']} | <strong>Surface:</strong> {node['surface']}</p>
        <p>{tags_html}</p>
        <blockquote style="border-left: 4px solid #4fc1ff; padding-left: 15px; font-style: italic; color: #b0b0b0;">
            {node['text']}
        </blockquote>
    </div>
    """
html_content += "</body></html>"

with open(out_html, "w", encoding="utf-8") as f:
    f.write(html_content)

print(f"Compilation successful. Processed {evidence_graph['statistics']['total_nodes']} nodes.")
print(f"-> Markdown Wiki: {out_md}")
print(f"-> Runtime JSON: {out_json}")
print(f"-> Interactive HTML: {out_html}")