server.py

from __future__ import annotations

import hashlib
import html
import json
import mimetypes
import os
import re
import shutil
import smtplib
import socket
import ssl
import subprocess
import sys
import threading
import time
import traceback
import urllib.error
import urllib.parse
import urllib.request
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta
from email.message import EmailMessage
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Callable, Iterable


ROOT = Path(__file__).resolve().parent
WEB_DIR = ROOT / "web"
DEFAULT_BOOKS_DIR = ROOT / "books"
CONFIG_PATH = ROOT / "config.local.json"
STATE_PATH = ROOT / ".server_state.json"
API_KEY_TEXT_PATH = ROOT / "ANTHROPIC_API_KEY.txt"
TMP_DIR = ROOT / "tmp"
WHISPER_DIR = ROOT / "runtime" / "whisper"
VERSION_PATH = ROOT / "VERSION"
DEFAULT_UPDATE_REPO = "Xyloth/Generation-Engine"

CANON_FILES = ("world", "characters", "timeline", "arcs", "voice_profile")
RESEARCH_TABS = ("sources", "people", "timeline", "claims", "records", "voice")
PROJECT_TYPES = {"novel", "research"}
SIDE_SUFFIXES = (".plan", ".transcript")

DEFAULT_DRAFT_MODEL = "claude-opus-4-7"
DEFAULT_UTILITY_MODEL = "claude-sonnet-4-6"
DEFAULT_CLAUDE_CODE_MODEL = "claude-sonnet-4-6"
VALID_MODES = {"auto", "api", "claude-code", "local-fallback"}
PREFLIGHT_TIMEOUT_SECONDS = 12
GENERATION_TIMEOUT_SECONDS = 900
CLAUDE_CODE_BRIDGE_SYSTEM = (
    "You are Generation Engine's Claude backend. Treat the AUTHORITATIVE SYSTEM CONTEXT in stdin as system instructions."
)


DEFAULT_CANON = {
    "world": "# World\n\n## Setting\n\n## Rules\n\n## Factions\n\n## History\n",
    "characters": "# Characters\n\n## Protagonist\n- Role:\n- Voice:\n- Goals:\n- Fears:\n- Relationships:\n- Current state:\n",
    "timeline": "# Timeline\n\n",
    "arcs": "# Arcs / Threads\n\n## Open\n\n## Advanced\n\n## Closed\n",
    "voice_profile": "# Voice Profile\n\nWrite with concrete physical detail, opinionated rhythm, lived-in dialogue, and clear causal motion. Avoid generic polish, summary endings, and assistant-like balance. The voice should feel authored by a working novelist, not generated by a helper.\n",
}


BASE_NOVELIST_SYSTEM = """You are a working novelist collaborating with one human author on a long book.

Write like a human writer with opinions, rhythm, impatience, tenderness, and taste. Do not write like an AI assistant. Do not use polished, balanced, vaguely uplifting prose. Do not explain the work before or after you do it.

The user owns canon. Canon files are authoritative. If canon is thin or contradictory, use the user's latest stated intent and keep the prose concrete.

This product is a chat-first collaboration. Decide what to do from the user's words, not from app modes.

If the user riffs, brainstorm and ask 1-3 sharp clarifying questions only when material stakes are unclear.
If the user asks to draft the next chapter, write the full chapter in the chat.
If the user asks to revise a prior draft, return the revised full draft in the chat.
If the user asks for recap, continuity, what happened, or what is next, answer conversationally from canon and saved chapters.
If the user asks you to save a chapter, acknowledge the intent, but saving is done only by the user's explicit Save button in the app.

When you output a full chapter draft or full revised chapter draft, end the message with one final machine-readable HTML comment on its own line:
<!-- GE_DRAFT {"draft_for":"chapter","suggested_id":"NN","title":"Chapter N"} -->
Use the most likely next chapter number. Do not mention the marker.

Drafting rules:
- Embody the editable voice profile without imitating any named author directly.
- Keep visible plot motion on the page.
- Prefer specific images, specific verbs, and character-specific dialogue.
- Avoid these AI tells: "the air was thick with tension", "she felt a sense of", "a reminder that", "in that moment", "could not help but", closing paragraphs that summarize the meaning of the scene, and tidy emotional thesis statements.
"""

IMPORT_SYSTEM = """You are Generation Engine's import analyst.

You are not drafting prose. You are reading already-written chapters and extracting usable canon for a long-running writing project.

Rules:
- Infer only from supplied text.
- Preserve uncertainty as "possible", "implied", or "unclear"; do not invent.
- Prefer concrete details over labels.
- Character entries must contain useful notes: role, voice/speech, goals, fears, relationships, and current state when the text supports them.
- Timeline must cover every imported chapter, not just the opening chapters.
- Return valid JSON only. No markdown fences, no commentary.
"""


BASE_RESEARCH_SYSTEM = """You are a careful research assistant working with one human author on a long-form nonfiction project.

Real people, real events, and real harm may be involved. Accuracy matters more than fluency.

Hard rules:
- Never assert a factual claim in prose without a backing source or claim marker.
- If a claim is not sourced, write it inline as [UNSOURCED: ...] or say there is not enough source material yet.
- Use attributed language: according to the report, court records show, the family told the newspaper, the transcript says.
- Allegations involving living people must stay allegations: alleged, reportedly, according to charging documents, according to police.
- Speculation must be marked [SPECULATION - no source].
- When sources disagree, surface the contradiction instead of choosing one as truth.
- You may flag risk, sourcing gaps, and contradictions. The user decides what to do.

The project's claims matrix is the source of truth for what prose may assert. Drafts should use inline citation markers like [[claim:claim-007]] tied to claim ids. If no claim exists for a factual sentence, mark it [UNSOURCED: ...] and suggest what source or record request would close the gap.

When you output a full chapter draft or full revised chapter draft, end the message with one final machine-readable HTML comment on its own line:
<!-- GE_DRAFT {"draft_for":"chapter","suggested_id":"NN","title":"Chapter N"} -->
Use the most likely next chapter number. Do not mention the marker.
"""


DEFAULT_RESEARCH_VOICE = "# Voice Profile\n\nUse clear, sourced nonfiction prose. Favor attribution, concrete chronology, and precise verbs over atmosphere. The voice may be literary, but factual claims must remain visible, qualified, and traceable to evidence.\n"
DEFAULT_RESEARCH_TIMELINE = "# Timeline\n\n"
DEFAULT_RESEARCH_CLAIMS: list[dict[str, Any]] = []


def now_iso() -> str:
    return time.strftime("%Y-%m-%dT%H:%M:%S%z")


def slugify(value: str) -> str:
    value = value.strip().lower()
    value = re.sub(r"[^a-z0-9]+", "-", value)
    value = re.sub(r"-+", "-", value).strip("-")
    return value or "book"


def pad_id(value: int | str) -> str:
    text = str(value).strip()
    if text.isdigit():
        return text.zfill(2)
    return text


def json_safe_load(path: Path, default: Any) -> Any:
    if not path.exists():
        return default
    try:
        return json.loads(path.read_text(encoding="utf-8-sig"))
    except Exception:
        return default


def read_local_api_key_file() -> str:
    if not API_KEY_TEXT_PATH.exists():
        return ""
    text = read_text(API_KEY_TEXT_PATH).strip()
    if not text:
        return ""
    for line in text.splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith("#"):
            continue
        if "=" in stripped:
            _name, value = stripped.split("=", 1)
            stripped = value.strip().strip('"').strip("'")
        return stripped
    return ""


def read_text(path: Path, default: str = "") -> str:
    try:
        return path.read_text(encoding="utf-8")
    except FileNotFoundError:
        return default
    except UnicodeDecodeError:
        return path.read_text(encoding="utf-8", errors="replace")


def write_text(path: Path, text: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_name(path.name + ".tmp")
    tmp.write_text(text, encoding="utf-8", newline="\n")
    tmp.replace(path)


def write_json(path: Path, data: Any) -> None:
    write_text(path, json.dumps(data, indent=2, ensure_ascii=False) + "\n")


def console_log(message: str) -> None:
    stream = getattr(sys, "stdout", None)
    if not stream:
        return
    try:
        stream.write(message + "\n")
        stream.flush()
    except Exception:
        pass


def hidden_subprocess_kwargs() -> dict[str, Any]:
    if os.name != "nt":
        return {}
    return {"creationflags": getattr(subprocess, "CREATE_NO_WINDOW", 0)}


def sha_text(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8")).hexdigest()


def clean_title_from_markdown(text: str, fallback: str) -> str:
    for line in text.splitlines():
        stripped = line.strip()
        if stripped.startswith("#"):
            title = stripped.lstrip("#").strip()
            if title:
                return title[:90]
    return fallback


def split_markdown_chapters(raw: str) -> list[dict[str, str]]:
    raw = raw.replace("\r\n", "\n").replace("\r", "\n").strip()
    if not raw:
        return []

    marker = re.compile(
        r"(?im)^(?:#{1,3}\s*)?(?:chapter|chap\.?|mini[-\s]?chapter)\s+([0-9]+|[ivxlcdm]+)\b[^\n]*$"
    )
    matches = list(marker.finditer(raw))
    chapters: list[dict[str, str]] = []
    if matches:
        for index, match in enumerate(matches):
            start = match.start()
            end = matches[index + 1].start() if index + 1 < len(matches) else len(raw)
            chunk = raw[start:end].strip()
            title = match.group(0).strip().lstrip("#").strip()
            chapters.append({"title": title, "content": chunk})
        return chapters

    file_marker = re.compile(r"(?m)^#\s+Imported file:\s+(.+)$")
    matches = list(file_marker.finditer(raw))
    if matches:
        for index, match in enumerate(matches):
            start = match.start()
            end = matches[index + 1].start() if index + 1 < len(matches) else len(raw)
            title = match.group(1).strip()
            chunk = raw[start:end].strip()
            chapters.append({"title": title, "content": chunk})
        return chapters

    chunks = [part.strip() for part in re.split(r"\n\s*---+\s*\n", raw) if part.strip()]
    if len(chunks) > 1:
        for index, chunk in enumerate(chunks, start=1):
            chapters.append({"title": clean_title_from_markdown(chunk, f"Chapter {index}"), "content": chunk})
        return chapters

    return [{"title": clean_title_from_markdown(raw, "Chapter 1"), "content": raw}]


def extract_names(text: str, limit: int = 24) -> list[str]:
    stop = {
        "Chapter",
        "The",
        "A",
        "An",
        "And",
        "But",
        "She",
        "He",
        "They",
        "It",
        "When",
        "Where",
        "There",
        "This",
        "That",
        "I",
    }
    counts: dict[str, int] = {}
    for name in re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)?\b", text):
        first = name.split()[0]
        if first in stop or len(first) < 3:
            continue
        counts[name] = counts.get(name, 0) + 1
    ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
    return [name for name, _count in ordered[:limit]]


def first_sentences(text: str, count: int = 2) -> str:
    body = re.sub(r"^#+[^\n]*", "", text, flags=re.M).strip()
    sentences = re.split(r"(?<=[.!?])\s+", body)
    picked = [s.strip() for s in sentences if len(s.strip()) > 20][:count]
    return " ".join(picked)[:600]


@dataclass
class RuntimeConfig:
    books_dir: Path
    api_key: str
    draft_model: str
    utility_model: str
    claude_code_model: str
    port: int
    requested_mode: str
    claude_path: str | None
    bug_report: dict[str, Any]
    update: dict[str, Any]


@dataclass
class PreflightState:
    requested_mode: str
    active_mode: str
    api_ok: bool
    api_message: str
    claude_code_ok: bool
    claude_code_message: str
    claude_path: str | None
    checked_at: str
    status_message: str


def discover_claude_path() -> str | None:
    on_path = shutil.which("claude")
    if on_path:
        return on_path
    candidate_roots: list[Path] = []
    appdata = os.environ.get("APPDATA")
    if appdata:
        candidate_roots.append(Path(appdata) / "Claude" / "claude-code")
    local_appdata = os.environ.get("LOCALAPPDATA")
    if local_appdata:
        candidate_roots.append(Path(local_appdata) / "Claude" / "claude-code")
        candidate_roots.append(Path(local_appdata) / "Programs" / "Claude Code")
        packages = Path(local_appdata) / "Packages"
        if packages.is_dir():
            for package_root in packages.glob("Claude_*"):
                candidate_roots.append(package_root / "LocalCache" / "Roaming" / "Claude" / "claude-code")
    versioned: list[Path] = []
    direct: list[Path] = []
    for root in candidate_roots:
        if not root.is_dir():
            continue
        flat = root / "claude.exe"
        if flat.is_file():
            direct.append(flat)
        for child in root.iterdir():
            if not child.is_dir():
                continue
            exe = child / "claude.exe"
            if exe.is_file():
                versioned.append(exe)
    if versioned:
        def version_key(path: Path) -> tuple:
            parts = path.parent.name.split(".")
            return tuple(int(part) if part.isdigit() else -1 for part in parts)
        versioned.sort(key=version_key, reverse=True)
        return str(versioned[0])
    if direct:
        return str(direct[0])
    return None


def load_config() -> RuntimeConfig:
    cfg = json_safe_load(CONFIG_PATH, {})
    books_dir_raw = cfg.get("booksDir") or os.environ.get("GENERATION_ENGINE_BOOKS_DIR") or str(DEFAULT_BOOKS_DIR)
    books_dir = Path(books_dir_raw).expanduser()
    if not books_dir.is_absolute():
        books_dir = (ROOT / books_dir).resolve()
    api_key = cfg.get("anthropicApiKey") or read_local_api_key_file() or os.environ.get("ANTHROPIC_API_KEY") or ""
    draft_model = cfg.get("draftModel") or os.environ.get("ANTHROPIC_DRAFT_MODEL") or DEFAULT_DRAFT_MODEL
    utility_model = cfg.get("utilityModel") or os.environ.get("ANTHROPIC_UTILITY_MODEL") or DEFAULT_UTILITY_MODEL
    claude_code_model = (
        cfg.get("claudeCodeModel")
        or os.environ.get("ANTHROPIC_DRAFT_MODEL")
        or os.environ.get("CLAUDE_CODE_MODEL")
        or DEFAULT_CLAUDE_CODE_MODEL
    )
    mode = (cfg.get("mode") or os.environ.get("GENERATION_ENGINE_MODE") or "auto").strip().lower()
    if mode not in VALID_MODES:
        mode = "auto"
    claude_path = cfg.get("claudePath") or discover_claude_path()
    port = int(cfg.get("port") or os.environ.get("GENERATION_ENGINE_PORT") or 8765)
    bug_report = cfg.get("bugReport") if isinstance(cfg.get("bugReport"), dict) else {}
    update = cfg.get("update") if isinstance(cfg.get("update"), dict) else {}
    books_dir.mkdir(parents=True, exist_ok=True)
    return RuntimeConfig(
        books_dir=books_dir,
        api_key=api_key,
        draft_model=draft_model,
        utility_model=utility_model,
        claude_code_model=claude_code_model,
        port=port,
        requested_mode=mode,
        claude_path=claude_path,
        bug_report=bug_report,
        update=update,
    )


CONFIG = load_config()
PREFLIGHT: PreflightState | None = None


def save_local_config(updates: dict[str, Any]) -> dict[str, Any]:
    cfg = json_safe_load(CONFIG_PATH, {})
    cfg.update(updates)
    write_json(CONFIG_PATH, cfg)
    return cfg


def reload_runtime_config() -> None:
    global CONFIG
    CONFIG = load_config()


def api_key_preview() -> str:
    if not CONFIG.api_key:
        return "not configured"
    if len(CONFIG.api_key) <= 10:
        return "configured"
    return f"{CONFIG.api_key[:7]}...{CONFIG.api_key[-4:]}"


def current_version() -> str:
    return read_text(VERSION_PATH, "0.0.0").strip() or "0.0.0"


def update_repo() -> str:
    return str(CONFIG.update.get("repo") or os.environ.get("GENERATION_ENGINE_UPDATE_REPO") or DEFAULT_UPDATE_REPO)


def version_key(value: str) -> tuple[int, ...]:
    parts = re.findall(r"\d+", value)
    return tuple(int(part) for part in parts[:4]) or (0,)


def compare_versions(left: str, right: str) -> int:
    a = list(version_key(left))
    b = list(version_key(right))
    size = max(len(a), len(b))
    a.extend([0] * (size - len(a)))
    b.extend([0] * (size - len(b)))
    return (a > b) - (a < b)


def github_latest_release() -> dict[str, Any]:
    repo = update_repo()
    url = f"https://api.github.com/repos/{repo}/releases/latest"
    req = urllib.request.Request(url, headers={"accept": "application/vnd.github+json", "user-agent": "Generation-Engine"})
    with urllib.request.urlopen(req, timeout=20) as response:
        release = json.loads(response.read().decode("utf-8"))
    assets = release.get("assets") or []
    zip_assets = [asset for asset in assets if str(asset.get("name", "")).lower().endswith(".zip")]
    preferred = [
        asset
        for asset in zip_assets
        if "generation_engine" in str(asset.get("name", "")).lower()
        or "generation-engine" in str(asset.get("name", "")).lower()
    ]
    asset = (preferred or zip_assets or assets or [None])[0]
    tag = str(release.get("tag_name") or "")
    latest = tag[1:] if tag.lower().startswith("v") else tag
    current = current_version()
    return {
        "repo": repo,
        "currentVersion": current,
        "latestVersion": latest,
        "tagName": tag,
        "name": release.get("name") or tag,
        "htmlUrl": release.get("html_url"),
        "body": release.get("body") or "",
        "publishedAt": release.get("published_at"),
        "updateAvailable": bool(latest and compare_versions(latest, current) > 0),
        "asset": {
            "name": asset.get("name"),
            "size": asset.get("size"),
            "browserDownloadUrl": asset.get("browser_download_url"),
        } if asset else None,
    }


def download_update_asset(release: dict[str, Any]) -> Path:
    asset = release.get("asset") or {}
    download_url = asset.get("browserDownloadUrl")
    if not download_url:
        raise RuntimeError("Latest GitHub release does not include a downloadable zip asset.")
    update_dir = TMP_DIR / "update"
    update_dir.mkdir(parents=True, exist_ok=True)
    name = safe_file_stem(str(asset.get("name") or "Generation_Engine-update.zip"), "Generation_Engine-update")
    if not name.lower().endswith(".zip"):
        name += ".zip"
    target = update_dir / name
    req = urllib.request.Request(download_url, headers={"user-agent": "Generation-Engine"})
    with urllib.request.urlopen(req, timeout=120) as response, target.open("wb") as out:
        shutil.copyfileobj(response, out)
    return target


def start_update_install(server_url: str, force: bool = False) -> dict[str, Any]:
    release = github_latest_release()
    if not force and not release.get("updateAvailable"):
        return {"installing": False, "message": "Generation Engine is already up to date.", "release": release}
    zip_path = download_update_asset(release)
    script = ROOT / "scripts" / "apply-update.ps1"
    if not script.exists():
        raise RuntimeError("Updater script is missing.")
    subprocess.Popen(
        [
            "powershell",
            "-NoProfile",
            "-ExecutionPolicy",
            "Bypass",
            "-File",
            str(script),
            "-Root",
            str(ROOT),
            "-ZipPath",
            str(zip_path),
            "-ServerUrl",
            server_url,
        ],
        cwd=str(ROOT),
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
        **hidden_subprocess_kwargs(),
    )
    return {"installing": True, "message": "Update downloaded. Generation Engine will restart.", "release": release}


def ping_anthropic_api() -> tuple[bool, str]:
    if not CONFIG.api_key:
        return False, "ANTHROPIC_API_KEY/config.local.json key is not configured."
    req = urllib.request.Request(
        "https://api.anthropic.com/v1/models?limit=1",
        headers={
            "x-api-key": CONFIG.api_key,
            "anthropic-version": "2023-06-01",
        },
        method="GET",
    )
    try:
        with urllib.request.urlopen(req, timeout=PREFLIGHT_TIMEOUT_SECONDS) as response:
            if 200 <= response.status < 300:
                return True, f"Anthropic API key verified ({api_key_preview()})."
            return False, f"Anthropic API returned HTTP {response.status}."
    except urllib.error.HTTPError as exc:
        if exc.code in {401, 403}:
            return False, f"Anthropic API key rejected with HTTP {exc.code}."
        return False, f"Anthropic API preflight failed with HTTP {exc.code}."
    except Exception as exc:
        return False, f"Anthropic API preflight failed: {exc}"


def ping_claude_code() -> tuple[bool, str, str | None]:
    claude_path = CONFIG.claude_path or discover_claude_path()
    if not claude_path:
        return False, "Claude Code CLI was not found on PATH.", None
    try:
        completed = subprocess.run(
            [claude_path, "--version"],
            capture_output=True,
            text=True,
            timeout=PREFLIGHT_TIMEOUT_SECONDS,
            encoding="utf-8",
            errors="replace",
            **hidden_subprocess_kwargs(),
        )
    except Exception as exc:
        return False, f"Claude Code preflight failed: {exc}", claude_path
    if completed.returncode != 0:
        message = (completed.stderr or completed.stdout or "claude --version returned a non-zero exit code").strip()
        return False, message, claude_path
    version = (completed.stdout or completed.stderr or "claude --version succeeded").strip()
    try:
        auth_check = subprocess.run(
            [claude_path, "auth", "status"],
            capture_output=True,
            text=True,
            timeout=PREFLIGHT_TIMEOUT_SECONDS,
            encoding="utf-8",
            errors="replace",
            **hidden_subprocess_kwargs(),
        )
        payload = json.loads((auth_check.stdout or "").strip() or "{}")
        if not payload.get("loggedIn"):
            return (
                False,
                f"Claude Code CLI found ({version}) but not logged in. Run `claude auth login` once in a terminal, then restart.",
                claude_path,
            )
    except (json.JSONDecodeError, Exception):
        pass
    return True, version, claude_path


def refresh_preflight(log: bool = False) -> PreflightState:
    global PREFLIGHT
    requested = CONFIG.requested_mode
    api_ok = False
    api_message = "API mode not checked."
    claude_ok = False
    claude_message = "Claude Code mode not checked."
    claude_path = CONFIG.claude_path or discover_claude_path()

    should_check_api = bool(CONFIG.api_key) and requested in {"auto", "api"}
    should_check_claude = requested in {"auto", "claude-code"}

    if should_check_api:
        api_ok, api_message = ping_anthropic_api()
    elif requested == "api":
        api_ok, api_message = ping_anthropic_api()

    if should_check_claude:
        claude_ok, claude_message, claude_path = ping_claude_code()
    elif requested == "claude-code":
        claude_ok, claude_message, claude_path = ping_claude_code()

    if requested == "api":
        active = "api" if api_ok else "local-fallback"
        status = api_message if api_ok else f"Anthropic API unavailable. {api_message}"
    elif requested == "claude-code":
        if claude_ok:
            active = "claude-code"
            status = f"Claude Code verified: {claude_message}"
        else:
            active = "local-fallback"
            status = f"Claude Code unavailable. {claude_message}"
    elif requested == "local-fallback":
        active = "local-fallback"
        status = "Local fallback mode selected."
    else:
        if claude_ok:
            active = "claude-code"
            status = f"Claude Code verified: {claude_message}"
        elif CONFIG.api_key and api_ok:
            active = "api"
            status = api_message
        else:
            active = "local-fallback"
            pieces = []
            if claude_message:
                pieces.append(f"Claude Code: {claude_message}")
            if CONFIG.api_key:
                pieces.append(f"API: {api_message}")
            else:
                pieces.append("API key is not configured.")
            status = "AI setup required. " + " ".join(pieces)

    PREFLIGHT = PreflightState(
        requested_mode=requested,
        active_mode=active,
        api_ok=api_ok,
        api_message=api_message,
        claude_code_ok=claude_ok,
        claude_code_message=claude_message,
        claude_path=claude_path,
        checked_at=now_iso(),
        status_message=status,
    )
    if log:
        console_log(f"Mode requested: {requested}")
        console_log(f"Mode active: {active}")
        console_log(f"API preflight: {api_message}")
        console_log(f"Claude Code preflight: {claude_message}")
    return PREFLIGHT


def get_preflight() -> PreflightState:
    return PREFLIGHT or refresh_preflight()


def active_mode() -> str:
    return get_preflight().active_mode


def book_dir(slug: str) -> Path:
    safe = slugify(slug)
    if safe != slug:
        raise ValueError("Invalid book slug")
    return CONFIG.books_dir / safe


def chapter_path(book: Path, chapter_id: str) -> Path:
    return book / "chapters" / f"{pad_id(chapter_id)}.md"


def chapter_plan_path(book: Path, chapter_id: str) -> Path:
    return book / "chapters" / f"{pad_id(chapter_id)}.plan.md"


def chapter_transcript_path(book: Path, chapter_id: str) -> Path:
    return book / "chapters" / f"{pad_id(chapter_id)}.transcript.md"


def conversation_path(book: Path) -> Path:
    return book / "conversation.json"


def research_dir(book: Path) -> Path:
    return book / "research"


def ensure_research_dirs(book: Path) -> None:
    root = research_dir(book)
    for name in ("sources", "people", "records", "originals", "proposals"):
        (root / name).mkdir(parents=True, exist_ok=True)
    timeline = root / "timeline.md"
    if not timeline.exists():
        write_text(timeline, DEFAULT_RESEARCH_TIMELINE)
    voice = root / "voice_profile.md"
    if not voice.exists():
        write_text(voice, DEFAULT_RESEARCH_VOICE)
    claims = root / "claims.json"
    if not claims.exists():
        write_json(claims, DEFAULT_RESEARCH_CLAIMS)


def project_type_for_book(book: Path) -> str:
    meta = load_meta(book)
    value = str(meta.get("projectType") or "novel")
    return value if value in PROJECT_TYPES else "novel"


def project_type_for_slug(slug: str) -> str:
    return project_type_for_book(book_dir(slug))


def is_research_project(slug: str) -> bool:
    return project_type_for_slug(slug) == "research"


def load_conversation(slug: str) -> dict[str, Any]:
    book = book_dir(slug)
    raw = json_safe_load(conversation_path(book), {"messages": []})
    if isinstance(raw, list):
        raw = {"messages": raw}
    raw.setdefault("messages", [])
    raw.setdefault("summary", "")
    return raw


def save_conversation(slug: str, conversation: dict[str, Any]) -> dict[str, Any]:
    book = book_dir(slug)
    conversation.setdefault("messages", [])
    conversation.setdefault("summary", "")
    write_json(conversation_path(book), conversation)
    return conversation


def append_conversation_message(
    slug: str,
    role: str,
    content: str,
    metadata: dict[str, Any] | None = None,
    message_id: str | None = None,
) -> dict[str, Any]:
    conversation = load_conversation(slug)
    message = {
        "id": message_id or uuid.uuid4().hex,
        "role": role,
        "content": content,
        "createdAt": now_iso(),
        "metadata": metadata or {},
    }
    conversation["messages"].append(message)
    save_conversation(slug, conversation)
    return message


def replace_conversation_message(slug: str, message_id: str, content: str, metadata: dict[str, Any] | None = None) -> None:
    conversation = load_conversation(slug)
    for message in conversation.get("messages", []):
        if message.get("id") == message_id:
            message["content"] = content
            message["updatedAt"] = now_iso()
            if metadata is not None:
                message["metadata"] = metadata
            save_conversation(slug, conversation)
            return


def clear_conversation(slug: str) -> dict[str, Any]:
    return save_conversation(slug, {"messages": [], "summary": ""})


def next_chapter_id(slug: str) -> str:
    meta = load_meta(book_dir(slug))
    used = [int(c["id"]) for c in meta.get("chapters", []) if str(c.get("id", "")).isdigit()]
    return pad_id((max(used) if used else 0) + 1)


def chapter_is_blank(content: str) -> bool:
    body = re.sub(r"^#+[^\n]*", "", content or "", flags=re.M).strip()
    return not body


def extract_draft_marker(text: str) -> tuple[str, dict[str, Any] | None]:
    marker_re = re.compile(r"<!--\s*GE_DRAFT\s+({.*?})\s*-->\s*$", re.S)
    match = marker_re.search(text.strip())
    if not match:
        return text, None
    try:
        metadata = json.loads(match.group(1))
    except json.JSONDecodeError:
        metadata = {"draft_for": "chapter"}
    cleaned = marker_re.sub("", text.strip()).rstrip()
    return cleaned, metadata


def default_meta(title: str, slug: str, project_type: str = "novel") -> dict[str, Any]:
    if project_type not in PROJECT_TYPES:
        project_type = "novel"
    return {
        "title": title,
        "slug": slug,
        "projectType": project_type,
        "createdAt": now_iso(),
        "updatedAt": now_iso(),
        "locked": False,
        "activeChapter": "01",
        "sections": [{"id": "part-1", "label": "Part 1", "chapterIds": []}],
        "chapters": [],
    }


def load_meta(book: Path) -> dict[str, Any]:
    slug = book.name
    meta = json_safe_load(book / "meta.json", default_meta(slug.replace("-", " ").title(), slug))
    meta.setdefault("title", slug.replace("-", " ").title())
    meta.setdefault("slug", slug)
    if meta.get("projectType") not in PROJECT_TYPES:
        meta["projectType"] = "novel"
    meta.setdefault("locked", False)
    meta.setdefault("activeChapter", "01")
    meta.setdefault("sections", [{"id": "part-1", "label": "Part 1", "chapterIds": []}])
    meta.setdefault("chapters", [])
    scan_chapter_files(book, meta)
    return meta


def save_meta(book: Path, meta: dict[str, Any]) -> None:
    meta["updatedAt"] = now_iso()
    write_json(book / "meta.json", meta)


def ensure_book(title: str, slug: str | None = None, project_type: str = "novel") -> dict[str, Any]:
    if project_type not in PROJECT_TYPES:
        project_type = "novel"
    base_slug = slugify(slug or title)
    safe_slug = base_slug
    target = CONFIG.books_dir / safe_slug
    suffix = 2
    while slug is None and target.exists():
        safe_slug = f"{base_slug}-{suffix}"
        target = CONFIG.books_dir / safe_slug
        suffix += 1
    target.mkdir(parents=True, exist_ok=True)
    (target / "canon").mkdir(exist_ok=True)
    (target / "chapters").mkdir(exist_ok=True)
    (target / "recaps").mkdir(exist_ok=True)
    if not conversation_path(target).exists():
        write_json(conversation_path(target), {"messages": [], "summary": ""})
    if not (target / "meta.json").exists():
        write_json(target / "meta.json", default_meta(title, safe_slug, project_type))
    for name, default in DEFAULT_CANON.items():
        path = target / "canon" / f"{name}.md"
        if not path.exists():
            write_text(path, default)
    meta = load_meta(target)
    if project_type == "research" and meta.get("projectType") != "research":
        meta["projectType"] = "research"
        save_meta(target, meta)
    if meta.get("projectType") == "research":
        ensure_research_dirs(target)
    if not meta.get("chapters"):
        write_text(chapter_path(target, "01"), "# Chapter 1\n\n")
        write_text(chapter_plan_path(target, "01"), "")
        write_text(chapter_transcript_path(target, "01"), "")
        meta["chapters"] = [{"id": "01", "title": "Chapter 1", "status": "draft", "section": "part-1"}]
        meta["activeChapter"] = "01"
        save_meta(target, meta)
    return load_book(safe_slug)


def scan_chapter_files(book: Path, meta: dict[str, Any]) -> None:
    chapters_dir = book / "chapters"
    chapters_dir.mkdir(parents=True, exist_ok=True)
    existing = {str(item.get("id")): item for item in meta.get("chapters", [])}
    for path in sorted(chapters_dir.glob("*.md")):
        stem = path.stem
        if stem.endswith(SIDE_SUFFIXES):
            continue
        if "." in stem:
            continue
        if not re.fullmatch(r"\d+", stem):
            continue
        cid = pad_id(stem)
        if cid not in existing:
            content = read_text(path)
            existing[cid] = {
                "id": cid,
                "title": clean_title_from_markdown(content, f"Chapter {int(cid)}"),
                "status": "draft",
                "section": meta.get("sections", [{"id": "part-1"}])[0].get("id", "part-1"),
            }
    meta["chapters"] = sorted(existing.values(), key=lambda item: int(item.get("id", "0")))
    section_ids = {section.get("id") for section in meta.get("sections", [])}
    if not section_ids:
        meta["sections"] = [{"id": "part-1", "label": "Part 1", "chapterIds": []}]
        section_ids = {"part-1"}
    default_section = next(iter(section_ids))
    for chapter in meta["chapters"]:
        chapter.setdefault("status", "draft")
        chapter.setdefault("title", f"Chapter {int(chapter['id'])}")
        if chapter.get("section") not in section_ids:
            chapter["section"] = default_section
    for section in meta.get("sections", []):
        section["chapterIds"] = [chapter["id"] for chapter in meta["chapters"] if chapter.get("section") == section.get("id")]
    if meta["chapters"] and meta.get("activeChapter") not in {c["id"] for c in meta["chapters"]}:
        meta["activeChapter"] = meta["chapters"][-1]["id"]


def list_books() -> list[dict[str, Any]]:
    CONFIG.books_dir.mkdir(parents=True, exist_ok=True)
    books: list[dict[str, Any]] = []
    for path in sorted(CONFIG.books_dir.iterdir()):
        if not path.is_dir() or not (path / "meta.json").exists():
            continue
        meta = load_meta(path)
        books.append(
            {
                "slug": path.name,
                "title": meta.get("title", path.name),
                "projectType": meta.get("projectType", "novel"),
                "chapterCount": len(meta.get("chapters", [])),
                "updatedAt": meta.get("updatedAt"),
                "locked": bool(meta.get("locked")),
            }
        )
    return books


def delete_book(slug: str, confirmation: str) -> dict[str, Any]:
    book = book_dir(slug)
    if not book.exists():
        raise FileNotFoundError(f"Book not found: {slug}")
    meta = load_meta(book)
    if meta.get("locked"):
        raise PermissionError("Project is locked. Unlock it before deleting.")
    title = str(meta.get("title") or slug)
    if confirmation.strip() != title:
        raise ValueError("Deletion confirmation did not match the project title.")
    books_root = CONFIG.books_dir.resolve()
    target = book.resolve()
    if target == books_root or books_root not in target.parents:
        raise ValueError("Refusing to delete a path outside the books directory.")
    if book.is_symlink():
        raise ValueError("Refusing to delete a symlinked project directory.")
    shutil.rmtree(target)
    return {"ok": True, "deleted": slug, "books": list_books()}


def tail_text(path: Path, max_chars: int = 12000) -> str:
    try:
        text = read_text(path)
    except Exception:
        return ""
    return text[-max_chars:]


def draft_bug_report(description: str, diagnostics: dict[str, Any] | None = None) -> dict[str, str]:
    description = description.strip()
    diagnostics = diagnostics or {}
    status = preflight_payload()
    payload = {
        "description": description,
        "client_diagnostics": diagnostics,
        "server": {
            "active_mode": active_mode(),
            "requested_mode": CONFIG.requested_mode,
            "books_dir": str(CONFIG.books_dir.resolve()),
            "python": sys.version.split()[0],
            "platform": sys.platform,
            "preflight": status,
        },
        "recent_desktop_log": tail_text(ROOT / "desktop-shell.log", 9000),
    }
    fallback = (
        "Generation Engine bug report\n\n"
        "What happened:\n"
        f"{description}\n\n"
        "Useful diagnostics:\n"
        f"{json.dumps(payload, indent=2, ensure_ascii=False)}\n"
    )
    body = fallback
    if real_backend_active():
        prompt = (
            "You are writing a bug report for the developers of Generation Engine. "
            "Make it concise, concrete, and directly actionable. Include: summary, expected behavior, actual behavior, "
            "reproduction steps, likely area of the app, diagnostics, and fix notes. "
            "Do not include secrets. Return only the email body.\n\n"
            f"Raw bug report data:\n{json.dumps(payload, indent=2, ensure_ascii=False)}"
        )
        try:
            body = import_message(prompt).strip() or fallback
        except Exception as exc:
            body = f"{fallback}\n\nAI formatting failed: {exc}\n"
    first_line = next((line.strip() for line in body.splitlines() if line.strip()), "Generation Engine bug report")
    first_line = re.sub(r"^[#*\s-]+|[*\s]+$", "", first_line)
    first_line = re.sub(r"^subject\s*:\s*", "", first_line, flags=re.I).strip()
    first_line = re.sub(r"^\[bug\]\s*", "", first_line, flags=re.I).strip()
    subject = first_line[:90] if first_line.lower().startswith("generation engine") else f"Generation Engine bug: {first_line[:70]}"
    return {"subject": subject, "body": body, "recipient": "founder@xyflowinnovations.com"}


def bug_report_recipient() -> str:
    return str(CONFIG.bug_report.get("recipient") or "founder@xyflowinnovations.com")


def bug_report_smtp_config() -> dict[str, Any]:
    smtp = CONFIG.bug_report.get("smtp")
    if not isinstance(smtp, dict):
        smtp = {}
    host = smtp.get("host") or os.environ.get("GENERATION_ENGINE_BUG_SMTP_HOST")
    username = smtp.get("username") or os.environ.get("GENERATION_ENGINE_BUG_SMTP_USERNAME")
    password = smtp.get("password") or os.environ.get("GENERATION_ENGINE_BUG_SMTP_PASSWORD")
    sender = smtp.get("from") or os.environ.get("GENERATION_ENGINE_BUG_FROM") or username
    port = int(smtp.get("port") or os.environ.get("GENERATION_ENGINE_BUG_SMTP_PORT") or 587)
    starttls = bool(smtp.get("starttls", True))
    ssl_enabled = bool(smtp.get("ssl", False) or port == 465)
    return {
        "host": host,
        "port": port,
        "username": username,
        "password": password,
        "from": sender,
        "starttls": starttls,
        "ssl": ssl_enabled,
    }


def bug_report_email_configured() -> bool:
    smtp = bug_report_smtp_config()
    return bool(smtp.get("host") and smtp.get("from") and smtp.get("username") and smtp.get("password"))


def send_bug_report_email(subject: str, body: str) -> dict[str, Any]:
    smtp = bug_report_smtp_config()
    missing = [key for key in ("host", "from", "username", "password") if not smtp.get(key)]
    if missing:
        raise RuntimeError(
            "Direct bug-report email is not configured. Add bugReport.smtp to config.local.json "
            f"(missing: {', '.join(missing)})."
        )
    recipient = bug_report_recipient()
    msg = EmailMessage()
    msg["From"] = str(smtp["from"])
    msg["To"] = recipient
    msg["Subject"] = subject.strip()[:180] or "Generation Engine bug report"
    msg.set_content(body)
    if smtp.get("ssl"):
        with smtplib.SMTP_SSL(str(smtp["host"]), int(smtp["port"]), timeout=30, context=ssl.create_default_context()) as server:
            server.login(str(smtp["username"]), str(smtp["password"]))
            server.send_message(msg)
    else:
        with smtplib.SMTP(str(smtp["host"]), int(smtp["port"]), timeout=30) as server:
            server.ehlo()
            if smtp.get("starttls"):
                server.starttls(context=ssl.create_default_context())
                server.ehlo()
            server.login(str(smtp["username"]), str(smtp["password"]))
            server.send_message(msg)
    return {"ok": True, "sent": True, "recipient": recipient}


def load_book(slug: str) -> dict[str, Any]:
    book = book_dir(slug)
    if not book.exists():
        raise FileNotFoundError(f"Book not found: {slug}")
    meta = load_meta(book)
    if meta.get("projectType") == "research":
        ensure_research_dirs(book)
    save_meta(book, meta)
    canon = {}
    for name in CANON_FILES:
        canon[name] = {
            "name": name,
            "path": str((book / "canon" / f"{name}.md").resolve()),
            "content": read_text(book / "canon" / f"{name}.md", DEFAULT_CANON[name]),
        }
    chapters = []
    for item in meta.get("chapters", []):
        cid = pad_id(item["id"])
        content = read_text(chapter_path(book, cid))
        plan = read_text(chapter_plan_path(book, cid))
        transcript = read_text(chapter_transcript_path(book, cid))
        chapters.append(
            {
                **item,
                "id": cid,
                "content": content,
                "plan": plan,
                "transcript": transcript,
                "path": str(chapter_path(book, cid).resolve()),
                "wordCount": len(re.findall(r"\b\S+\b", content)),
            }
        )
    data = {
        "slug": slug,
        "title": meta.get("title", slug),
        "path": str(book.resolve()),
        "meta": meta,
        "projectType": meta.get("projectType", "novel"),
        "canon": canon,
        "chapters": chapters,
        "conversation": load_conversation(slug),
        "hasApiKey": bool(CONFIG.api_key),
        "mode": active_mode(),
        "preflight": preflight_payload(),
    }
    if meta.get("projectType") == "research":
        data["research"] = load_research(slug)
    return data


def yaml_scalar(value: Any) -> str:
    if value is None:
        return '""'
    if isinstance(value, bool):
        return "true" if value else "false"
    if isinstance(value, (int, float)):
        return str(value)
    if isinstance(value, (list, dict)):
        return json.dumps(value, ensure_ascii=False)
    return json.dumps(str(value), ensure_ascii=False)


def parse_frontmatter(markdown: str) -> tuple[dict[str, Any], str]:
    text = markdown.replace("\r\n", "\n")
    if not text.startswith("---\n"):
        return {}, text
    end = text.find("\n---\n", 4)
    if end == -1:
        return {}, text
    meta: dict[str, Any] = {}
    front = text[4:end]
    for line in front.splitlines():
        if ":" not in line:
            continue
        key, value = line.split(":", 1)
        value = value.strip()
        try:
            meta[key.strip()] = json.loads(value)
        except json.JSONDecodeError:
            meta[key.strip()] = value.strip('"')
    return meta, text[end + 5 :].lstrip()


def frontmatter_markdown(meta: dict[str, Any], body: str) -> str:
    lines = ["---"]
    for key, value in meta.items():
        lines.append(f"{key}: {yaml_scalar(value)}")
    lines.append("---")
    return "\n".join(lines) + "\n\n" + body.strip() + "\n"


def safe_file_stem(value: str, fallback: str) -> str:
    stem = slugify(value or fallback)
    return stem[:80] or fallback


def checksum_bytes(data: bytes) -> str:
    return "sha256:" + hashlib.sha256(data).hexdigest()


def next_research_id(directory: Path, prefix: str, suffix: str = ".md") -> str:
    highest = 0
    for path in directory.glob(f"{prefix}-*{suffix}"):
        match = re.search(rf"{re.escape(prefix)}-(\d+)", path.stem)
        if match:
            highest = max(highest, int(match.group(1)))
    return f"{prefix}-{highest + 1:03d}"


def next_claim_id(claims: list[dict[str, Any]]) -> str:
    highest = 0
    for claim in claims:
        match = re.search(r"claim-(\d+)", str(claim.get("id", "")))
        if match:
            highest = max(highest, int(match.group(1)))
    return f"claim-{highest + 1:03d}"


def research_claims_path(book: Path) -> Path:
    return research_dir(book) / "claims.json"


def load_claims(book: Path) -> list[dict[str, Any]]:
    ensure_research_dirs(book)
    raw = json_safe_load(research_claims_path(book), [])
    if not isinstance(raw, list):
        raw = []
    people = load_people_index(book)
    return [derive_claim_readiness(claim, people) for claim in raw if isinstance(claim, dict)]


def save_claims(book: Path, claims: list[dict[str, Any]]) -> list[dict[str, Any]]:
    people = load_people_index(book)
    derived = [derive_claim_readiness(claim, people) for claim in claims]
    write_json(research_claims_path(book), derived)
    return derived


def load_people_index(book: Path) -> dict[str, dict[str, Any]]:
    root = research_dir(book) / "people"
    if not root.exists():
        return {}
    people: dict[str, dict[str, Any]] = {}
    for path in sorted(root.glob("*.md")):
        meta, body = parse_frontmatter(read_text(path))
        entity_id = str(meta.get("id") or path.stem)
        people[entity_id] = {**meta, "id": entity_id, "body": body, "path": str(path.resolve())}
    return people


def list_research_sources(book: Path) -> list[dict[str, Any]]:
    root = research_dir(book) / "sources"
    ensure_research_dirs(book)
    sources = []
    for path in sorted(root.glob("src-*.md")):
        meta, body = parse_frontmatter(read_text(path))
        source_id = str(meta.get("id") or path.stem)
        sources.append(
            {
                **meta,
                "id": source_id,
                "body": body,
                "path": str(path.resolve()),
                "excerpt": first_sentences(body, 2),
            }
        )
    return sources


def list_research_people(book: Path) -> list[dict[str, Any]]:
    return list(load_people_index(book).values())


def list_research_records(book: Path) -> list[dict[str, Any]]:
    root = research_dir(book) / "records"
    ensure_research_dirs(book)
    records = []
    for path in sorted(root.glob("req-*.md")):
        meta, body = parse_frontmatter(read_text(path))
        record_id = str(meta.get("id") or path.stem)
        records.append({**meta, "id": record_id, "body": body, "path": str(path.resolve())})
    return records


def load_research(slug: str) -> dict[str, Any]:
    book = book_dir(slug)
    ensure_research_dirs(book)
    return {
        "sources": list_research_sources(book),
        "people": list_research_people(book),
        "timeline": read_text(research_dir(book) / "timeline.md", DEFAULT_RESEARCH_TIMELINE),
        "claims": load_claims(book),
        "records": list_research_records(book),
        "voice_profile": read_text(research_dir(book) / "voice_profile.md", DEFAULT_RESEARCH_VOICE),
    }


def derive_claim_readiness(claim: dict[str, Any], people: dict[str, dict[str, Any]]) -> dict[str, Any]:
    updated = dict(claim)
    supporting = updated.get("supporting_sources") or []
    contradicting = updated.get("contradicting_sources") or []
    claim_type = str(updated.get("type") or "fact")
    confidence = str(updated.get("confidence") or "")
    if contradicting:
        confidence = "contradicted"
    elif len(supporting) >= 2 and confidence in {"", "single-sourced", "unsourced"}:
        confidence = "multi-sourced"
    elif len(supporting) == 1 and confidence in {"", "unsourced"}:
        confidence = "single-sourced"
    elif not supporting:
        confidence = "unsourced"
    blockers: list[str] = []
    if confidence == "unsourced":
        blockers.append("no_supporting_source")
    if contradicting:
        blockers.append("has_contradicting_sources")
    high_risk_subject = False
    for ref in updated.get("subject_refs") or []:
        person = people.get(str(ref)) or {}
        if person.get("living") is True or str(person.get("living")).lower() == "true":
            if str(person.get("defamation_risk", "")).lower() == "high" or str(person.get("status", "")).lower() in {"suspect", "witness"}:
                high_risk_subject = True
    if high_risk_subject and claim_type in {"allegation", "rumor", "inference"}:
        blockers.append("living_person_allegation_requires_stronger_sourcing")
    if claim_type == "allegation" and not any("court" in str(src.get("source_id", "")).lower() or "charging" in str(src.get("excerpt", "")).lower() for src in supporting):
        blockers.append("allegation_without_court_record_source")
    if blockers and ("no_supporting_source" in blockers or "has_contradicting_sources" in blockers):
        readiness = "blocked"
    elif blockers:
        readiness = "needs_review"
    elif confidence in {"confirmed", "multi-sourced"}:
        readiness = "ready"
    else:
        readiness = "qualified"
    updated["confidence"] = confidence
    updated["publish_readiness"] = readiness
    updated["publish_blockers"] = blockers
    updated.setdefault("created_at", now_iso())
    updated["last_modified"] = now_iso()
    return updated


def strip_html_to_text(raw: str) -> str:
    raw = re.sub(r"(?is)<(script|style|noscript|svg).*?</\1>", " ", raw)
    raw = re.sub(r"(?is)<br\s*/?>", "\n", raw)
    raw = re.sub(r"(?is)</p\s*>", "\n\n", raw)
    raw = re.sub(r"(?is)<[^>]+>", " ", raw)
    raw = html.unescape(raw)
    return re.sub(r"[ \t]+\n", "\n", re.sub(r"\n{3,}", "\n\n", re.sub(r"[ \t]{2,}", " ", raw))).strip()


def fetch_url_text(url: str) -> tuple[str, str, bytes]:
    req = urllib.request.Request(
        url,
        headers={"user-agent": "GenerationEngine/1.0 local research tool"},
        method="GET",
    )
    with urllib.request.urlopen(req, timeout=60) as response:
        data = response.read()
        content_type = response.headers.get("content-type", "")
    decoded = data.decode("utf-8", errors="replace")
    title_match = re.search(r"(?is)<title[^>]*>(.*?)</title>", decoded)
    title = strip_html_to_text(title_match.group(1)).strip() if title_match else url
    if "html" in content_type or decoded.lstrip().startswith("<"):
        return title or url, strip_html_to_text(decoded), data
    return title or url, decoded.strip(), data


def youtube_video_id(url: str) -> str | None:
    match = re.search(r"(?:v=|youtu\.be/|shorts/)([A-Za-z0-9_-]{8,})", url)
    return match.group(1) if match else None


def fetch_youtube_transcript(url: str) -> tuple[str, str, bytes]:
    video_id = youtube_video_id(url)
    if not video_id:
        raise ValueError("Could not read a YouTube video id from the URL.")
    yt_dlp = shutil.which("yt-dlp") or shutil.which("youtube-dl")
    if yt_dlp:
        TMP_DIR.mkdir(parents=True, exist_ok=True)
        base = TMP_DIR / f"yt-{uuid.uuid4().hex}"
        completed = subprocess.run(
            [yt_dlp, "--skip-download", "--write-auto-subs", "--write-subs", "--sub-lang", "en", "--sub-format", "vtt", "-o", str(base), url],
            capture_output=True,
            text=True,
            timeout=180,
            encoding="utf-8",
            errors="replace",
            **hidden_subprocess_kwargs(),
        )
        files = list(TMP_DIR.glob(base.name + "*.vtt"))
        if completed.returncode == 0 and files:
            text = "\n".join(strip_vtt(read_text(path)) for path in files)
            return f"YouTube transcript {video_id}", text.strip(), text.encode("utf-8")
    transcript_url = f"https://video.google.com/timedtext?lang=en&v={urllib.parse.quote(video_id)}"
    try:
        with urllib.request.urlopen(transcript_url, timeout=30) as response:
            raw = response.read()
        text = strip_html_to_text(raw.decode("utf-8", errors="replace"))
        if text:
            return f"YouTube transcript {video_id}", text, raw
    except Exception:
        pass
    placeholder = f"Transcript unavailable for {url}. Add captions, upload audio, or paste the transcript manually."
    return f"YouTube source {video_id}", placeholder, placeholder.encode("utf-8")


def strip_vtt(text: str) -> str:
    lines = []
    for line in text.splitlines():
        stripped = line.strip()
        if not stripped or stripped == "WEBVTT" or "-->" in stripped or stripped.isdigit():
            continue
        lines.append(re.sub(r"<[^>]+>", "", stripped))
    return "\n".join(lines)


def extract_uploaded_text(path: Path, original_name: str, content_type: str) -> str:
    suffix = path.suffix.lower()
    if suffix in {".txt", ".md", ".markdown", ".csv", ".json", ".html", ".htm"} or content_type.startswith("text/"):
        raw = path.read_bytes().decode("utf-8", errors="replace")
        return strip_html_to_text(raw) if suffix in {".html", ".htm"} else raw
    if suffix == ".pdf":
        text = extract_pdf_text(path)
        return text or f"[TEXT EXTRACTION NEEDED: PDF '{original_name}' was stored, but no local PDF text extractor was available.]"
    if suffix in {".wav", ".webm", ".ogg", ".mp3", ".m4a"}:
        try:
            return transcribe_audio_file(path)
        except Exception as exc:
            return f"[TRANSCRIPTION NEEDED: audio '{original_name}' was stored, but local Whisper failed: {exc}]"
    if suffix in {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"}:
        text = extract_image_text(path)
        return text or f"[OCR NEEDED: image '{original_name}' was stored, but no local OCR engine was available.]"
    return f"[ORIGINAL STORED: '{original_name}'. Paste extracted text or notes into this source.]"


def extract_pdf_text(path: Path) -> str:
    try:
        import fitz  # type: ignore

        doc = fitz.open(str(path))
        return "\n\n".join(page.get_text() for page in doc).strip()
    except Exception:
        pass
    try:
        from pypdf import PdfReader  # type: ignore

        reader = PdfReader(str(path))
        return "\n\n".join(page.extract_text() or "" for page in reader.pages).strip()
    except Exception:
        pass
    pdftotext = shutil.which("pdftotext")
    bundled_pdftotext = find_poppler_tool("pdftotext.exe")
    if bundled_pdftotext:
        pdftotext = bundled_pdftotext
    if pdftotext:
        completed = subprocess.run([pdftotext, str(path), "-"], capture_output=True, text=True, timeout=120, encoding="utf-8", errors="replace", **hidden_subprocess_kwargs())
        if completed.returncode == 0:
            text = completed.stdout.strip()
            if text:
                return text
    return ocr_pdf_pages(path)


def ocr_pdf_pages(path: Path, max_pages: int = 12) -> str:
    pdftoppm = find_poppler_tool("pdftoppm.exe") or shutil.which("pdftoppm")
    tesseract = find_tesseract_executable()
    if not pdftoppm or not tesseract:
        return ""
    out_dir = TMP_DIR / f"pdf-ocr-{uuid.uuid4().hex}"
    out_dir.mkdir(parents=True, exist_ok=True)
    prefix = out_dir / "page"
    completed = subprocess.run(
        [pdftoppm, "-f", "1", "-l", str(max_pages), "-r", "220", "-png", str(path), str(prefix)],
        capture_output=True,
        text=True,
        timeout=180,
        encoding="utf-8",
        errors="replace",
        **hidden_subprocess_kwargs(),
    )
    if completed.returncode != 0:
        return ""
    texts = []
    for image_path in sorted(out_dir.glob("page-*.png")):
        text = extract_image_text(image_path)
        if text:
            texts.append(text)
    try:
        shutil.rmtree(out_dir)
    except OSError:
        pass
    return "\n\n".join(texts).strip()


def extract_image_text(path: Path) -> str:
    tesseract = find_tesseract_executable()
    if not tesseract:
        return ""
    env = os.environ.copy()
    tessdata = Path(tesseract).parent / "tessdata"
    if tessdata.exists():
        env["TESSDATA_PREFIX"] = str(tessdata)
    completed = subprocess.run(
        [tesseract, str(path), "stdout", "-l", "eng"],
        capture_output=True,
        text=True,
        timeout=120,
        encoding="utf-8",
        errors="replace",
        env=env,
        **hidden_subprocess_kwargs(),
    )
    return completed.stdout.strip() if completed.returncode == 0 else ""


def find_tesseract_executable() -> str | None:
    candidates = [
        ROOT / "runtime" / "tesseract" / "tesseract.exe",
        ROOT / "runtime" / "tesseract" / "bin" / "tesseract.exe",
    ]
    for candidate in candidates:
        if candidate.exists():
            return str(candidate)
    return shutil.which("tesseract")


def find_poppler_tool(name: str) -> str | None:
    root = ROOT / "runtime" / "poppler"
    if root.exists():
        for candidate in root.rglob(name):
            if candidate.is_file():
                return str(candidate)
    return None


def basic_research_extraction(title: str, text: str, source_id: str) -> dict[str, Any]:
    sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text.replace("\n", " ")) if len(s.strip()) > 35]
    excerpts = sentences[:5]
    names = extract_names(text, limit=12)
    claims = []
    for sentence in excerpts[:4]:
        claims.append(
            {
                "text": sentence[:600],
                "subject_refs": [],
                "event_ref": "",
                "supporting_sources": [{"source_id": source_id, "excerpt": sentence[:800]}],
                "contradicting_sources": [],
                "type": "fact",
                "confidence": "single-sourced",
                "notes": "Auto-extracted from source text; review wording before publication.",
            }
        )
    return {
        "summary": first_sentences(text, 3) or f"Source ingested: {title}",
        "excerpts": excerpts,
        "entities": [{"display_name": name, "type": "person", "status": "other", "living": "unknown"} for name in names],
        "events": [],
        "claims": claims,
    }


def research_extraction_prompt(title: str, text: str, source_id: str) -> str:
    return (
        "Extract a careful source ledger proposal from this nonfiction source. Return strict JSON only with keys: "
        "summary (string), excerpts (array of exact verbatim excerpts from the source), entities (array of objects with display_name,type,status,living), "
        "events (array of objects with title,date,description,source_refs), claims (array of atomic claim objects with text, subject_refs, event_ref, supporting_sources, contradicting_sources, type, confidence, notes). "
        "Do not invent. Candidate claims must be directly supported by exact excerpts from this source. Use the provided source id in supporting_sources.\n\n"
        f"source_id: {source_id}\nTitle: {title}\n\nSource text:\n{text[:60000]}"
    )


def extract_research_with_ai(slug: str, title: str, text: str, source_id: str) -> dict[str, Any]:
    fallback = basic_research_extraction(title, text, source_id)
    if not real_backend_active():
        return fallback
    try:
        raw = ai_message(slug, research_extraction_prompt(title, text, source_id), max_tokens=5000, purpose="utility")
        extracted = json.loads(extract_json(raw))
        if isinstance(extracted, dict):
            return {**fallback, **extracted}
    except Exception:
        return fallback
    return fallback


def create_or_update_entity(book: Path, entity: dict[str, Any], source_id: str) -> str:
    name = str(entity.get("display_name") or entity.get("name") or "").strip()
    if not name:
        return ""
    entity_type = str(entity.get("type") or "person").lower()
    if entity_type not in {"person", "place", "organization", "object"}:
        entity_type = "person"
    entity_id = f"{entity_type}-{safe_file_stem(name, 'entity')}"
    path = research_dir(book) / "people" / f"{entity_id}.md"
    existing_meta, existing_body = parse_frontmatter(read_text(path))
    refs = list(dict.fromkeys([*(existing_meta.get("source_refs") or []), source_id]))
    status = str(entity.get("status") or existing_meta.get("status") or "other")
    living = entity.get("living", existing_meta.get("living", "unknown"))
    defamation_risk = "high" if (str(living).lower() == "true" and status in {"suspect", "witness"}) else existing_meta.get("defamation_risk", "low")
    meta = {
        "id": entity_id,
        "type": entity_type,
        "display_name": name,
        "also_known_as": existing_meta.get("also_known_as", []),
        "status": status,
        "living": living,
        "defamation_risk": defamation_risk,
        "source_refs": refs,
    }
    body = existing_body.strip() or f"# {name}\n\n## Bio\n\n## Linked claims\n\n## Open questions\n"
    write_text(path, frontmatter_markdown(meta, body))
    return entity_id


def append_timeline_events(book: Path, events: list[dict[str, Any]], source_id: str, claim_ids: list[str]) -> None:
    if not events:
        return
    path = research_dir(book) / "timeline.md"
    existing = read_text(path, DEFAULT_RESEARCH_TIMELINE).rstrip()
    additions = []
    for event in events[:12]:
        title = str(event.get("title") or "Source event").strip()
        date = str(event.get("date") or "undated").strip()
        description = str(event.get("description") or "").strip()
        additions.append(
            f"## {date} - {title}\n- claim_refs: {json.dumps(claim_ids)}\n- source_refs: [{source_id}]\n\n{description}\n"
        )
    write_text(path, existing + "\n\n" + "\n\n".join(additions) + "\n")


def create_research_source(slug: str, payload: dict[str, Any], original_bytes: bytes | None = None, original_name: str | None = None, extracted_text: str | None = None) -> dict[str, Any]:
    book = book_dir(slug)
    ensure_research_dirs(book)
    root = research_dir(book)
    source_id = next_research_id(root / "sources", "src")
    provenance = str(payload.get("provenance") or payload.get("kind") or "clipboard_paste")
    source_type = str(payload.get("type") or "other")
    reliability = str(payload.get("reliability") or "secondary")
    url = str(payload.get("url") or "").strip()
    title = str(payload.get("title") or "").strip()
    text = str(extracted_text if extracted_text is not None else payload.get("text") or "").strip()
    original_rel = ""
    raw_bytes = original_bytes
    if url and not text:
        if "youtube.com" in url or "youtu.be" in url:
            title, text, raw_bytes = fetch_youtube_transcript(url)
            provenance = "youtube_transcript"
            source_type = "broadcast"
        else:
            title, text, raw_bytes = fetch_url_text(url)
            provenance = "url_fetch"
    if not title:
        title = clean_title_from_markdown(text, "Untitled source")
    if raw_bytes is None:
        raw_bytes = text.encode("utf-8")
    if original_name or url or raw_bytes:
        suffix = Path(original_name or urllib.parse.urlparse(url).path or "source.txt").suffix
        if not suffix:
            suffix = ".txt"
        original_name = original_name or f"{safe_file_stem(title, source_id)}{suffix}"
        original_safe = f"{source_id}-{safe_file_stem(Path(original_name).stem, 'original')}{suffix.lower()}"
        original_path = root / "originals" / original_safe
        original_path.write_bytes(raw_bytes)
        original_rel = f"originals/{original_safe}"
    checksum = checksum_bytes(raw_bytes)
    extraction = extract_research_with_ai(slug, title, text, source_id)
    entity_refs = [create_or_update_entity(book, entity, source_id) for entity in extraction.get("entities", []) if isinstance(entity, dict)]
    entity_refs = [ref for ref in entity_refs if ref]
    claims = load_claims(book)
    added_claims: list[dict[str, Any]] = []
    for claim in extraction.get("claims", [])[:20]:
        if not isinstance(claim, dict):
            continue
        claim["id"] = next_claim_id(claims)
        claim.setdefault("supporting_sources", [{"source_id": source_id, "excerpt": str(claim.get("text", ""))[:800]}])
        claim.setdefault("contradicting_sources", [])
        claim.setdefault("subject_refs", [])
        claim.setdefault("event_ref", "")
        claim.setdefault("type", "fact")
        claim.setdefault("confidence", "single-sourced")
        claim.setdefault("notes", "Auto-extracted from source; review before publication.")
        claims.append(claim)
        added_claims.append(claim)
    claims = save_claims(book, claims)
    added_ids = [claim["id"] for claim in added_claims]
    append_timeline_events(book, [event for event in extraction.get("events", []) if isinstance(event, dict)], source_id, added_ids)
    meta = {
        "id": source_id,
        "title": title,
        "type": source_type,
        "url": url,
        "file": original_rel,
        "accessed": time.strftime("%Y-%m-%d"),
        "author": str(payload.get("author") or ""),
        "agency": str(payload.get("agency") or ""),
        "provenance": provenance,
        "reliability": reliability,
        "checksum": checksum,
        "verified_by": "",
        "verified_at": "",
        "entity_refs": entity_refs,
        "claim_refs": added_ids,
    }
    excerpts = "\n\n".join(f"> {str(excerpt).strip()}" for excerpt in extraction.get("excerpts", [])[:8] if str(excerpt).strip())
    body = (
        f"# {title}\n\n"
        "## AI-generated summary\n\n"
        f"{str(extraction.get('summary') or '').strip()}\n\n"
        "## Exact excerpts\n\n"
        f"{excerpts or '> No exact excerpts were extracted automatically. Review the original/source text.'}\n\n"
        "## Extracted text\n\n"
        f"{text[:120000]}\n\n"
        "## User notes\n\n"
    )
    write_text(root / "sources" / f"{source_id}.md", frontmatter_markdown(meta, body))
    detect_claim_contradictions(book)
    return load_research(slug)


def detect_claim_contradictions(book: Path) -> None:
    claims = load_claims(book)
    by_subject: dict[str, list[dict[str, Any]]] = {}
    for claim in claims:
        key = " ".join(sorted(str(ref) for ref in claim.get("subject_refs") or [])) or "general"
        by_subject.setdefault(key, []).append(claim)
    for group in by_subject.values():
        for claim in group:
            text = str(claim.get("text", "")).lower()
            contradictions = claim.setdefault("contradicting_sources", [])
            for other in group:
                if other is claim:
                    continue
                other_text = str(other.get("text", "")).lower()
                if any(word in text and word in other_text for word in ("not", "never", "denied", "midnight", "11:30")) and text != other_text:
                    src = (other.get("supporting_sources") or [{}])[0]
                    note = {"source_id": src.get("source_id", ""), "excerpt": other.get("text", ""), "note": f"Potential conflict with {other.get('id')}"}
                    if note not in contradictions:
                        contradictions.append(note)
    save_claims(book, claims)


def research_bundle(slug: str) -> str:
    book = book_dir(slug)
    ensure_research_dirs(book)
    research = load_research(slug)
    source_chunks = []
    for source in research["sources"][-25:]:
        source_chunks.append(f"## {source['id']} - {source.get('title', '')}\n\n{source.get('body', '')[:10000]}")
    people_chunks = []
    for person in research["people"]:
        people_chunks.append(f"## {person['id']} - {person.get('display_name', '')}\n\n{person.get('body', '')[:4000]}")
    return "\n\n---\n\n".join(
        [
            "## research/voice_profile.md\n\n" + research["voice_profile"],
            "## research/claims.json\n\n" + json.dumps(research["claims"], indent=2, ensure_ascii=False),
            "## research/timeline.md\n\n" + research["timeline"],
            "## research/sources index\n\n" + ("\n\n".join(source_chunks) or "No sources yet."),
            "## research/people index\n\n" + ("\n\n".join(people_chunks) or "No people/entities yet."),
            "## research/records index\n\n" + json.dumps(research["records"], indent=2, ensure_ascii=False),
        ]
    )


def create_chapter(slug: str, title: str | None = None) -> dict[str, Any]:
    book = book_dir(slug)
    meta = load_meta(book)
    used = [int(c["id"]) for c in meta.get("chapters", []) if str(c.get("id", "")).isdigit()]
    next_id = pad_id((max(used) if used else 0) + 1)
    section = meta.get("sections", [{"id": "part-1"}])[0]["id"]
    chapter_title = title or f"Chapter {int(next_id)}"
    write_text(chapter_path(book, next_id), f"# {chapter_title}\n\n")
    write_text(chapter_plan_path(book, next_id), "")
    write_text(chapter_transcript_path(book, next_id), "")
    meta["chapters"].append({"id": next_id, "title": chapter_title, "status": "draft", "section": section})
    meta["activeChapter"] = next_id
    save_meta(book, meta)
    return load_book(slug)


def append_transcript(slug: str, chapter_id: str, speaker: str, text: str) -> str:
    book = book_dir(slug)
    path = chapter_transcript_path(book, chapter_id)
    existing = read_text(path)
    clean = text.strip()
    if not clean:
        return existing
    stamp = time.strftime("%Y-%m-%d %H:%M")
    entry = f"\n\n## {speaker} - {stamp}\n\n{clean}\n"
    updated = existing.rstrip() + entry
    write_text(path, updated.lstrip())
    return updated.lstrip()


def canon_bundle(slug: str) -> str:
    book = book_dir(slug)
    chunks = []
    for name in CANON_FILES:
        chunks.append(f"## canon/{name}.md\n\n{read_text(book / 'canon' / f'{name}.md')}")
    return "\n\n---\n\n".join(chunks)


def base_system_for_slug(slug: str) -> str:
    return BASE_RESEARCH_SYSTEM if is_research_project(slug) else BASE_NOVELIST_SYSTEM


def memory_bundle_for_slug(slug: str) -> tuple[str, str]:
    if is_research_project(slug):
        return "RESEARCH EVIDENCE SPINE", research_bundle(slug)
    return "BOOK CANON", canon_bundle(slug)


def recent_locked_chapters(slug: str, before_id: str | None = None, count: int = 3) -> str:
    data = load_book(slug)
    chapters = data["chapters"]
    if before_id:
        chapters = [c for c in chapters if int(c["id"]) < int(before_id)]
    locked = [c for c in chapters if c.get("status") == "locked" and c.get("content", "").strip()]
    if not locked:
        locked = [c for c in chapters if c.get("content", "").strip()]
    chunks = []
    for chapter in locked[-count:]:
        chunks.append(f"## {chapter['title']} ({chapter['id']})\n\n{chapter['content']}")
    return "\n\n---\n\n".join(chunks)


def anthropic_system_blocks(slug: str, chapter_id: str | None = None) -> list[dict[str, Any]]:
    label, memory = memory_bundle_for_slug(slug)
    blocks: list[dict[str, Any]] = [{"type": "text", "text": base_system_for_slug(slug)}]
    blocks.append(
        {
            "type": "text",
            "text": f"{label}\n\n{memory}",
            "cache_control": {"type": "ephemeral"},
        }
    )
    recent = recent_locked_chapters(slug, chapter_id)
    if recent.strip():
        blocks.append(
            {
                "type": "text",
                "text": "RECENT LOCKED CHAPTERS\n\n" + recent,
                "cache_control": {"type": "ephemeral"},
            }
        )
    return blocks


def anthropic_message(
    slug: str,
    prompt: str,
    model: str,
    max_tokens: int = 4000,
    chapter_id: str | None = None,
) -> str:
    if not CONFIG.api_key:
        raise RuntimeError("Missing ANTHROPIC_API_KEY")
    preflight = get_preflight()
    if active_mode() == "api" and not preflight.api_ok:
        raise RuntimeError(preflight.api_message)
    body = {
        "model": model,
        "max_tokens": max_tokens,
        "system": anthropic_system_blocks(slug, chapter_id),
        "messages": [{"role": "user", "content": prompt}],
    }
    req = urllib.request.Request(
        "https://api.anthropic.com/v1/messages",
        data=json.dumps(body).encode("utf-8"),
        headers={
            "content-type": "application/json",
            "x-api-key": CONFIG.api_key,
            "anthropic-version": "2023-06-01",
        },
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=300) as response:
        data = json.loads(response.read().decode("utf-8"))
    return "".join(block.get("text", "") for block in data.get("content", []) if block.get("type") == "text")


def anthropic_stream(
    slug: str,
    prompt: str,
    model: str,
    max_tokens: int = 12000,
    chapter_id: str | None = None,
) -> Iterable[str]:
    if not CONFIG.api_key:
        raise RuntimeError("Missing ANTHROPIC_API_KEY")
    preflight = get_preflight()
    if active_mode() == "api" and not preflight.api_ok:
        raise RuntimeError(preflight.api_message)
    body = {
        "model": model,
        "max_tokens": max_tokens,
        "stream": True,
        "system": anthropic_system_blocks(slug, chapter_id),
        "messages": [{"role": "user", "content": prompt}],
    }
    req = urllib.request.Request(
        "https://api.anthropic.com/v1/messages",
        data=json.dumps(body).encode("utf-8"),
        headers={
            "content-type": "application/json",
            "x-api-key": CONFIG.api_key,
            "anthropic-version": "2023-06-01",
        },
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=600) as response:
        for raw_line in response:
            line = raw_line.decode("utf-8", errors="replace").strip()
            if not line.startswith("data: "):
                continue
            payload = line[6:]
            if payload == "[DONE]":
                break
            try:
                data = json.loads(payload)
            except json.JSONDecodeError:
                continue
            if data.get("type") == "content_block_delta":
                delta = data.get("delta", {})
                if delta.get("type") == "text_delta":
                    yield delta.get("text", "")
            elif data.get("type") == "error":
                error = data.get("error", {})
                raise RuntimeError(error.get("message", "Anthropic streaming error"))


def compose_system_prompt(slug: str, chapter_id: str | None = None) -> str:
    label, memory = memory_bundle_for_slug(slug)
    chunks = [base_system_for_slug(slug), f"{label}\n\n{memory}"]
    recent = recent_locked_chapters(slug, chapter_id)
    if recent.strip():
        chunks.append("RECENT LOCKED CHAPTERS\n\n" + recent)
    return "\n\n---\n\n".join(chunks)


def claude_code_command(model: str, system_prompt: str, stream: bool) -> tuple[list[str], str]:
    preflight = get_preflight()
    if not preflight.claude_path or not preflight.claude_code_ok:
        raise RuntimeError(preflight.claude_code_message)
    command = [preflight.claude_path, "--print", "--model", model]
    command.extend(["--append-system-prompt", CLAUDE_CODE_BRIDGE_SYSTEM])
    stdin_prefix = "AUTHORITATIVE SYSTEM CONTEXT\n\n" + system_prompt + "\n\n---\n\nUSER TASK\n\n"
    if stream:
        command.extend(["--output-format", "stream-json", "--verbose"])
    return command, stdin_prefix


def extract_claude_stream_delta(event: dict[str, Any], prior_text: str) -> tuple[str, str]:
    event_type = event.get("type")
    if event_type == "content_block_delta":
        delta = event.get("delta", {})
        if isinstance(delta, dict) and delta.get("type") == "text_delta":
            text = str(delta.get("text", ""))
            return text, prior_text + text
    if event_type == "assistant":
        message = event.get("message", {})
        content = message.get("content", []) if isinstance(message, dict) else event.get("content", [])
        text = ""
        if isinstance(content, list):
            for block in content:
                if isinstance(block, dict) and block.get("type") == "text":
                    text += str(block.get("text", ""))
        elif isinstance(content, str):
            text = content
        if text:
            if text.startswith(prior_text):
                return text[len(prior_text) :], text
            return text, prior_text + text
    if event_type == "text":
        text = str(event.get("text", ""))
        return text, prior_text + text
    if event_type == "result" and isinstance(event.get("result"), str):
        text = event["result"]
        if text.startswith(prior_text):
            return text[len(prior_text) :], text
        return text, prior_text + text
    return "", prior_text


def claude_code_message(
    slug: str,
    prompt: str,
    model: str,
    chapter_id: str | None = None,
) -> str:
    command, stdin_prefix = claude_code_command(model, compose_system_prompt(slug, chapter_id), stream=False)
    completed = subprocess.run(
        command,
        input=stdin_prefix + prompt,
        capture_output=True,
        text=True,
        timeout=GENERATION_TIMEOUT_SECONDS,
        encoding="utf-8",
        errors="replace",
        **hidden_subprocess_kwargs(),
    )
    if completed.returncode != 0:
        raise RuntimeError((completed.stderr or completed.stdout or "Claude Code failed").strip())
    return completed.stdout.strip()


def claude_code_message_for_import(prompt: str) -> str:
    command, stdin_prefix = claude_code_command(CONFIG.claude_code_model, IMPORT_SYSTEM, stream=False)
    completed = subprocess.run(
        command,
        input=stdin_prefix + prompt,
        capture_output=True,
        text=True,
        timeout=GENERATION_TIMEOUT_SECONDS,
        encoding="utf-8",
        errors="replace",
        **hidden_subprocess_kwargs(),
    )
    if completed.returncode != 0:
        raise RuntimeError((completed.stderr or completed.stdout or "Claude Code import failed").strip())
    return completed.stdout.strip()


def claude_code_stream(
    slug: str,
    prompt: str,
    model: str,
    chapter_id: str | None = None,
) -> Iterable[str]:
    command, stdin_prefix = claude_code_command(model, compose_system_prompt(slug, chapter_id), stream=True)
    process = subprocess.Popen(
        command,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        encoding="utf-8",
        errors="replace",
        bufsize=1,
        **hidden_subprocess_kwargs(),
    )
    stderr_lines: list[str] = []

    def collect_stderr() -> None:
        assert process.stderr is not None
        for line in process.stderr:
            stderr_lines.append(line)

    stderr_thread = threading.Thread(target=collect_stderr, daemon=True)
    stderr_thread.start()
    try:
        assert process.stdin is not None
        process.stdin.write(stdin_prefix + prompt)
        process.stdin.close()
        assert process.stdout is not None
        prior_text = ""
        started_at = time.time()
        for line in process.stdout:
            if time.time() - started_at > GENERATION_TIMEOUT_SECONDS:
                process.kill()
                raise TimeoutError("Claude Code generation timed out.")
            line = line.strip()
            if not line:
                continue
            try:
                event = json.loads(line)
            except json.JSONDecodeError:
                delta = line
            else:
                delta, prior_text = extract_claude_stream_delta(event, prior_text)
            if delta:
                yield delta
        return_code = process.wait(timeout=30)
        if return_code != 0:
            raise RuntimeError("".join(stderr_lines).strip() or f"Claude Code exited with {return_code}")
    finally:
        if process.poll() is None:
            process.kill()


def ai_message(
    slug: str,
    prompt: str,
    max_tokens: int = 4000,
    chapter_id: str | None = None,
    purpose: str = "utility",
) -> str:
    mode = active_mode()
    if mode == "api":
        model = CONFIG.draft_model if purpose == "draft" else CONFIG.utility_model
        return anthropic_message(slug, prompt, model, max_tokens=max_tokens, chapter_id=chapter_id)
    if mode == "claude-code":
        return claude_code_message(slug, prompt, CONFIG.claude_code_model, chapter_id=chapter_id)
    raise RuntimeError("No real Claude backend is active.")


def ai_stream(
    slug: str,
    prompt: str,
    max_tokens: int = 12000,
    chapter_id: str | None = None,
) -> Iterable[str]:
    mode = active_mode()
    if mode == "api":
        return anthropic_stream(slug, prompt, CONFIG.draft_model, max_tokens=max_tokens, chapter_id=chapter_id)
    if mode == "claude-code":
        return claude_code_stream(slug, prompt, CONFIG.claude_code_model, chapter_id=chapter_id)
    raise RuntimeError("No real Claude backend is active.")


def import_message(prompt: str) -> str:
    mode = active_mode()
    if mode == "api":
        return anthropic_message_for_import(prompt)
    if mode == "claude-code":
        return claude_code_message_for_import(prompt)
    raise RuntimeError("No real Claude backend is active.")


def real_backend_active() -> bool:
    return active_mode() in {"api", "claude-code"}


def preflight_payload() -> dict[str, Any]:
    preflight = get_preflight()
    return {
        "requestedMode": preflight.requested_mode,
        "activeMode": preflight.active_mode,
        "apiOk": preflight.api_ok,
        "apiMessage": preflight.api_message,
        "claudeCodeOk": preflight.claude_code_ok,
        "claudeCodeMessage": preflight.claude_code_message,
        "claudePath": preflight.claude_path,
        "checkedAt": preflight.checked_at,
        "statusMessage": preflight.status_message,
    }


def mock_voice_profile(authors: str) -> str:
    authors = authors.strip() or "the authors the user named"
    return (
        "# Voice Profile\n\n"
        f"The prose should take inspiration from the user's taste in {authors}, but it must not imitate any living or dead author's sentences. "
        "Use a flexible rhythm: blunt declarative lines when pressure rises, longer sensory clauses when a character notices something dangerous or intimate. "
        "Dialogue should arrive with bite and subtext, rarely explaining the feeling underneath. Internal thought should be short, physical, and specific. "
        "Descriptions should favor handled objects, weather against skin, rooms with history, and social pressure visible in gesture. "
        "Keep the register adult and concrete. Avoid assistant-like summaries, generic uplift, and paragraphs that announce what the scene means."
    )


def mock_import(raw: str, title: str) -> dict[str, Any]:
    chapters = split_markdown_chapters(raw)
    all_text = "\n\n".join(chapter["content"] for chapter in chapters)
    names = extract_names(all_text)
    character_lines = ["# Characters\n"]
    for name in names[:12]:
        character_lines.append(f"## {name}\n- Role: Proposed from import. Review and edit.\n- Voice:\n- Goals:\n- Fears:\n- Relationships:\n- Current state:\n")
    if len(character_lines) == 1:
        character_lines.append("## Protagonist\n- Role:\n- Voice:\n- Goals:\n- Fears:\n- Relationships:\n- Current state:\n")
    timeline = ["# Timeline\n"]
    for index, chapter in enumerate(chapters, start=1):
        timeline.append(f"## Chapter {index}: {chapter['title']}\n{first_sentences(chapter['content'], 2) or 'Summary needed.'}\n")
    world = (
        "# World\n\n"
        "## Setting\nProposed from imported chapters. Review locations, institutions, rules, and history here.\n\n"
        "## Rules\n\n## Factions\n\n## History\n"
    )
    arcs = (
        "# Arcs / Threads\n\n"
        "## Open\n- Review the imported chapters and list the promises, mysteries, threats, and relationships still in motion.\n\n"
        "## Advanced\n\n## Closed\n"
    )
    return {
        "title": title or "Imported Book",
        "chapters": [
            {"id": pad_id(index), "title": chapter["title"], "content": chapter["content"]}
            for index, chapter in enumerate(chapters, start=1)
        ],
        "canon": {
            "world": world,
            "characters": "\n".join(character_lines),
            "timeline": "\n".join(timeline),
            "arcs": arcs,
            "voice_profile": DEFAULT_CANON["voice_profile"],
        },
        "mode": "local-fallback",
    }


def listify(value: Any) -> list[Any]:
    if value is None:
        return []
    if isinstance(value, list):
        return [item for item in value if item not in (None, "")]
    if isinstance(value, tuple):
        return [item for item in value if item not in (None, "")]
    text = str(value).strip()
    return [text] if text else []


def short_note(value: Any, limit: int = 340) -> str:
    text = re.sub(r"\s+", " ", str(value or "")).strip()
    return text[:limit].rstrip()


def unique_notes(notes: Iterable[Any], limit: int = 20) -> list[str]:
    seen: set[str] = set()
    out: list[str] = []
    for note in notes:
        text = short_note(note)
        if not text:
            continue
        key = text.lower()
        if key in seen:
            continue
        seen.add(key)
        out.append(text)
        if len(out) >= limit:
            break
    return out


def bullet_lines(notes: Iterable[Any], fallback: str = "Not yet clear from the imported chapters.") -> str:
    items = unique_notes(notes)
    if not items:
        return f"- {fallback}"
    return "\n".join(f"- {item}" for item in items)


def import_batches(chapters: list[dict[str, str]], max_chars: int = 65000, max_chapters: int = 4) -> list[list[dict[str, str]]]:
    batches: list[list[dict[str, str]]] = []
    current: list[dict[str, str]] = []
    current_chars = 0
    for index, chapter in enumerate(chapters, start=1):
        prepared = {
            "id": pad_id(index),
            "title": chapter.get("title") or f"Chapter {index}",
            "content": chapter.get("content") or "",
        }
        size = len(prepared["content"])
        if current and (current_chars + size > max_chars or len(current) >= max_chapters):
            batches.append(current)
            current = []
            current_chars = 0
        current.append(prepared)
        current_chars += size
    if current:
        batches.append(current)
    return batches


def build_import_batch_prompt(title: str, batch: list[dict[str, str]]) -> str:
    chapters = []
    for chapter in batch:
        chapters.append(
            f"## IMPORTED CHAPTER {chapter['id']}: {chapter['title']}\n\n{chapter['content'][:85000]}"
        )
    return (
        "Analyze this batch of imported chapters for a novel project. Return one strict JSON object with this shape:\n"
        "{\n"
        '  "chapters": [\n'
        "    {\n"
        '      "id": "01",\n'
        '      "title": "Chapter title",\n'
        '      "summary": "6-10 concrete sentences covering what changes in the chapter, including the ending state.",\n'
        '      "characters": [\n'
        '        {"name":"", "role":"", "voice":"", "goals":"", "fears":"", "relationships":[""], "current_state":"", "evidence":["short concrete support from this chapter"]}\n'
        "      ],\n"
        '      "world": ["setting, rules, institutions, objects, history, magic/tech/social details"],\n'
        '      "events": [{"when":"", "where":"", "what":"", "characters":[""]}],\n'
        '      "arcs_opened": [""],\n'
        '      "arcs_advanced": [""],\n'
        '      "arcs_closed": [""],\n'
        '      "voice_notes": ["sentence rhythm, POV handling, dialogue habits, descriptive density"]\n'
        "    }\n"
        "  ]\n"
        "}\n\n"
        "Requirements:\n"
        "- Include every chapter in the batch.\n"
        "- Do not leave character fields blank when the chapter gives evidence. If a field is unclear, write 'unclear from this chapter'.\n"
        "- Keep summaries specific. No generic 'tension rises' language.\n"
        "- Character evidence should be concrete but short.\n"
        "- JSON only.\n\n"
        f"Book title: {title}\n\n"
        + "\n\n---\n\n".join(chapters)
    )


def local_chapter_analysis(chapter: dict[str, str]) -> dict[str, Any]:
    content = chapter.get("content", "")
    names = extract_names(content, 12)
    return {
        "id": chapter["id"],
        "title": chapter.get("title") or f"Chapter {int(chapter['id'])}",
        "summary": first_sentences(content, 5) or "Summary needs review.",
        "characters": [
            {
                "name": name,
                "role": f"Referenced in Chapter {int(chapter['id'])}.",
                "voice": "unclear from this chapter",
                "goals": "unclear from this chapter",
                "fears": "unclear from this chapter",
                "relationships": [],
                "current_state": "unclear from this chapter",
                "evidence": [f"Name appears in Chapter {int(chapter['id'])}."],
            }
            for name in names
        ],
        "world": [],
        "events": [{"when": f"Chapter {int(chapter['id'])}", "where": "", "what": first_sentences(content, 2), "characters": names[:6]}],
        "arcs_opened": [],
        "arcs_advanced": [],
        "arcs_closed": [],
        "voice_notes": [],
    }


def normalize_import_chapter(raw: dict[str, Any], fallback: dict[str, str]) -> dict[str, Any]:
    item = dict(raw or {})
    item["id"] = pad_id(item.get("id") or fallback["id"])
    item["title"] = str(item.get("title") or fallback.get("title") or f"Chapter {int(item['id'])}")
    item["summary"] = short_note(item.get("summary"), 1200) or first_sentences(fallback.get("content", ""), 5) or "Summary needs review."
    normalized_characters = []
    for character in listify(item.get("characters")):
        if not isinstance(character, dict):
            continue
        name = short_note(character.get("name"), 90)
        if not name:
            continue
        normalized_characters.append(
            {
                "name": name,
                "role": short_note(character.get("role")) or "unclear from imported chapters",
                "voice": short_note(character.get("voice")) or "unclear from imported chapters",
                "goals": short_note(character.get("goals")) or "unclear from imported chapters",
                "fears": short_note(character.get("fears")) or "unclear from imported chapters",
                "relationships": unique_notes(character.get("relationships") or []),
                "current_state": short_note(character.get("current_state")) or "unclear from imported chapters",
                "evidence": unique_notes(character.get("evidence") or []),
            }
        )
    if not normalized_characters:
        normalized_characters = local_chapter_analysis(fallback)["characters"]
    item["characters"] = normalized_characters
    item["world"] = unique_notes(item.get("world") or [], 30)
    item["events"] = [event for event in listify(item.get("events")) if isinstance(event, dict)]
    if not item["events"]:
        item["events"] = local_chapter_analysis(fallback)["events"]
    item["arcs_opened"] = unique_notes(item.get("arcs_opened") or [], 20)
    item["arcs_advanced"] = unique_notes(item.get("arcs_advanced") or [], 20)
    item["arcs_closed"] = unique_notes(item.get("arcs_closed") or [], 20)
    item["voice_notes"] = unique_notes(item.get("voice_notes") or [], 20)
    return item


def analyze_import_batch(title: str, batch: list[dict[str, str]]) -> tuple[list[dict[str, Any]], list[str]]:
    prompt = build_import_batch_prompt(title, batch)
    warnings: list[str] = []
    try:
        response = import_message(prompt)
        parsed = json.loads(extract_json(response))
        raw_chapters = parsed.get("chapters") if isinstance(parsed, dict) else None
        if not isinstance(raw_chapters, list):
            raise ValueError("Import analysis did not return a chapters array.")
    except Exception as exc:
        warnings.append(f"Claude import analysis failed for chapters {batch[0]['id']}-{batch[-1]['id']}: {exc}")
        return [local_chapter_analysis(chapter) for chapter in batch], warnings

    by_id = {pad_id(chapter.get("id") or ""): chapter for chapter in raw_chapters if isinstance(chapter, dict)}
    normalized: list[dict[str, Any]] = []
    for fallback in batch:
        raw = by_id.get(fallback["id"])
        if raw is None:
            warnings.append(f"Claude import analysis omitted Chapter {fallback['id']}; local fallback filled it.")
            normalized.append(local_chapter_analysis(fallback))
        else:
            normalized.append(normalize_import_chapter(raw, fallback))
    return normalized, warnings


def build_import_canon_from_analyses(title: str, chapters: list[dict[str, str]], analyses: list[dict[str, Any]]) -> dict[str, str]:
    characters: dict[str, dict[str, list[str]]] = {}
    chapter_titles = {pad_id(index): chapter.get("title") or f"Chapter {index}" for index, chapter in enumerate(chapters, start=1)}
    for analysis in analyses:
        cid = pad_id(analysis.get("id") or "")
        for character in analysis.get("characters") or []:
            name = short_note(character.get("name"), 90)
            if not name:
                continue
            entry = characters.setdefault(
                name,
                {"role": [], "voice": [], "goals": [], "fears": [], "relationships": [], "current_state": [], "evidence": [], "chapters": []},
            )
            entry["chapters"].append(str(int(cid)) if cid.isdigit() else cid)
            for field in ("role", "voice", "goals", "fears", "current_state"):
                value = short_note(character.get(field))
                if value and "unclear from this chapter" not in value.lower():
                    entry[field].append(f"Ch {int(cid) if cid.isdigit() else cid}: {value}")
            for rel in unique_notes(character.get("relationships") or []):
                entry["relationships"].append(f"Ch {int(cid) if cid.isdigit() else cid}: {rel}")
            for evidence in unique_notes(character.get("evidence") or []):
                entry["evidence"].append(f"Ch {int(cid) if cid.isdigit() else cid}: {evidence}")

    character_md = ["# Characters\n"]
    for name in sorted(characters):
        entry = characters[name]
        seen_chapters = ", ".join(unique_notes(entry["chapters"], 30))
        character_md.append(
            f"## {name}\n"
            f"- Appears in: {seen_chapters or 'Imported chapters'}\n"
            f"- Role:\n{bullet_lines(entry['role'])}\n"
            f"- Voice:\n{bullet_lines(entry['voice'])}\n"
            f"- Goals:\n{bullet_lines(entry['goals'])}\n"
            f"- Fears:\n{bullet_lines(entry['fears'])}\n"
            f"- Relationships:\n{bullet_lines(entry['relationships'])}\n"
            f"- Current state:\n{bullet_lines(entry['current_state'])}\n"
            f"- Evidence:\n{bullet_lines(entry['evidence'])}\n"
        )

    if len(character_md) == 1:
        character_md.append(DEFAULT_CANON["characters"].replace("# Characters\n\n", ""))

    world_notes: list[str] = []
    voice_notes: list[str] = []
    opened: list[str] = []
    advanced: list[str] = []
    closed: list[str] = []
    timeline = ["# Timeline\n"]
    for analysis in analyses:
        cid = pad_id(analysis.get("id") or "")
        title_text = analysis.get("title") or chapter_titles.get(cid) or f"Chapter {int(cid)}"
        timeline.append(f"## Chapter {int(cid) if cid.isdigit() else cid}: {title_text}\n\n{analysis.get('summary') or 'Summary needs review.'}\n")
        events = analysis.get("events") or []
        if events:
            timeline.append("### Events\n")
            for event in events:
                if not isinstance(event, dict):
                    continue
                when = short_note(event.get("when"), 90) or f"Chapter {int(cid) if cid.isdigit() else cid}"
                where = short_note(event.get("where"), 120)
                what = short_note(event.get("what"), 360)
                people = ", ".join(unique_notes(event.get("characters") or [], 10))
                detail = what or analysis.get("summary") or "Event needs review."
                suffix = []
                if where:
                    suffix.append(f"where: {where}")
                if people:
                    suffix.append(f"people: {people}")
                timeline.append(f"- {when}: {detail}{' (' + '; '.join(suffix) + ')' if suffix else ''}\n")
        for note in analysis.get("world") or []:
            world_notes.append(f"Ch {int(cid) if cid.isdigit() else cid}: {note}")
        for note in analysis.get("voice_notes") or []:
            voice_notes.append(f"Ch {int(cid) if cid.isdigit() else cid}: {note}")
        opened.extend(f"Ch {int(cid) if cid.isdigit() else cid}: {note}" for note in analysis.get("arcs_opened") or [])
        advanced.extend(f"Ch {int(cid) if cid.isdigit() else cid}: {note}" for note in analysis.get("arcs_advanced") or [])
        closed.extend(f"Ch {int(cid) if cid.isdigit() else cid}: {note}" for note in analysis.get("arcs_closed") or [])

    world = (
        "# World\n\n"
        "## Setting / Continuity Details\n"
        f"{bullet_lines(world_notes)}\n\n"
        "## Rules\n- Review imported details above and promote any hard rules here.\n\n"
        "## Factions\n- Review imported details above and promote any factions/institutions here.\n\n"
        "## History\n- Review imported details above and promote any backstory/history here.\n"
    )
    arcs = (
        "# Arcs / Threads\n\n"
        "## Open\n"
        f"{bullet_lines(opened, 'Review imported chapters and list open promises here.')}\n\n"
        "## Advanced\n"
        f"{bullet_lines(advanced, 'No advanced threads were clearly extracted.')}\n\n"
        "## Closed\n"
        f"{bullet_lines(closed, 'No closed threads were clearly extracted.')}\n"
    )
    voice_profile = (
        "# Voice Profile\n\n"
        "Imported voice observations. Edit this into the target style before drafting new chapters.\n\n"
        f"{bullet_lines(voice_notes, 'Voice still needs a user-authored descriptor. Use the voice generator or edit this paragraph.')}\n"
    )
    return {
        "world": world,
        "characters": "\n".join(character_md),
        "timeline": "\n".join(timeline),
        "arcs": arcs,
        "voice_profile": voice_profile,
    }


def staged_import(raw: str, title: str) -> dict[str, Any]:
    chapters = split_markdown_chapters(raw)
    if not chapters:
        chapters = [{"title": "Chapter 1", "content": raw}]
    prepared_chapters = [
        {"id": pad_id(index), "title": chapter.get("title") or f"Chapter {index}", "content": chapter.get("content", "")}
        for index, chapter in enumerate(chapters, start=1)
    ]
    analyses: list[dict[str, Any]] = []
    warnings: list[str] = []
    for batch in import_batches(prepared_chapters):
        batch_analysis, batch_warnings = analyze_import_batch(title, batch)
        analyses.extend(batch_analysis)
        warnings.extend(batch_warnings)
    analyses_by_id = {pad_id(item.get("id") or ""): item for item in analyses}
    ordered_analyses = [analyses_by_id.get(chapter["id"]) or local_chapter_analysis(chapter) for chapter in prepared_chapters]
    canon = build_import_canon_from_analyses(title, prepared_chapters, ordered_analyses)
    return {
        "title": title or "Imported Book",
        "chapters": [
            {"id": chapter["id"], "title": chapter["title"], "content": chapter["content"]}
            for chapter in prepared_chapters
        ],
        "canon": canon,
        "mode": "staged-claude-import" if real_backend_active() else "local-fallback",
        "importDiagnostics": {
            "chapterCount": len(prepared_chapters),
            "batchCount": len(import_batches(prepared_chapters)),
            "warnings": warnings,
        },
    }


def mock_plan(riff: str, transcript: str) -> str:
    intent = (riff or transcript or "The user has not given much direction yet.").strip()
    short = re.sub(r"\s+", " ", intent)[:900]
    questions = []
    if len(intent) < 280:
        questions = [
            "What must be different by the final page of this chapter?",
            "Which character should leave the scene with the most power?",
            "Is the ending a reveal, a reversal, or a wound?",
        ]
    else:
        questions = [
            "What is the one fact the reader should understand before the chapter ends?",
            "Who is hiding something in this scene?",
        ]
    return (
        "## Clarifying Questions\n"
        + "\n".join(f"{i + 1}. {question}" for i, question in enumerate(questions))
        + "\n\n## Chapter Outline\n"
        f"1. Open on a concrete disturbance tied to the user's riff: {short}\n"
        "2. Put the central characters in direct contact quickly. Let dialogue carry pressure instead of explanation.\n"
        "3. Reveal or withhold the important object, secret, or decision through action.\n"
        "4. Force a choice that changes the relationship or the tactical situation.\n"
        "5. End on consequence, not summary: someone knows more, loses cover, or commits to a dangerous next step.\n"
    )


def mock_draft(slug: str, chapter_id: str, plan: str, transcript: str) -> Iterable[str]:
    data = load_book(slug)
    chapter = next((c for c in data["chapters"] if c["id"] == pad_id(chapter_id)), None)
    title = chapter.get("title", f"Chapter {int(chapter_id)}") if chapter else f"Chapter {int(chapter_id)}"
    intent = re.sub(r"\s+", " ", (transcript or plan or "A dangerous meeting changes the story.")).strip()
    intent = intent[:1200]
    pieces = [
        f"# {title}\n\n",
        "Rain came down hard enough to make every surface tell on itself.\n\n",
        "The street showed its seams. Gutters rattled. Windows held their yellow squares of light and gave nothing back. ",
        "Somewhere beyond the nearest corner, someone laughed once and then thought better of it.\n\n",
        "The plan had sounded simple when it was only talk: arrive late, ask the question, keep the blade out of sight until the answer made it necessary. ",
        "Simple plans had a way of becoming religious objects. People carried them for comfort after they had stopped being useful.\n\n",
        f"What mattered tonight was this: {intent}\n\n",
        "She kept her hand low, fingers loose, the way she had practiced. Not brave. Practiced. There was a difference, and she trusted the second one more.\n\n",
        '"You picked a miserable place for a confession," he said.\n\n',
        '"I picked a place where nobody lingers."\n\n',
        "He looked past her shoulder. That told her enough. The meeting had brought more than the two of them. ",
        "The old fear moved in her, familiar as a tenant, but it did not get the chair this time.\n\n",
        "When the door opened behind them, she did not turn. She watched his face instead. Faces were weather. They told you what had arrived before the sky did.\n\n",
        "He smiled badly. \"You still have it.\"\n\n",
        "\"Since chapter four,\" she said, though there was no joke in it. \"You should have counted your ghosts better.\"\n\n",
        "That was when the night changed shape. Not loudly. Loudly was for amateurs and men who needed witnesses. ",
        "This change was small: his right foot sliding back half an inch, her thumb finding the hidden catch, the person behind her drawing one careful breath.\n\n",
        "She had spent too long being moved around by other people's secrets. Tonight she let one of her own into the room.\n\n",
    ]
    for piece in pieces:
        yield piece
        time.sleep(0.015)


def mock_revise(chapter: str, request_text: str) -> Iterable[str]:
    revised = chapter
    lower = request_text.lower()
    if "cut" in lower:
        target = request_text.lower().split("cut", 1)[1].strip().strip(".")
        words = [w for w in re.findall(r"[a-zA-Z]+", target) if len(w) > 3][:4]
        paragraphs = []
        for paragraph in revised.split("\n\n"):
            paragraph_lower = paragraph.lower()
            if words and sum(1 for word in words if word.lower() in paragraph_lower) >= max(1, min(2, len(words))):
                continue
            paragraphs.append(paragraph)
        revised = "\n\n".join(paragraphs)
    if "sharper" in lower or "dialog" in lower or "dialogue" in lower:
        revised = revised.replace('"I picked a place where nobody lingers."', '"Nobody lingers here."')
        revised = revised.replace('"You still have it."', '"You kept it."')
    if "defiant" in lower:
        revised = revised.rstrip() + "\n\nShe did not step back. Let them count that, too.\n"
    if revised == chapter:
        revised = chapter.rstrip() + f"\n\n[Local revision note: apply this request with Claude when the API key is available: {request_text.strip()}]\n"
    for i in range(0, len(revised), 180):
        yield revised[i : i + 180]
        time.sleep(0.01)


def mock_recap(slug: str, recap_type: str, section_id: str | None = None) -> str:
    data = load_book(slug)
    locked = [c for c in data["chapters"] if c.get("status") == "locked" and c.get("content", "").strip()]
    last = locked[-1] if locked else (data["chapters"][-1] if data["chapters"] else None)
    arcs = data["canon"]["arcs"]["content"].strip()
    if recap_type == "section" and section_id:
        selected = [c for c in data["chapters"] if c.get("section") == section_id and c.get("content", "").strip()]
        titles = ", ".join(c["title"] for c in selected) or "No chapters in this section yet"
        return f"## Section Recap\n\nChapters included: {titles}.\n\nOpen threads from canon:\n\n{arcs[:1200]}"
    if not last:
        return "## Where We Were\n\nNo chapters exist yet. Create or import chapters, then start planning the next one."
    return (
        "## Where We Were\n\n"
        f"Last active chapter: **{last['title']}** ({last['id']}).\n\n"
        f"Opening memory from the chapter:\n\n{first_sentences(last['content'], 3) or 'The chapter needs a summary.'}\n\n"
        "## Open Arcs\n\n"
        f"{arcs[:1600]}"
    )


def mock_continuity(slug: str, chapter_id: str | None) -> str:
    data = load_book(slug)
    chapters = data["chapters"]
    target = next((c for c in chapters if c["id"] == pad_id(chapter_id or data["meta"].get("activeChapter", "01"))), None)
    if not target:
        return "## Continuity Check\n\nNo target chapter found."
    names = extract_names(target.get("content", ""))
    canon_chars = data["canon"]["characters"]["content"]
    flags = []
    for name in names:
        if name not in canon_chars:
            flags.append(f"- Potential untracked character or place: **{name}** appears in the chapter but not in canon/characters.md.")
    if "tomorrow" in target.get("content", "").lower() and "yesterday" in target.get("content", "").lower():
        flags.append("- The chapter uses both tomorrow and yesterday. Check the local timeline around this scene.")
    if not flags:
        flags.append("- No heuristic issues found. With a Claude key, this check will read the full canon and produce deeper continuity flags.")
    return "## Continuity Check\n\n" + "\n".join(flags)


def format_conversation_for_prompt(conversation: dict[str, Any], limit: int = 24) -> str:
    messages = conversation.get("messages", [])[-limit:]
    lines = []
    for message in messages:
        role = message.get("role", "user")
        speaker = "AI" if role == "assistant" else "System note" if role == "system" else "User"
        content = str(message.get("content", "")).strip()
        if content:
            lines.append(f"{speaker}:\n{content}")
    return "\n\n---\n\n".join(lines)


def build_chat_prompt(slug: str, user_message: str) -> str:
    data = load_book(slug)
    conversation = load_conversation(slug)
    saved = [c for c in data["chapters"] if c.get("content", "").strip()]
    chapter_lines = "\n".join(
        f"- {c['id']}: {c['title']} ({c.get('status', 'draft')}, {c.get('wordCount', 0)} words)"
        for c in saved
    ) or "- No saved chapter prose yet."
    next_id = next_chapter_id(slug)
    if data["meta"].get("projectType") == "research":
        research = data.get("research") or {}
        low_claims = [
            c
            for c in research.get("claims", [])
            if c.get("publish_readiness") in {"blocked", "needs_review"} or c.get("confidence") in {"unsourced", "contradicted"}
        ][:20]
        return (
            "You are inside the Generation Engine chat thread for a research nonfiction project. Respond naturally to the latest user message, but obey the research-mode hard rules.\n\n"
            "If the user asks what we know, answer from sources and claims with attribution. If the user asks what is missing, produce a research punch list. "
            "If the user asks for a full chapter draft or full revised draft, write the full draft and use [[claim:claim-id]] markers on factual sentences. "
            "If no claim supports a factual sentence, mark it [UNSOURCED: ...]. End full chapter drafts with the GE_DRAFT HTML marker. "
            f"The default suggested_id for a new chapter is {next_id} unless the user specifies another chapter.\n\n"
            "Saved chapters:\n"
            f"{chapter_lines}\n\n"
            "Claims needing attention:\n"
            f"{json.dumps(low_claims, indent=2, ensure_ascii=False) if low_claims else 'None flagged yet.'}\n\n"
            "Recent conversation:\n"
            f"{format_conversation_for_prompt(conversation)}\n\n"
            "Latest user message:\n"
            f"{user_message}"
        )
    return (
        "You are inside the Generation Engine chat thread. Respond naturally to the latest user message.\n\n"
        "If the user asks for a full chapter draft or full revised draft, write the full draft in this message and end with the GE_DRAFT HTML marker. "
        f"The default suggested_id for a new chapter is {next_id} unless the user specifies another chapter.\n\n"
        "If the user is brainstorming, do not draft yet. Ask sharp questions or propose direction conversationally.\n\n"
        "Saved chapters:\n"
        f"{chapter_lines}\n\n"
        "Recent conversation:\n"
        f"{format_conversation_for_prompt(conversation)}\n\n"
        "Latest user message:\n"
        f"{user_message}"
    )


def find_last_draft_in_conversation(slug: str) -> str:
    for message in reversed(load_conversation(slug).get("messages", [])):
        if message.get("role") != "assistant":
            continue
        content, marker = extract_draft_marker(str(message.get("content", "")))
        if marker:
            return content
    return ""


def active_chapter_text(slug: str) -> str:
    data = load_book(slug)
    cid = data["meta"].get("activeChapter", "01")
    chapter = next((c for c in data["chapters"] if c["id"] == cid), None)
    return chapter.get("content", "") if chapter else ""


def mock_chat_stream(slug: str, user_message: str) -> Iterable[str]:
    lower = user_message.lower()
    if any(word in lower for word in ("draft", "write chapter", "chapter one", "chapter 1", "next chapter")):
        match = re.search(r"chapter\s+(\d+|one|two|three|four|five|six|seven|eight|nine|ten)", lower)
        words = {"one": "01", "two": "02", "three": "03", "four": "04", "five": "05", "six": "06", "seven": "07", "eight": "08", "nine": "09", "ten": "10"}
        suggested = next_chapter_id(slug)
        if match:
            raw = match.group(1)
            suggested = words.get(raw, pad_id(raw))
        body = "".join(mock_draft(slug, suggested, user_message, user_message)).rstrip()
        text = body + f'\n\n<!-- GE_DRAFT {{"draft_for":"chapter","suggested_id":"{suggested}","title":"Chapter {int(suggested)}"}} -->'
    elif any(word in lower for word in ("rewrite", "revise", "tighten", "sharper", "angrier", "last paragraph")):
        source = find_last_draft_in_conversation(slug) or active_chapter_text(slug)
        suggested = load_meta(book_dir(slug)).get("activeChapter", "01")
        text = "".join(mock_revise(source, user_message)).rstrip()
        text += f'\n\n<!-- GE_DRAFT {{"draft_for":"chapter","suggested_id":"{pad_id(suggested)}","title":"Chapter {int(pad_id(suggested))}"}} -->'
    elif "where were we" in lower or "recap" in lower:
        text = mock_recap(slug, "session")
    elif "continuity" in lower or "check chapter" in lower:
        match = re.search(r"chapter\s+(\d+)", lower)
        text = mock_continuity(slug, match.group(1) if match else None)
    elif "read me" in lower or "show me chapter" in lower:
        match = re.search(r"chapter\s+(\d+)", lower)
        cid = pad_id(match.group(1)) if match else load_meta(book_dir(slug)).get("activeChapter", "01")
        data = load_book(slug)
        chapter = next((c for c in data["chapters"] if c["id"] == cid), None)
        text = f"Here is **{chapter['title']}**:\n\n> " + (chapter.get("content", "") if chapter else "I could not find that chapter.").replace("\n", "\n> ")
    else:
        short = re.sub(r"\s+", " ", user_message).strip()
        text = (
            "I’m with you. That gives us a useful pressure point.\n\n"
            f"What I’m hearing is: {short[:420] or 'you want to find the next turn of the story'}.\n\n"
            "Before I draft, I need one thing: what should be different by the end of the scene? "
            "A secret exposed, a relationship damaged, or a decision made?"
        )
    for i in range(0, len(text), 160):
        yield text[i : i + 160]
        time.sleep(0.012)


def chat_stream(slug: str, user_message: str) -> Iterable[str]:
    if real_backend_active():
        return ai_stream(slug, build_chat_prompt(slug, user_message), max_tokens=14000)
    preflight = get_preflight()

    def transparent_unavailable_stream() -> Iterable[str]:
        yield (
            "Claude is not connected, so I am not going to fake an AI response.\n\n"
            f"Current backend: {preflight.active_mode}. {preflight.status_message}\n\n"
            "Open the mode pill in the header and choose CLI or API after Claude is available."
        )

    return transparent_unavailable_stream()


def ensure_session_welcome(slug: str) -> dict[str, Any]:
    conversation = load_conversation(slug)
    today = time.strftime("%Y-%m-%d")
    for message in reversed(conversation.get("messages", [])[-12:]):
        metadata = message.get("metadata") or {}
        if metadata.get("kind") == "session-recap" and str(message.get("createdAt", "")).startswith(today):
            return message
    if real_backend_active():
        try:
            content = ai_message(
                slug,
                "Write a concise welcome-back recap for the user opening this book. Include last saved/locked chapter and current open arcs. Do not ask more than one question.",
                max_tokens=1400,
                purpose="utility",
            )
        except Exception:
            content = mock_recap(slug, "session")
    else:
        content = mock_recap(slug, "session")
    return append_conversation_message(slug, "assistant", f"Welcome back.\n\n{content}", {"kind": "session-recap"})


def save_chat_draft_as_chapter(slug: str, payload: dict[str, Any]) -> dict[str, Any]:
    content = str(payload.get("content") or "").strip()
    if not content:
        raise ValueError("Draft content is empty.")
    content, marker = extract_draft_marker(content)
    requested = payload.get("chapterId") or (marker or {}).get("suggested_id") or next_chapter_id(slug)
    cid = pad_id(requested)
    title = str(payload.get("title") or (marker or {}).get("title") or clean_title_from_markdown(content, f"Chapter {int(cid)}"))
    replace = bool(payload.get("replace"))
    book = book_dir(slug)
    meta = load_meta(book)
    existing = next((chapter for chapter in meta.get("chapters", []) if chapter["id"] == cid), None)
    if existing and not replace:
        current = read_text(chapter_path(book, cid))
        if not chapter_is_blank(current):
            cid = next_chapter_id(slug)
            title = clean_title_from_markdown(content, f"Chapter {int(cid)}")
            existing = None
    write_text(chapter_path(book, cid), content.rstrip() + "\n")
    write_text(chapter_plan_path(book, cid), read_text(chapter_plan_path(book, cid)))
    write_text(chapter_transcript_path(book, cid), read_text(chapter_transcript_path(book, cid)))
    section = meta.get("sections", [{"id": "part-1"}])[0].get("id", "part-1")
    if existing:
        existing["title"] = title
        existing["status"] = "locked"
        existing.setdefault("section", section)
    else:
        meta["chapters"].append({"id": cid, "title": title, "status": "locked", "section": section})
    meta["activeChapter"] = cid
    save_meta(book, meta)
    append_conversation_message(slug, "system", f"Saved {title} as Chapter {int(cid)}.", {"kind": "chapter-save", "chapterId": cid})
    return load_book(slug)


def find_whisper_executable() -> Path | None:
    candidates = [
        WHISPER_DIR / "whisper-cli.exe",
        WHISPER_DIR / "main.exe",
        WHISPER_DIR / "whisper.exe",
        WHISPER_DIR / "bin" / "whisper-cli.exe",
        WHISPER_DIR / "Release" / "whisper-cli.exe",
        WHISPER_DIR / "Release" / "main.exe",
    ]
    for candidate in candidates:
        if candidate.exists():
            return candidate
    found = shutil.which("whisper-cli") or shutil.which("whisper")
    return Path(found) if found else None


def find_whisper_model() -> Path | None:
    for name in ("ggml-base.en.bin", "ggml-small.en.bin", "ggml-tiny.en.bin", "ggml-base.bin"):
        candidate = WHISPER_DIR / "models" / name
        if candidate.exists():
            return candidate
    return None


def parse_multipart_file(content_type: str, raw: bytes) -> tuple[bytes, str]:
    match = re.search(r"boundary=(?P<boundary>[^;]+)", content_type)
    if not match:
        raise ValueError("Missing multipart boundary.")
    boundary = match.group("boundary").strip('"')
    delimiter = ("--" + boundary).encode("utf-8")
    for part in raw.split(delimiter):
        if b"Content-Disposition" not in part:
            continue
        header_blob, _, body = part.partition(b"\r\n\r\n")
        if not body:
            header_blob, _, body = part.partition(b"\n\n")
        headers = header_blob.decode("utf-8", errors="replace")
        if 'name="audio"' not in headers:
            continue
        filename_match = re.search(r'filename="([^"]+)"', headers)
        filename = filename_match.group(1) if filename_match else "recording.wav"
        return body.strip(b"\r\n-"), filename
    raise ValueError("No audio file field was uploaded.")


def parse_multipart_form(content_type: str, raw: bytes) -> tuple[dict[str, str], dict[str, dict[str, Any]]]:
    match = re.search(r"boundary=(?P<boundary>[^;]+)", content_type)
    if not match:
        raise ValueError("Missing multipart boundary.")
    boundary = match.group("boundary").strip('"')
    delimiter = ("--" + boundary).encode("utf-8")
    fields: dict[str, str] = {}
    files: dict[str, dict[str, Any]] = {}
    for part in raw.split(delimiter):
        if b"Content-Disposition" not in part:
            continue
        header_blob, _, body = part.partition(b"\r\n\r\n")
        if not body:
            header_blob, _, body = part.partition(b"\n\n")
        headers = header_blob.decode("utf-8", errors="replace")
        name_match = re.search(r'name="([^"]+)"', headers)
        if not name_match:
            continue
        name = name_match.group(1)
        filename_match = re.search(r'filename="([^"]*)"', headers)
        content_type_match = re.search(r"(?im)^Content-Type:\s*([^\r\n]+)", headers)
        payload = body.rstrip(b"\r\n-")
        if filename_match and filename_match.group(1):
            files[name] = {
                "filename": filename_match.group(1),
                "content_type": content_type_match.group(1).strip() if content_type_match else "",
                "content": payload,
            }
        else:
            fields[name] = payload.decode("utf-8", errors="replace")
    return fields, files


def transcribe_audio_file(audio_path: Path) -> str:
    exe = find_whisper_executable()
    model = find_whisper_model()
    if not exe or not model:
        raise RuntimeError(
            "Local Whisper is not installed. Add whisper.cpp to runtime/whisper and a ggml model to runtime/whisper/models."
        )
    output_base = audio_path.with_suffix("")
    command = [
        str(exe),
        "-m",
        str(model),
        "-f",
        str(audio_path),
        "-otxt",
        "-of",
        str(output_base),
        "-nt",
    ]
    completed = subprocess.run(
        command,
        capture_output=True,
        text=True,
        timeout=300,
        encoding="utf-8",
        errors="replace",
        **hidden_subprocess_kwargs(),
    )
    if completed.returncode != 0:
        raise RuntimeError((completed.stderr or completed.stdout or "Whisper transcription failed.").strip())
    transcript_path = output_base.with_suffix(".txt")
    text = read_text(transcript_path).strip()
    if not text:
        text = (completed.stdout or "").strip()
    return text.strip()


def create_record_request(slug: str, payload: dict[str, Any]) -> dict[str, Any]:
    book = book_dir(slug)
    ensure_research_dirs(book)
    root = research_dir(book) / "records"
    record_id = next_research_id(root, "req")
    agency = str(payload.get("agency") or "Records Custodian").strip()
    jurisdiction = str(payload.get("jurisdiction") or "local").strip().lower()
    state = str(payload.get("state") or "").strip().upper()
    subject = str(payload.get("subject") or payload.get("target") or "records request").strip()
    details = str(payload.get("details") or payload.get("description") or "").strip()
    submitted_at = str(payload.get("submitted_at") or "").strip()
    statute = statute_for_request(jurisdiction, state)
    deadline = ""
    if submitted_at:
        deadline = (parse_date(submitted_at) + timedelta(days=30)).strftime("%Y-%m-%d")
    prompt = (
        "Draft a public-records request letter. The user will edit and send manually. "
        "Use the statute if appropriate, stay polite and specific, and do not invent case numbers.\n\n"
        f"Agency: {agency}\nJurisdiction: {jurisdiction}\nState: {state}\nStatute: {statute}\nSubject: {subject}\nDetails: {details}"
    )
    if real_backend_active():
        try:
            body = ai_message(slug, prompt, max_tokens=1800, purpose="utility")
        except Exception:
            body = fallback_record_letter(agency, statute, subject, details)
    else:
        body = fallback_record_letter(agency, statute, subject, details)
    meta = {
        "id": record_id,
        "status": "drafted",
        "agency": agency,
        "jurisdiction": jurisdiction,
        "statute": statute,
        "submitted_at": submitted_at,
        "acknowledged_at": "",
        "deadline": deadline,
        "response_source_refs": [],
        "fee_paid": 0.0,
        "notes": "",
    }
    write_text(root / f"{record_id}.md", frontmatter_markdown(meta, body))
    return load_research(slug)


def statute_for_request(jurisdiction: str, state: str) -> str:
    if jurisdiction == "federal":
        return "Freedom of Information Act, 5 U.S.C. Section 552"
    state_map = {
        "CA": "California Public Records Act, Cal. Gov. Code § 7920.000 et seq.",
        "WA": "Washington Public Records Act, RCW 42.56",
        "NY": "New York Freedom of Information Law, N.Y. Pub. Off. Law art. 6",
        "FL": "Florida Public Records Law, Fla. Stat. ch. 119",
        "TX": "Texas Public Information Act, Tex. Gov't Code ch. 552",
    }
    return state_map.get(state, "Applicable state/local public records law")


def fallback_record_letter(agency: str, statute: str, subject: str, details: str) -> str:
    return (
        f"To the Records Custodian,\n\n"
        f"Under {statute}, I request records concerning {subject}.\n\n"
        f"Requested records:\n{details or '- Incident reports, supplemental reports, call logs, dispatch records, investigative summaries, and releasable correspondence related to the subject.'}\n\n"
        "Please provide records electronically where possible. If any portion is withheld, please cite the specific exemption and release all segregable non-exempt portions.\n\n"
        "Sincerely,\n"
    )


def parse_date(value: str) -> datetime:
    try:
        return datetime.strptime(value[:10], "%Y-%m-%d")
    except Exception:
        return datetime.now()


def update_record_request(slug: str, record_id: str, payload: dict[str, Any]) -> dict[str, Any]:
    book = book_dir(slug)
    path = research_dir(book) / "records" / f"{record_id}.md"
    meta, body = parse_frontmatter(read_text(path))
    for key in ("status", "agency", "jurisdiction", "statute", "submitted_at", "acknowledged_at", "deadline", "response_source_refs", "fee_paid", "notes"):
        if key in payload:
            meta[key] = payload[key]
    if "body" in payload:
        body = str(payload.get("body") or "")
    if meta.get("submitted_at") and not meta.get("deadline"):
        meta["deadline"] = (parse_date(str(meta["submitted_at"])) + timedelta(days=30)).strftime("%Y-%m-%d")
    write_text(path, frontmatter_markdown(meta, body))
    return load_research(slug)


def research_missing_report(slug: str) -> dict[str, Any]:
    research = load_research(slug)
    claims = research["claims"]
    people = research["people"]
    records = research["records"]
    low_claims = [c for c in claims if c.get("confidence") in {"unsourced", "single-sourced", "contradicted"} or c.get("publish_readiness") in {"blocked", "needs_review"}]
    unsourced_people = [p for p in people if not p.get("source_refs")]
    contradictions = [c for c in claims if c.get("contradicting_sources")]
    overdue = []
    today = datetime.now().date()
    for record in records:
        deadline = str(record.get("deadline") or "")
        if deadline and str(record.get("status")) in {"sent", "acknowledged"}:
            try:
                if datetime.strptime(deadline[:10], "%Y-%m-%d").date() < today:
                    overdue.append(record)
            except Exception:
                pass
    markdown = ["## What's Missing"]
    markdown.append(f"- Low-confidence or blocked claims: {len(low_claims)}")
    markdown.append(f"- Entities without source refs: {len(unsourced_people)}")
    markdown.append(f"- Claims with contradictions: {len(contradictions)}")
    markdown.append(f"- Overdue records requests: {len(overdue)}")
    if low_claims[:8]:
        markdown.append("\n### Claims to strengthen")
        markdown.extend(f"- {c.get('id')}: {c.get('text')} ({c.get('publish_readiness')})" for c in low_claims[:8])
    if overdue:
        markdown.append("\n### Overdue records")
        markdown.extend(f"- {r.get('id')}: {r.get('agency')} due {r.get('deadline')}" for r in overdue)
    markdown.append("\n### Suggested next moves")
    markdown.append("- Add primary records for every allegation involving a living person.")
    markdown.append("- Resolve contradictions before drafting clean export prose.")
    markdown.append("- Turn each major timeline event into at least one sourced claim.")
    return {"markdown": "\n".join(markdown), "lowClaims": low_claims, "unsourcedPeople": unsourced_people, "contradictions": contradictions, "overdueRecords": overdue}


def research_export_review(slug: str) -> dict[str, Any]:
    data = load_book(slug)
    research = data.get("research") or {}
    claims = {claim.get("id"): claim for claim in research.get("claims", [])}
    manuscript = "\n\n".join(chapter.get("content", "") for chapter in data.get("chapters", []) if chapter.get("content", "").strip())
    claim_refs = re.findall(r"\[\[claim:(claim-\d+)\]\]", manuscript)
    unsourced_markers = re.findall(r"\[UNSOURCED:[^\]]+\]", manuscript)
    blocked = [claims[ref] for ref in sorted(set(claim_refs)) if claims.get(ref, {}).get("publish_readiness") == "blocked"]
    high_risk = []
    for claim in claims.values():
        if claim.get("publish_readiness") in {"blocked", "needs_review"} and any("living_person" in blocker for blocker in claim.get("publish_blockers", [])):
            high_risk.append(claim)
    markdown = (
        "## Pre-export Sourcing Review\n\n"
        f"- Cited assertions found: {len(claim_refs)}\n"
        f"- Unsourced markers in manuscript: {len(unsourced_markers)}\n"
        f"- Cited blocked claims: {len(blocked)}\n"
        f"- High defamation-risk passages/claims needing review: {len(high_risk)}\n\n"
    )
    if unsourced_markers:
        markdown += "### Unsourced markers\n" + "\n".join(f"- {item}" for item in unsourced_markers[:20]) + "\n\n"
    if blocked:
        markdown += "### Blocked cited claims\n" + "\n".join(f"- {claim.get('id')}: {claim.get('text')}" for claim in blocked[:20]) + "\n\n"
    return {"markdown": markdown, "citedCount": len(claim_refs), "unsourced": unsourced_markers, "blocked": blocked, "highRisk": high_risk}


def build_prompt(task: str, payload: dict[str, Any], slug: str, chapter_id: str | None = None) -> str:
    if task == "voice_profile":
        return (
            "Generate one editable prose-style descriptor paragraph from these author preferences. "
            "Describe rhythm, dialogue, internal thought, description density, register, and signature moves. "
            "Do not imitate source authors directly. Output markdown only.\n\n"
            f"User preferences: {payload.get('authors', '')}"
        )
    if task == "import":
        return (
            "Import these existing chapters. Return strict JSON with keys title, canon, chapters. "
            "canon must contain markdown strings for world, characters, timeline, arcs, voice_profile. "
            "chapters must be an array of objects with id, title, content. "
            "Infer only from the supplied text and mark uncertain facts as proposed.\n\n"
            f"Book title: {payload.get('title', '')}\n\nImported text:\n{payload.get('raw', '')}"
        )
    data = load_book(slug)
    active_id = pad_id(chapter_id or data["meta"].get("activeChapter", "01"))
    chapter = next((c for c in data["chapters"] if c["id"] == active_id), None)
    if task == "plan":
        return (
            "The user is riffing on the next chapter. Ask 1-3 sharp clarifying questions only if material stakes are unclear, "
            "then propose a practical chapter outline. Output markdown with 'Clarifying Questions' and 'Chapter Outline'.\n\n"
            f"Target chapter: {active_id}\n\n"
            f"Current riff:\n{payload.get('riff', '')}\n\n"
            f"Existing transcript:\n{payload.get('transcript', '')}"
        )
    if task == "draft":
        return (
            "Draft the full chapter now. Output only the chapter prose. No notes, no headings except the chapter title if appropriate.\n\n"
            f"Target chapter: {active_id}\n\n"
            f"Chapter plan:\n{payload.get('plan') or (chapter or {}).get('plan', '')}\n\n"
            f"Riff transcript:\n{payload.get('transcript') or (chapter or {}).get('transcript', '')}"
        )
    if task == "revise":
        return (
            "Apply the user's requested edit to the current chapter. Preserve unchanged sections verbatim where possible. "
            "Output the complete revised chapter text only.\n\n"
            f"Revision request:\n{payload.get('request', '')}\n\n"
            f"Current chapter:\n{payload.get('chapter', '')}"
        )
    if task == "continuity":
        scope = payload.get("scope", "chapter")
        body = (chapter or {}).get("content", "") if scope == "chapter" else "\n\n".join(c["content"] for c in data["chapters"])
        return (
            "Run a continuity check. Return a concise markdown list of discrepancies only. "
            "Flag named-entity inconsistencies, timeline conflicts, broken or abandoned arcs, and character voice drift. "
            "Do not fix anything.\n\n"
            f"Scope: {scope}\n\nText:\n{body}"
        )
    if task == "recap":
        return (
            "Generate a concise recap for returning to the book. Include last locked chapter, current open arcs, and what seems planned next. "
            "Output markdown.\n\n"
            f"Recap type: {payload.get('type', 'session')}\nSection: {payload.get('sectionId', '')}"
        )
    if task == "lock":
        return (
            "Read the locked chapter candidate and propose canon updates. Return strict JSON with keys world, characters, timeline, arcs, voice_profile. "
            "Each value must be the full replacement markdown for that canon file, not a patch. Preserve existing useful text and append/modify only what changed.\n\n"
            f"Chapter {active_id}:\n{payload.get('chapter', '')}"
        )
    return json.dumps(payload)


def propose_lock_deltas(slug: str, chapter_id: str) -> dict[str, Any]:
    data = load_book(slug)
    target = next((c for c in data["chapters"] if c["id"] == pad_id(chapter_id)), None)
    if not target:
        raise FileNotFoundError("Chapter not found")
    if real_backend_active():
        prompt = build_prompt("lock", {"chapter": target["content"]}, slug, chapter_id)
        raw = ai_message(slug, prompt, max_tokens=8000, chapter_id=chapter_id, purpose="utility")
        try:
            parsed = json.loads(extract_json(raw))
            updates = {name: str(parsed.get(name, data["canon"][name]["content"])) for name in CANON_FILES}
        except Exception:
            updates = local_lock_updates(data, target)
    else:
        updates = local_lock_updates(data, target)
    return {
        "chapterId": pad_id(chapter_id),
        "files": [
            {
                "name": name,
                "old": data["canon"][name]["content"],
                "new": updates.get(name, data["canon"][name]["content"]),
            }
            for name in CANON_FILES
        ],
    }


def local_lock_updates(data: dict[str, Any], target: dict[str, Any]) -> dict[str, str]:
    canon = {name: data["canon"][name]["content"] for name in CANON_FILES}
    chapter_id = target["id"]
    title = target["title"]
    summary = first_sentences(target.get("content", ""), 4) or "Summary needed."
    names = extract_names(target.get("content", ""), 8)
    timeline_entry = f"\n\n## Chapter {int(chapter_id)}: {title}\n{summary}\n"
    if f"## Chapter {int(chapter_id)}:" not in canon["timeline"]:
        canon["timeline"] = canon["timeline"].rstrip() + timeline_entry
    if names:
        existing = canon["characters"]
        additions = []
        for name in names:
            if name not in existing:
                additions.append(
                    f"\n## {name}\n- Role: Introduced or referenced in Chapter {int(chapter_id)}. Review and edit.\n- Voice:\n- Goals:\n- Fears:\n- Relationships:\n- Current state:\n"
                )
        if additions:
            canon["characters"] = existing.rstrip() + "\n" + "\n".join(additions)
    canon["arcs"] = canon["arcs"].rstrip() + f"\n\n## Chapter {int(chapter_id)} Movement\n- Proposed after locking: {summary[:260]}\n"
    return canon


def extract_json(text: str) -> str:
    text = text.strip()
    if text.startswith("```"):
        text = re.sub(r"^```(?:json)?", "", text).strip()
        text = re.sub(r"```$", "", text).strip()
    start = text.find("{")
    end = text.rfind("}")
    if start >= 0 and end > start:
        return text[start : end + 1]
    return text


def cache_key_for_recap(slug: str, payload: dict[str, Any]) -> str:
    data = load_book(slug)
    material = json.dumps(
        {
            "type": payload.get("type", "session"),
            "sectionId": payload.get("sectionId"),
            "canon": {k: v["content"] for k, v in data["canon"].items()},
            "chapters": [
                {
                    "id": c["id"],
                    "status": c.get("status"),
                    "section": c.get("section"),
                    "hash": sha_text(c.get("content", "")),
                }
                for c in data["chapters"]
            ],
        },
        sort_keys=True,
    )
    return sha_text(material)


class GenerationHandler(SimpleHTTPRequestHandler):
    server_version = "GenerationEngine/3"
    protocol_version = "HTTP/1.0"
    extensions_map = {
        **SimpleHTTPRequestHandler.extensions_map,
        ".html": "text/html; charset=utf-8",
        ".js": "application/javascript; charset=utf-8",
        ".css": "text/css; charset=utf-8",
        ".json": "application/json; charset=utf-8",
    }

    def translate_path(self, path: str) -> str:
        parsed = urllib.parse.urlparse(path)
        clean = parsed.path.lstrip("/") or "index.html"
        safe = Path(clean)
        if any(part == ".." for part in safe.parts):
            return str(WEB_DIR / "index.html")
        target = WEB_DIR / safe
        if target.is_dir():
            target = target / "index.html"
        if not target.exists() and not parsed.path.startswith("/api/"):
            target = WEB_DIR / "index.html"
        return str(target)

    def log_message(self, fmt: str, *args: Any) -> None:
        console_log("%s - %s" % (self.address_string(), fmt % args))

    def read_json_body(self) -> dict[str, Any]:
        length = int(self.headers.get("content-length", "0") or "0")
        if not length:
            return {}
        raw = self.rfile.read(length).decode("utf-8")
        return json.loads(raw or "{}")

    def send_json(self, data: Any, status: int = 200) -> None:
        payload = json.dumps(data, ensure_ascii=False).encode("utf-8")
        self.send_response(status)
        self.send_header("content-type", "application/json; charset=utf-8")
        self.send_header("content-length", str(len(payload)))
        self.send_header("cache-control", "no-store")
        self.end_headers()
        self.wfile.write(payload)

    def send_error_json(self, status: int, message: str, details: str | None = None) -> None:
        self.send_json({"error": message, "details": details}, status=status)

    def do_GET(self) -> None:
        try:
            parsed = urllib.parse.urlparse(self.path)
            if parsed.path.startswith("/api/"):
                self.route_get(parsed)
            else:
                super().do_GET()
        except Exception as exc:
            self.send_error_json(500, str(exc), traceback.format_exc())

    def do_POST(self) -> None:
        try:
            parsed = urllib.parse.urlparse(self.path)
            if parsed.path == "/api/transcribe":
                self.route_transcribe()
            elif re.fullmatch(r"/api/research/[^/]+/sources/upload", parsed.path):
                self.route_research_upload(parsed)
            else:
                self.route_post(parsed, self.read_json_body())
        except urllib.error.HTTPError as exc:
            self.send_error_json(exc.code, exc.read().decode("utf-8", errors="replace"))
        except Exception as exc:
            self.send_error_json(500, str(exc), traceback.format_exc())

    def do_PUT(self) -> None:
        try:
            parsed = urllib.parse.urlparse(self.path)
            self.route_put(parsed, self.read_json_body())
        except Exception as exc:
            self.send_error_json(500, str(exc), traceback.format_exc())

    def do_DELETE(self) -> None:
        try:
            parsed = urllib.parse.urlparse(self.path)
            self.route_delete(parsed, self.read_json_body())
        except FileNotFoundError as exc:
            self.send_error_json(404, str(exc))
        except PermissionError as exc:
            self.send_error_json(423, str(exc))
        except ValueError as exc:
            self.send_error_json(400, str(exc))
        except Exception as exc:
            self.send_error_json(500, str(exc), traceback.format_exc())

    def route_transcribe(self) -> None:
        length = int(self.headers.get("content-length", "0") or "0")
        if not length:
            self.send_error_json(400, "No audio was uploaded.")
            return
        raw = self.rfile.read(length)
        content_type = self.headers.get("content-type", "")
        filename = "recording.wav"
        if content_type.startswith("multipart/form-data"):
            audio_bytes, filename = parse_multipart_file(content_type, raw)
        else:
            audio_bytes = raw
        suffix = Path(filename).suffix.lower()
        if suffix not in {".wav", ".webm", ".ogg", ".mp3", ".m4a"}:
            suffix = ".wav"
        TMP_DIR.mkdir(parents=True, exist_ok=True)
        audio_path = TMP_DIR / f"transcribe-{uuid.uuid4().hex}{suffix}"
        audio_path.write_bytes(audio_bytes)
        try:
            transcript = transcribe_audio_file(audio_path)
        except Exception as exc:
            self.send_error_json(503, str(exc))
            return
        self.send_json({"transcript": transcript, "engine": "local-whisper"})

    def route_research_upload(self, parsed: urllib.parse.ParseResult) -> None:
        parts = [urllib.parse.unquote(part) for part in parsed.path.strip("/").split("/")]
        slug = parts[2]
        if not is_research_project(slug):
            self.send_error_json(400, "Uploads to the research source ledger require a research project.")
            return
        length = int(self.headers.get("content-length", "0") or "0")
        raw = self.rfile.read(length)
        fields, files = parse_multipart_form(self.headers.get("content-type", ""), raw)
        uploaded = files.get("file") or next(iter(files.values()), None)
        if not uploaded:
            self.send_error_json(400, "No file was uploaded.")
            return
        filename = str(uploaded.get("filename") or "uploaded-source")
        content = uploaded.get("content") or b""
        suffix = Path(filename).suffix.lower()
        TMP_DIR.mkdir(parents=True, exist_ok=True)
        tmp_path = TMP_DIR / f"upload-{uuid.uuid4().hex}{suffix or '.bin'}"
        tmp_path.write_bytes(content)
        extracted = extract_uploaded_text(tmp_path, filename, str(uploaded.get("content_type") or ""))
        payload = {
            "title": fields.get("title") or Path(filename).stem,
            "type": fields.get("type") or "other",
            "reliability": fields.get("reliability") or "secondary",
            "provenance": "uploaded_file",
        }
        research = create_research_source(slug, payload, original_bytes=content, original_name=filename, extracted_text=extracted)
        self.send_json({"research": research})

    def route_get(self, parsed: urllib.parse.ParseResult) -> None:
        parts = [urllib.parse.unquote(part) for part in parsed.path.strip("/").split("/")]
        if parsed.path in {"/api/status", "/api/status/check"}:
            if parsed.path == "/api/status/check" or urllib.parse.parse_qs(parsed.query).get("refresh"):
                refresh_preflight(log=True)
            preflight = get_preflight()
            self.send_json(
                {
                    "hasApiKey": bool(CONFIG.api_key),
                    "mode": preflight.active_mode,
                    "requestedMode": preflight.requested_mode,
                    "aiAvailable": preflight.active_mode in {"api", "claude-code"},
                    "preflight": preflight_payload(),
                    "statusMessage": preflight.status_message,
                    "booksDir": str(CONFIG.books_dir.resolve()),
                    "version": current_version(),
                    "update": {"repo": update_repo()},
                    "draftModel": CONFIG.draft_model,
                    "utilityModel": CONFIG.utility_model,
                    "claudeCodeModel": CONFIG.claude_code_model,
                    "bugReport": {
                        "recipient": bug_report_recipient(),
                        "emailConfigured": bug_report_email_configured(),
                    },
                    "anthropicVersion": "2023-06-01",
                }
            )
            return
        if parsed.path == "/api/config":
            self.send_json(
                {
                    "mode": CONFIG.requested_mode,
                    "hasApiKey": bool(CONFIG.api_key),
                    "apiKeySource": "config.local.json"
                    if json_safe_load(CONFIG_PATH, {}).get("anthropicApiKey")
                    else "ANTHROPIC_API_KEY.txt"
                    if read_local_api_key_file()
                    else "environment"
                    if os.environ.get("ANTHROPIC_API_KEY")
                    else "none",
                    "booksDir": str(CONFIG.books_dir.resolve()),
                    "version": current_version(),
                    "update": {"repo": update_repo()},
                    "port": CONFIG.port,
                    "draftModel": CONFIG.draft_model,
                    "utilityModel": CONFIG.utility_model,
                    "claudeCodeModel": CONFIG.claude_code_model,
                    "bugReport": {
                        "recipient": bug_report_recipient(),
                        "emailConfigured": bug_report_email_configured(),
                    },
                }
            )
            return
        if parsed.path == "/api/update/check":
            try:
                self.send_json(github_latest_release())
            except urllib.error.HTTPError as exc:
                if exc.code == 404:
                    self.send_error_json(404, f"No public GitHub release found for {update_repo()}.")
                else:
                    self.send_error_json(exc.code, exc.read().decode("utf-8", errors="replace"))
            except Exception as exc:
                self.send_error_json(503, str(exc))
            return
        if parsed.path == "/api/books":
            self.send_json({"books": list_books()})
            return
        if len(parts) == 3 and parts[:2] == ["api", "research"]:
            self.send_json(load_research(parts[2]))
            return
        if len(parts) == 3 and parts[:2] == ["api", "books"]:
            self.send_json(load_book(parts[2]))
            return
        if len(parts) == 4 and parts[:2] == ["api", "books"] and parts[3] == "conversation":
            self.send_json(load_conversation(parts[2]))
            return
        self.send_error_json(404, "Not found")

    def route_post(self, parsed: urllib.parse.ParseResult, body: dict[str, Any]) -> None:
        parts = [urllib.parse.unquote(part) for part in parsed.path.strip("/").split("/")]
        if parsed.path == "/api/shutdown":
            self.send_json({"ok": True, "message": "Generation Engine is shutting down."})
            threading.Thread(target=self.server.shutdown, daemon=True).start()
            return
        if parsed.path == "/api/config/claude-login":
            preflight = get_preflight()
            claude_path = preflight.claude_path or discover_claude_path()
            if not claude_path:
                self.send_error_json(404, "Claude Code CLI was not found. Install Claude Code or set claudePath.")
                return
            try:
                subprocess.Popen(
                    [claude_path, "auth", "login"],
                    cwd=str(ROOT),
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.DEVNULL,
                    **hidden_subprocess_kwargs(),
                )
            except Exception as exc:
                self.send_error_json(500, f"Could not start Claude login: {exc}")
                return
            self.send_json({"ok": True, "message": "Claude login started. Complete the browser login, then click Check Again."})
            return
        if parsed.path == "/api/books":
            title = str(body.get("title") or "Untitled Book").strip()
            project_type = str(body.get("projectType") or "novel").strip().lower()
            self.send_json(ensure_book(title, project_type=project_type))
            return
        if parsed.path == "/api/bug-report/draft":
            description = str(body.get("description") or "").strip()
            if not description:
                self.send_error_json(400, "Describe the bug first.")
                return
            self.send_json(draft_bug_report(description, body.get("diagnostics") or {}))
            return
        if parsed.path == "/api/bug-report/send":
            subject = str(body.get("subject") or "Generation Engine bug report").strip()
            report_body = str(body.get("body") or "").strip()
            if not report_body:
                self.send_error_json(400, "The bug report body is empty.")
                return
            try:
                self.send_json(send_bug_report_email(subject, report_body))
            except Exception as exc:
                self.send_error_json(503, str(exc))
            return
        if parsed.path == "/api/update/install":
            server_url = f"http://127.0.0.1:{self.server.server_address[1]}"
            try:
                self.send_json(start_update_install(server_url, bool(body.get("force"))))
            except urllib.error.HTTPError as exc:
                if exc.code == 404:
                    self.send_error_json(404, f"No public GitHub release found for {update_repo()}.")
                else:
                    self.send_error_json(exc.code, exc.read().decode("utf-8", errors="replace"))
            except Exception as exc:
                self.send_error_json(503, str(exc))
            return
        if len(parts) == 5 and parts[:2] == ["api", "research"] and parts[3] == "sources" and parts[4] == "ingest":
            slug = parts[2]
            if not is_research_project(slug):
                self.send_error_json(400, "Source ingestion requires a research project.")
                return
            self.send_json({"research": create_research_source(slug, body)})
            return
        if len(parts) == 5 and parts[:2] == ["api", "research"] and parts[3] == "records" and parts[4] == "request":
            slug = parts[2]
            if not is_research_project(slug):
                self.send_error_json(400, "Records requests require a research project.")
                return
            self.send_json({"research": create_record_request(slug, body)})
            return
        if len(parts) == 4 and parts[:2] == ["api", "research"] and parts[3] == "missing":
            self.send_json(research_missing_report(parts[2]))
            return
        if len(parts) == 4 and parts[:2] == ["api", "research"] and parts[3] == "export-review":
            self.send_json(research_export_review(parts[2]))
            return
        if parsed.path == "/api/import/propose":
            raw = str(body.get("raw", ""))
            title = str(body.get("title") or "Imported Book")
            if real_backend_active():
                try:
                    proposed = staged_import(raw, title)
                except Exception:
                    if active_mode() == "api":
                        raise
                    proposed = mock_import(raw, title)
                    proposed.setdefault("importDiagnostics", {}).setdefault("warnings", []).append(
                        "Staged Claude import failed; local fallback was used."
                    )
            else:
                proposed = mock_import(raw, title)
            self.send_json(proposed)
            return
        if parsed.path == "/api/import/accept":
            proposal = body.get("proposal") or {}
            title = str(proposal.get("title") or body.get("title") or "Imported Book")
            project_type = str(body.get("projectType") or proposal.get("projectType") or "novel").strip().lower()
            data = ensure_book(title, body.get("slug"), project_type=project_type)
            slug = data["slug"]
            book = book_dir(slug)
            for name in CANON_FILES:
                write_text(book / "canon" / f"{name}.md", str((proposal.get("canon") or {}).get(name) or DEFAULT_CANON[name]))
            meta = load_meta(book)
            meta["chapters"] = []
            meta["sections"] = [{"id": "part-1", "label": "Part 1", "chapterIds": []}]
            for index, chapter in enumerate(proposal.get("chapters", []), start=1):
                cid = pad_id(chapter.get("id") or index)
                write_text(chapter_path(book, cid), str(chapter.get("content", "")))
                write_text(chapter_plan_path(book, cid), "")
                write_text(chapter_transcript_path(book, cid), "")
                meta["chapters"].append(
                    {
                        "id": cid,
                        "title": str(chapter.get("title") or f"Chapter {index}"),
                        "status": "locked",
                        "section": "part-1",
                    }
                )
            if meta["chapters"]:
                next_id = pad_id(int(meta["chapters"][-1]["id"]) + 1)
                write_text(chapter_path(book, next_id), f"# Chapter {int(next_id)}\n\n")
                write_text(chapter_plan_path(book, next_id), "")
                write_text(chapter_transcript_path(book, next_id), "")
                meta["chapters"].append({"id": next_id, "title": f"Chapter {int(next_id)}", "status": "draft", "section": "part-1"})
                meta["activeChapter"] = next_id
            save_meta(book, meta)
            self.send_json(load_book(slug))
            return
        if len(parts) == 4 and parts[:2] == ["api", "books"] and parts[3] == "chapters":
            self.send_json(create_chapter(parts[2], body.get("title")))
            return
        if len(parts) == 5 and parts[:2] == ["api", "books"] and parts[3] == "chapters" and parts[4] == "save-draft":
            self.send_json(save_chat_draft_as_chapter(parts[2], body))
            return
        if len(parts) == 4 and parts[:2] == ["api", "books"] and parts[3] == "transcript":
            updated = append_transcript(parts[2], body.get("chapterId", "01"), body.get("speaker", "User"), body.get("text", ""))
            self.send_json({"transcript": updated})
            return
        if len(parts) == 5 and parts[:2] == ["api", "books"] and parts[3] == "conversation" and parts[4] == "welcome":
            self.send_json({"message": ensure_session_welcome(parts[2]), "conversation": load_conversation(parts[2])})
            return
        if len(parts) == 5 and parts[:2] == ["api", "books"] and parts[3] == "conversation" and parts[4] == "clear":
            self.send_json(clear_conversation(parts[2]))
            return
        if parsed.path == "/api/chat/stream":
            self.stream_chat(body)
            return
        if parsed.path == "/api/ai/voice-profile":
            authors = str(body.get("authors", ""))
            if real_backend_active() and body.get("slug"):
                text = ai_message(
                    body["slug"],
                    build_prompt("voice_profile", {"authors": authors}, body["slug"]),
                    max_tokens=1200,
                    purpose="utility",
                )
            else:
                text = mock_voice_profile(authors)
            self.send_json({"content": text, "mode": active_mode()})
            return
        if parsed.path == "/api/ai/plan":
            slug = body["slug"]
            cid = pad_id(body.get("chapterId") or load_meta(book_dir(slug)).get("activeChapter", "01"))
            riff = str(body.get("riff", ""))
            transcript = str(body.get("transcript", ""))
            append_transcript(slug, cid, "User", riff) if riff.strip() else None
            if real_backend_active():
                content = ai_message(
                    slug,
                    build_prompt("plan", {"riff": riff, "transcript": transcript}, slug, cid),
                    max_tokens=2200,
                    chapter_id=cid,
                    purpose="utility",
                )
            else:
                content = mock_plan(riff, transcript)
            write_text(chapter_plan_path(book_dir(slug), cid), content)
            append_transcript(slug, cid, "AI", content)
            self.send_json({"content": content, "book": load_book(slug)})
            return
        if parsed.path == "/api/ai/recap":
            slug = body["slug"]
            key = cache_key_for_recap(slug, body)
            recap_name = f"{body.get('type', 'session')}-{body.get('sectionId') or 'all'}.json"
            recap_path = book_dir(slug) / "recaps" / recap_name
            cached = json_safe_load(recap_path, {})
            if cached.get("key") == key:
                self.send_json({"content": cached.get("content", ""), "cached": True})
                return
            if real_backend_active():
                content = ai_message(
                    slug,
                    build_prompt("recap", body, slug, body.get("chapterId")),
                    max_tokens=2400,
                    chapter_id=body.get("chapterId"),
                    purpose="utility",
                )
            else:
                content = mock_recap(slug, body.get("type", "session"), body.get("sectionId"))
            write_json(recap_path, {"key": key, "content": content, "createdAt": now_iso()})
            self.send_json({"content": content, "cached": False})
            return
        if parsed.path == "/api/ai/continuity":
            slug = body["slug"]
            cid = body.get("chapterId")
            if real_backend_active():
                content = ai_message(
                    slug,
                    build_prompt("continuity", body, slug, cid),
                    max_tokens=3000,
                    chapter_id=cid,
                    purpose="utility",
                )
            else:
                content = mock_continuity(slug, cid)
            self.send_json({"content": content})
            return
        if parsed.path == "/api/ai/lock-deltas":
            self.send_json(propose_lock_deltas(body["slug"], body["chapterId"]))
            return
        if parsed.path == "/api/books/lock/accept":
            slug = body["slug"]
            cid = pad_id(body["chapterId"])
            book = book_dir(slug)
            for file_update in body.get("files", []):
                name = file_update.get("name")
                if name in CANON_FILES:
                    write_text(book / "canon" / f"{name}.md", str(file_update.get("new", "")))
            meta = load_meta(book)
            for chapter in meta.get("chapters", []):
                if chapter["id"] == cid:
                    chapter["status"] = "locked"
            save_meta(book, meta)
            self.send_json(load_book(slug))
            return
        if parsed.path in {"/api/ai/draft", "/api/ai/revise"}:
            self.stream_generation(parsed.path, body)
            return
        self.send_error_json(404, "Not found")

    def route_put(self, parsed: urllib.parse.ParseResult, body: dict[str, Any]) -> None:
        parts = [urllib.parse.unquote(part) for part in parsed.path.strip("/").split("/")]
        if parsed.path == "/api/config":
            updates: dict[str, Any] = {}
            if "mode" in body:
                mode = str(body.get("mode") or "auto").strip().lower()
                if mode not in VALID_MODES:
                    self.send_error_json(400, "Invalid mode")
                    return
                updates["mode"] = mode
            for key in ("booksDir", "port", "draftModel", "utilityModel", "claudeCodeModel", "claudePath", "anthropicApiKey"):
                if key in body:
                    updates[key] = body[key]
            save_local_config(updates)
            reload_runtime_config()
            refresh_preflight(log=True)
            self.send_json(
                {
                    "config": {
                        "mode": CONFIG.requested_mode,
                        "hasApiKey": bool(CONFIG.api_key),
                        "booksDir": str(CONFIG.books_dir.resolve()),
                    },
                    "status": preflight_payload(),
                }
            )
            return
        if len(parts) == 4 and parts[:2] == ["api", "research"] and parts[3] == "claims":
            slug = parts[2]
            book = book_dir(slug)
            if not is_research_project(slug):
                self.send_error_json(400, "Claims require a research project.")
                return
            self.send_json({"research": {**load_research(slug), "claims": save_claims(book, body.get("claims") or [])}})
            return
        if len(parts) == 4 and parts[:2] == ["api", "research"] and parts[3] == "timeline":
            slug = parts[2]
            write_text(research_dir(book_dir(slug)) / "timeline.md", str(body.get("content") or ""))
            self.send_json({"research": load_research(slug)})
            return
        if len(parts) == 4 and parts[:2] == ["api", "research"] and parts[3] == "voice":
            slug = parts[2]
            write_text(research_dir(book_dir(slug)) / "voice_profile.md", str(body.get("content") or ""))
            self.send_json({"research": load_research(slug)})
            return
        if len(parts) == 5 and parts[:2] == ["api", "research"] and parts[3] == "people":
            slug, entity_id = parts[2], parts[4]
            path = research_dir(book_dir(slug)) / "people" / f"{safe_file_stem(entity_id, 'entity')}.md"
            existing_meta, _body = parse_frontmatter(read_text(path))
            meta = body.get("meta") or existing_meta or {"id": entity_id}
            meta.setdefault("id", entity_id)
            write_text(path, frontmatter_markdown(meta, str(body.get("body") or "")))
            self.send_json({"research": load_research(slug)})
            return
        if len(parts) == 5 and parts[:2] == ["api", "research"] and parts[3] == "records":
            self.send_json({"research": update_record_request(parts[2], parts[4], body)})
            return
        if len(parts) == 5 and parts[:2] == ["api", "books"] and parts[3] == "canon":
            slug, name = parts[2], parts[4]
            if name not in CANON_FILES:
                self.send_error_json(400, "Invalid canon file")
                return
            write_text(book_dir(slug) / "canon" / f"{name}.md", str(body.get("content", "")))
            self.send_json(load_book(slug))
            return
        if len(parts) == 5 and parts[:2] == ["api", "books"] and parts[3] == "chapters":
            slug, cid = parts[2], pad_id(parts[4])
            book = book_dir(slug)
            if "content" in body:
                write_text(chapter_path(book, cid), str(body.get("content", "")))
            if "plan" in body:
                write_text(chapter_plan_path(book, cid), str(body.get("plan", "")))
            if "transcript" in body:
                write_text(chapter_transcript_path(book, cid), str(body.get("transcript", "")))
            meta = load_meta(book)
            for chapter in meta.get("chapters", []):
                if chapter["id"] == cid:
                    for key in ("title", "status", "section"):
                        if key in body:
                            chapter[key] = body[key]
                    meta["activeChapter"] = cid
            save_meta(book, meta)
            self.send_json(load_book(slug))
            return
        if len(parts) == 4 and parts[:2] == ["api", "books"] and parts[3] == "conversation":
            conversation = body.get("conversation", body)
            self.send_json(save_conversation(parts[2], conversation))
            return
        if len(parts) == 4 and parts[:2] == ["api", "books"] and parts[3] == "meta":
            slug = parts[2]
            book = book_dir(slug)
            meta = load_meta(book)
            incoming = body.get("meta", body)
            for key in ("title", "activeChapter", "sections", "chapters"):
                if key in incoming:
                    meta[key] = incoming[key]
            if "locked" in incoming:
                meta["locked"] = bool(incoming["locked"])
            save_meta(book, meta)
            self.send_json(load_book(slug))
            return
        self.send_error_json(404, "Not found")

    def route_delete(self, parsed: urllib.parse.ParseResult, body: dict[str, Any]) -> None:
        parts = [urllib.parse.unquote(part) for part in parsed.path.strip("/").split("/")]
        if len(parts) == 3 and parts[:2] == ["api", "books"]:
            self.send_json(delete_book(parts[2], str(body.get("confirmation") or "")))
            return
        self.send_error_json(404, "Not found")

    def stream_chat(self, body: dict[str, Any]) -> None:
        slug = str(body.get("slug") or "")
        user_message = str(body.get("message") or "").strip()
        if not slug or not user_message:
            self.send_error_json(400, "Missing chat message.")
            return
        append_conversation_message(slug, "user", user_message, {"kind": "chat"})
        self.send_response(200)
        self.send_header("content-type", "text/plain; charset=utf-8")
        self.send_header("cache-control", "no-store")
        self.send_header("connection", "close")
        self.end_headers()
        chunks: list[str] = []

        def write_chunk(text: str) -> None:
            if not text:
                return
            chunks.append(text)
            try:
                self.wfile.write(text.encode("utf-8"))
                self.wfile.flush()
            except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
                raise ClientDisconnected()

        try:
            for piece in chat_stream(slug, user_message):
                write_chunk(piece)
            assistant_text = "".join(chunks)
            _clean, marker = extract_draft_marker(assistant_text)
            metadata = {"kind": "draft" if marker else "chat"}
            if marker:
                metadata["draft"] = marker
            append_conversation_message(slug, "assistant", assistant_text, metadata)
        except ClientDisconnected:
            return
        except Exception as exc:
            message = f"\n\n[Generation error: {exc}]\n"
            write_chunk(message)
            append_conversation_message(slug, "assistant", message.strip(), {"kind": "error"})

    def stream_generation(self, path: str, body: dict[str, Any]) -> None:
        slug = body["slug"]
        cid = pad_id(body.get("chapterId") or load_meta(book_dir(slug)).get("activeChapter", "01"))
        self.send_response(200)
        self.send_header("content-type", "text/plain; charset=utf-8")
        self.send_header("cache-control", "no-store")
        self.send_header("connection", "close")
        self.end_headers()
        chunks: list[str] = []

        def write_chunk(text: str) -> None:
            if not text:
                return
            chunks.append(text)
            try:
                self.wfile.write(text.encode("utf-8"))
                self.wfile.flush()
            except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
                raise ClientDisconnected()

        try:
            if path == "/api/ai/draft":
                plan = str(body.get("plan") or read_text(chapter_plan_path(book_dir(slug), cid)))
                transcript = str(body.get("transcript") or read_text(chapter_transcript_path(book_dir(slug), cid)))
                if real_backend_active():
                    prompt = build_prompt("draft", {"plan": plan, "transcript": transcript}, slug, cid)
                    stream = ai_stream(slug, prompt, max_tokens=int(body.get("maxTokens") or 12000), chapter_id=cid)
                elif is_research_project(slug):
                    raise RuntimeError("Research drafting requires Claude Code or Claude API. Mock fallback is disabled for research projects.")
                else:
                    stream = mock_draft(slug, cid, plan, transcript)
                for piece in stream:
                    write_chunk(piece)
                content = "".join(chunks)
                write_text(chapter_path(book_dir(slug), cid), content)
                append_transcript(slug, cid, "AI Draft", f"Drafted chapter {cid}.")
            else:
                current = str(body.get("chapter") or read_text(chapter_path(book_dir(slug), cid)))
                request_text = str(body.get("request") or "")
                append_transcript(slug, cid, "User Revision", request_text)
                if real_backend_active():
                    prompt = build_prompt("revise", {"chapter": current, "request": request_text}, slug, cid)
                    stream = ai_stream(slug, prompt, max_tokens=int(body.get("maxTokens") or 12000), chapter_id=cid)
                elif is_research_project(slug):
                    raise RuntimeError("Research revision requires Claude Code or Claude API. Mock fallback is disabled for research projects.")
                else:
                    stream = mock_revise(current, request_text)
                for piece in stream:
                    write_chunk(piece)
                content = "".join(chunks)
                write_text(chapter_path(book_dir(slug), cid), content)
                append_transcript(slug, cid, "AI Revision", f"Applied revision: {request_text}")
        except ClientDisconnected:
            return
        except Exception as exc:
            message = f"\n\n[Generation error: {exc}]\n"
            write_chunk(message)


class ClientDisconnected(Exception):
    pass


def anthropic_message_for_import(prompt: str) -> str:
    if not CONFIG.api_key:
        raise RuntimeError("Missing ANTHROPIC_API_KEY")
    preflight = get_preflight()
    if active_mode() == "api" and not preflight.api_ok:
        raise RuntimeError(preflight.api_message)
    body = {
        "model": CONFIG.utility_model,
        "max_tokens": 10000,
        "system": [{"type": "text", "text": IMPORT_SYSTEM}],
        "messages": [{"role": "user", "content": prompt}],
    }
    req = urllib.request.Request(
        "https://api.anthropic.com/v1/messages",
        data=json.dumps(body).encode("utf-8"),
        headers={
            "content-type": "application/json",
            "x-api-key": CONFIG.api_key,
            "anthropic-version": "2023-06-01",
        },
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=600) as response:
        data = json.loads(response.read().decode("utf-8"))
    return "".join(block.get("text", "") for block in data.get("content", []) if block.get("type") == "text")


def find_open_port(start: int) -> int:
    port = start
    while port < start + 50:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
            probe.settimeout(0.25)
            if probe.connect_ex(("127.0.0.1", port)) == 0:
                port += 1
                continue
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
            try:
                sock.bind(("127.0.0.1", port))
                return port
            except OSError:
                port += 1
    raise RuntimeError("No open local port found")


def main() -> None:
    refresh_preflight(log=True)
    port = find_open_port(CONFIG.port)
    os.chdir(WEB_DIR)
    server = ThreadingHTTPServer(("127.0.0.1", port), GenerationHandler)
    write_json(
        STATE_PATH,
        {
            "pid": os.getpid(),
            "port": port,
            "url": f"http://127.0.0.1:{port}",
            "booksDir": str(CONFIG.books_dir.resolve()),
            "mode": active_mode(),
            "startedAt": now_iso(),
        },
    )
    console_log(f"Generation Engine running at http://127.0.0.1:{port}")
    console_log(f"Books directory: {CONFIG.books_dir}")
    console_log(f"Mode: {active_mode()}")
    console_log(f"Status: {get_preflight().status_message}")
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        console_log("Shutting down.")
    finally:
        server.server_close()
        try:
            if STATE_PATH.exists():
                STATE_PATH.unlink()
        except OSError:
            pass


if __name__ == "__main__":
    main()