From 56043138a7cbe171c5e23e44b028c6444272cfe5 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 18:24:27 +0800 Subject: [PATCH 01/14] docs: add Synthesis Document plan and update backlog (#027, #028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Redesign #027 from "Study Guide Export" to "Synthesis Document Generation" with a three-phase AI pipeline (decontextualize, cluster, synthesize) plus human-owned phases 0 and 4 - Add docs/synthesis-document-plan.md with full implementation spec: data types, prompt structures, file breakdown, open questions - Update #028 to reflect that Phase 4 (human review) requires no plugin code β€” handled externally via Claude Code / Gemini CLI - Close #022–#026 in Completed section (all shipped in plugin PR #2) Co-Authored-By: Claude Sonnet 4.6 --- BACKLOG.md | 137 ++++++------ docs/synthesis-document-plan.md | 384 ++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+), 71 deletions(-) create mode 100644 docs/synthesis-document-plan.md diff --git a/BACKLOG.md b/BACKLOG.md index 8b755ef..d4f516f 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -138,91 +138,86 @@ - **Robust Windows Handling**: Increased timeouts to 8 minutes and implemented non-blocking cleanup to prevent `EBUSY` resource locks. - **UI Integration**: Added a dedicated web-grounding toggle and descriptive feedback in the sidebar settings. -### [#016] TypeError in TileCard (icon of undefined) -- **Status**: `Closed` | **Resolved**: 2026-05-03 -- **Labels**: `Bug`, `Stability`, `UI` -- **Summary**: Fixed a runtime crash where `TileCard` and other UI components failed when encountering unknown content types hallucinated by the LLM. Implemented `getSafeContentTypeConfig` and added input validation. - ---- - -## πŸ”­ Upcoming Features - ### [#022] Obsidian Plugin β€” Core Infrastructure -- **Status**: `In Progress` | **Priority**: `P1` | **Labels**: `Feature`, `Obsidian`, `Architecture` -- **Created**: 2026-05-11 -- **Description**: Port the nodepad React UI into Obsidian as a first-class plugin using the `TextFileView` API. `.nodepad` files live directly in the vault and auto-save on every state change β€” no manual export step. Renders all three view modes (Tiling, Kanban, Graph) inside an Obsidian leaf, themed via Obsidian's own CSS variables. -- **Scope**: - - `plugin/src/main.ts` β€” registers `.nodepad` extension, ribbon icon, command palette entry +- **Status**: `Closed` | **Resolved**: 2026-05-11 +- **Labels**: `Feature`, `Obsidian`, `Architecture` +- **Summary**: Ported the nodepad React UI into Obsidian as a first-class plugin using the `TextFileView` API. `.nodepad` files live directly in the vault and auto-save on every state change. All three view modes (Tiling, Kanban, Graph) render inside an Obsidian leaf, themed via Obsidian's own CSS variables. +- **Technical Highlights**: + - `plugin/src/main.ts` β€” registers `.nodepad` extension, ribbon icon, command palette entry, folder right-click menu - `plugin/src/view.tsx` β€” mounts React into the Obsidian leaf, reads/writes vault file via `requestSave()` - - `plugin/src/styles.css` β€” maps Tailwind tokens to Obsidian CSS variables for automatic theme adaptation - - `plugin/esbuild.config.mjs` β€” bundles all shared `lib/` and `components/` from the local fork - - Component patches β€” `isPlugin` mode in `VimInput` (hides Projects nav), portal scoping in `StatusBar`, `AboutPanel`, `Sheet` -- **Reference**: Upstream PR [mskayyali/nodepad#47](https://github.com/mskayyali/nodepad/pull/47) β€” cherry-picking plugin infrastructure only, skipping Anthropic provider additions. + - `plugin/src/styles.css` β€” maps Tailwind tokens to Obsidian CSS variables; SVG and button `!important` overrides for Obsidian CSS cascade conflicts + - `plugin/esbuild.config.mjs` β€” CJS bundle of all shared `lib/` and `components/` from the local fork via path aliases + - Component patches β€” `isPlugin` mode in `VimInput` (hides Projects nav), portal scoping in `StatusBar`, `AboutPanel`, `Sheet`; transparent SVG `` fix for graph pan/zoom; minimap inline padding override ### [#023] Obsidian Plugin β€” Obsidian Settings UI -- **Status**: `Open` | **Priority**: `P1` | **Labels**: `Feature`, `Obsidian`, `UX` -- **Created**: 2026-05-11 -- **Description**: In-Obsidian settings tab (Settings β†’ Nodepad) for configuring the AI provider, model, and API key without touching the web app. Settings are stored in `.obsidian/plugins/nodepad/data.json`, local to the vault and never synced externally. -- **Scope**: - - Provider dropdown (OpenRouter, OpenAI, Z.ai, Ollama, Gemini CLI) - - API key field (hidden for keyless providers like Gemini CLI and local Ollama) - - Model ID input with per-provider defaults - - `plugin/src/settings.ts` β€” `NodepadSettingTab` extending Obsidian's `PluginSettingTab` +- **Status**: `Closed` | **Resolved**: 2026-05-11 +- **Labels**: `Feature`, `Obsidian`, `UX` +- **Summary**: Implemented an in-Obsidian settings tab (Settings β†’ Nodepad) for all five AI providers. Settings stored in `.obsidian/plugins/nodepad/data.json`, local to the vault. +- **Technical Highlights**: + - Provider dropdown (OpenRouter, OpenAI, Z.ai, Ollama, Gemini CLI) with per-provider key persistence via `providerKeys` record + - API key field with eye icon toggle (hidden for keyless providers); Ollama model auto-discovery on switch + - Web-grounding toggle with provider-specific descriptions for all applicable providers ### [#024] Obsidian Plugin β€” CLI Provider Bridge (`child_process`) -- **Status**: `Open` | **Priority**: `P1` | **Labels**: `Feature`, `Obsidian`, `Architecture` -- **Created**: 2026-05-11 -- **Description**: Shared infrastructure in `plugin/src/ai-adapter.ts` for invoking local CLI tools as subprocesses via Node.js `child_process` inside Obsidian's Electron environment. This is the prerequisite that makes both Gemini CLI and future Claude Code work in the plugin β€” neither can be called via HTTP, both need to be spawned as local processes. The web app already handles this server-side in `/api/ai`; this is the equivalent for the plugin context. -- **Scope**: - - `spawnCLI(binary, args, stdinPayload)` β€” generic subprocess helper with stdout capture and timeout - - Stdin piping for high-context prompts (mirrors existing web app pattern) - - JSON extraction from CLI response wrappers (reuse logic from `lib/ai-ghost.ts`) - - Error handling: binary not found, non-zero exit code, malformed output +- **Status**: `Closed` | **Resolved**: 2026-05-11 +- **Labels**: `Feature`, `Obsidian`, `Architecture` +- **Summary**: Generic `spawnCLI()` subprocess helper in `plugin/src/ai-adapter.ts` that enables CLI tools (Gemini CLI, future Claude Code) to be invoked directly from Obsidian's Electron environment. +- **Technical Highlights**: + - Stdin piping for high-context prompts (bypasses Windows argument length limits) + - JSON wrapper extraction from CLI response formats + - 8-minute timeout with non-blocking cleanup to prevent `EBUSY` resource locks on Windows ### [#025] Obsidian Plugin β€” Ollama Provider Support -- **Status**: `Open` | **Priority**: `P1` | **Labels**: `Feature`, `Obsidian`, `Ollama` -- **Created**: 2026-05-11 -- **Description**: Enable Ollama (both local and Cloud) as an AI provider within the Obsidian plugin. Ollama uses a different request shape than OpenAI-compatible providers (`/api/chat` instead of `/chat/completions`, no `response_format`, `stream: false`). Since `requestUrl()` in Obsidian's Electron bypasses CORS, local Ollama (`localhost:11434`) works directly without the `/api/ai` proxy β€” actually cleaner than the web app path. -- **Scope**: - - Ollama request shape in `plugin/src/ai-adapter.ts` (port from `lib/ai-enrich.ts`) - - Cloud vs local routing (mirrors existing `getBaseUrl` logic) - - Dynamic model discovery via `requestUrl("http://localhost:11434/api/tags")` - - No RAG/embedding support in initial version (scoped to enrichment and ghost synthesis) +- **Status**: `Closed` | **Resolved**: 2026-05-11 +- **Labels**: `Feature`, `Obsidian`, `Ollama` +- **Summary**: Enabled Ollama (local and Cloud) with Hybrid RAG in the Obsidian plugin. `requestUrl()` in Obsidian's Electron bypasses CORS, so local Ollama works directly without a proxy. +- **Technical Highlights**: + - Local vs Cloud routing; dynamic model discovery via `/api/tags` + - Hybrid RAG pipeline (web search β†’ `embeddinggemma` vectorization β†’ cosine similarity β†’ top-5 injection) ported from web app server route to plugin adapter + - `getProviderHeaders()` for correct `Authorization` handling per routing mode ### [#026] Obsidian Plugin β€” Gemini CLI Provider Support -- **Status**: `Open` | **Priority**: `P1` | **Labels**: `Feature`, `Obsidian`, `Gemini-CLI` -- **Created**: 2026-05-11 -- **Description**: Enable Gemini CLI as a provider in the Obsidian plugin via the `child_process` bridge (#024). In the web app, Gemini CLI calls are handled server-side in `/api/ai`; in the plugin they are spawned directly from Obsidian's Electron process. Requires Gemini CLI to be installed and authenticated on the host machine. Web grounding via Gemini's native `google_web_search` tool is preserved since it runs inside the CLI itself. -- **Scope**: - - `fetchGeminiCLI(prompt, options)` in `plugin/src/ai-adapter.ts` using the `child_process` bridge - - Stdin piping for enrichment prompts (same pattern as web app) - - `--policy simple` flag for structured output, two-stage web grounding for RAG - - Settings UI: no API key field; show auth status / binary detection - - Graceful error if `gemini` binary is not found on PATH +- **Status**: `Closed` | **Resolved**: 2026-05-11 +- **Labels**: `Feature`, `Obsidian`, `Gemini-CLI` +- **Summary**: Enabled Gemini CLI as a provider inside Obsidian via the `child_process` bridge. Web grounding runs natively inside the CLI (no separate RAG pass needed). +- **Technical Highlights**: + - Two-stage pipeline (Stage 1: agentic web research; Stage 2: `--policy simple` structured JSON enrichment) + - Keyless auth detection in settings UI; graceful error if `gemini` binary not on PATH + - JSON stats parsing for Ollama-style terminal logs (tool usage, model selection) -### [#027] Obsidian Plugin β€” Structured Study Guide Export to Vault +### [#016] TypeError in TileCard (icon of undefined) +- **Status**: `Closed` | **Resolved**: 2026-05-03 +- **Labels**: `Bug`, `Stability`, `UI` +- **Summary**: Fixed a runtime crash where `TileCard` and other UI components failed when encountering unknown content types hallucinated by the LLM. Implemented `getSafeContentTypeConfig` and added input validation. + +--- + +## πŸ”­ Upcoming Features + +### [#027] Obsidian Plugin β€” Synthesis Document Generation - **Status**: `Open` | **Priority**: `P2` | **Labels**: `Feature`, `Obsidian`, `AI` - **Created**: 2026-05-11 -- **Description**: On-demand command that takes enriched blocks from the current `.nodepad` canvas and writes a structured Obsidian-native `.md` file into the vault. Unlike the existing one-shot markdown export, this produces properly formatted study guides with frontmatter, `[[backlinks]]` to related vault notes, section headers per category, and source citations. Acts as the bridge from nodepad's "raw idea staging area" into Obsidian's permanent knowledge graph. -- **Scope**: - - AI enrichment pass over all blocks to generate section groupings and a study guide outline - - Markdown generation with YAML frontmatter (`tags`, `created`, `source: nodepad`) - - `[[wikilink]]` insertion for terms that match existing vault note titles (via `app.vault.getMarkdownFiles()`) - - Output path: configurable folder (default: vault root), slugified from canvas name - - Command palette entry: "Export as Study Guide" -- **Open Questions**: - - Should the export be a destructive replacement (overwrite) or always create a new versioned file? - - Should tags be pulled from nodepad's `category` field or generated fresh by the AI? - -### [#028] Obsidian Plugin β€” Human-in-the-Loop Review Before Vault Write -- **Status**: `Open` | **Priority**: `P2` | **Labels**: `Feature`, `Obsidian`, `UX` +- **Design spec**: [`docs/synthesis-document-plan.md`](docs/synthesis-document-plan.md) +- **Description**: On-demand command ("Generate Synthesis Document") that consolidates enriched nodes from a `.nodepad` canvas into a structured, contextualized Obsidian markdown document. Unlike the raw markdown export which dumps nodes grouped by type, this pipeline expands sparse notes into self-contained statements, clusters them into coherent thematic sections by meaning, and adds expounding prompts that push thinking into adjacent territory the notes don't cover. Acts as the bridge from nodepad's raw idea staging area into Obsidian's permanent knowledge graph. +- **Pipeline**: + - **Phase 0** (human): User adds a `reference`/`entity` node naming the source material. No code needed. + - **Phase 1** (no AI): Build edge map from `influencedByIndices` graph; detect source anchor nodes. + - **Phase 2a + 2b** (parallel AI calls): Decontextualize each node into a self-contained statement (Call A) while simultaneously clustering nodes into named sections (Call B). + - **Phase 2c** (sequential AI call): Merge A + B results; generate section intros, expounding prompts, gap markers, and overall summary. + - **Phase 3** (no AI): Render to Obsidian-native markdown, inject `[[wikilinks]]`, write to vault. + - **Phase 4** (human + external tools): User reviews output against source material via Claude Code, Gemini CLI, or NotebookLM. No code needed. +- **New files**: + - `plugin/src/synthesis.ts` β€” pipeline orchestration + - `lib/synthesis-export.ts` β€” Phase 3 markdown renderer +- **Modified files**: + - `plugin/src/main.ts` β€” register command + - `plugin/src/view.tsx` β€” expose `generateSynthesisDocument()` on the view + - `plugin/src/ai-adapter.ts` β€” add `callDecontextualize`, `callCluster`, `callSynthesize` + +### [#028] Obsidian Plugin β€” Human-in-the-Loop Review (Phase 4) +- **Status**: `Open` | **Priority**: `P3` | **Labels**: `Feature`, `Obsidian`, `UX` - **Created**: 2026-05-11 -- **Description**: Before any structured `.md` file is written to the vault (#027), present the user with a review panel showing the proposed study guide. The user can edit sections, approve, or cancel. Ensures the AI output is supervised before it becomes a permanent vault note. Fits the stated design goal: the AI enriches, the human decides. -- **Scope**: - - Preview panel (modal or sidebar) showing the rendered markdown before write - - Section-level approve/reject toggles (keep this section, drop that one) - - Inline editing of AI-generated text before commit - - "Write to Vault" confirmation button triggers the actual `app.vault.create()` call +- **Description**: Phase 4 of the Synthesis Document pipeline β€” reviewing the generated document against the source material and making corrections. This requires no plugin code: the output is a standard Obsidian markdown file that any AI tool with vault access (Claude Code, Gemini CLI, NotebookLM) can read, annotate, and help correct. The review process itself is pedagogically valuable β€” identifying and correcting AI errors demonstrates understanding of the source material. If a built-in review UI is later desired (diff view, section toggles), it can be added as a separate issue. --- diff --git a/docs/synthesis-document-plan.md b/docs/synthesis-document-plan.md new file mode 100644 index 0000000..4f665fa --- /dev/null +++ b/docs/synthesis-document-plan.md @@ -0,0 +1,384 @@ +# Synthesis Document Generation β€” Implementation Plan + +> Backlog: [#027](../BACKLOG.md) + +## What it is + +A three-phase AI pipeline that consolidates enriched nodes from a `.nodepad` canvas into a structured, contextualized Obsidian markdown document. Unlike the raw markdown export (which groups nodes by content type), the Synthesis Document expands sparse notes into self-contained statements, clusters them into thematic sections by meaning, and adds expounding prompts that push thinking beyond what the notes cover. + +The core problem it solves: nodepad notes are intentionally unstructured fragments. They only make full sense in the context of the source material being studied. The pipeline uses the enrichment graph (`influencedByIndices` edges, annotations, source anchor nodes) to reconstruct as much of that context as possible before synthesizing. + +--- + +## Pipeline Overview + +``` +Phase 0 [Human] Add source anchor node(s) to canvas +Phase 1 [No AI] Build edge map from block graph + ↓ +Phase 2a [AI Call A] ─────────────────────────────────────────── \ + Decontextualize each node into a self-contained statement β”œβ”€β†’ Phase 2c [AI Call C] β†’ Phase 3 [No AI] β†’ Vault +Phase 2b [AI Call B] ─────────────────────────────────────────── / + Cluster nodes into named sections + ↓ +Phase 4 [Human] Review output against source via external tools +``` + +Calls A and B run in parallel (`Promise.all`). Call C is sequential after both complete. + +--- + +## Phase 0 β€” Source Anchor (Human, no code) + +The user adds a `reference` or `entity` type node to the canvas naming the source material being studied. + +**Example:** *"Das Kapital, Chapter 1 β€” Karl Marx, 1867"* + +The pipeline detects these automatically (see Phase 1). If none exist, generation proceeds with a non-blocking warning: *"No source node found β€” adding a reference node naming your source improves output quality."* + +--- + +## Phase 1 β€” Edge Map Builder (No AI) + +**File:** `plugin/src/synthesis.ts` + +Builds a serialized representation of the block graph for use by both parallel AI calls. + +**Logic:** +1. Iterate all blocks on the canvas. +2. For each block, resolve `influencedByIndices` to get the actual neighbor block objects. +3. Identify source anchor candidates: + - Always: `contentType === "reference"` + - Heuristic: `contentType === "entity"` blocks referenced by many others but with few outgoing connections themselves +4. Separate anchor nodes from regular nodes. + +**Output types:** + +```typescript +interface NodeWithContext { + id: string + text: string + contentType: ContentType + category?: string + annotation?: string + neighborIds: string[] + neighborTexts: string[] // text + annotation of each neighbor, concatenated +} + +interface EdgeMap { + nodes: NodeWithContext[] + sourceAnchors: NodeWithContext[] +} +``` + +--- + +## Phase 2a β€” Decontextualization (AI Call A, parallel) + +**File:** `plugin/src/ai-adapter.ts` β†’ `callDecontextualize(plugin, edgeMap)` + +**Model:** Current provider's primary model (same as enrichment). + +**Task:** Rewrite each node as a self-contained statement using only the provided neighboring notes and source anchor as context. No external knowledge fill. + +**Prompt:** + +``` +GROUNDING RULES β€” CRITICAL: +Use ONLY the provided notes and context. Do not draw on external knowledge to fill gaps. +If a note is too sparse to expand meaningfully, return its original text unchanged. + +SOURCE MATERIAL: {sourceAnchors} + +For each note below, rewrite it as a single self-contained statement. +Use only the note's annotation and neighboring notes as context. +Resolve implicit references (pronouns, abbreviations, topic shortcuts). +Do not add information not present in the provided context. + +Notes: +[ + { + "id": "abc123", + "text": "labour theory β€” socially necessary time", + "annotation": "Marx argues value is determined by socially necessary labour time", + "neighbors": [ + "Karl Marx β€” author of Das Kapital, foundational work of Marxist economic theory", + "Surplus value: the difference between value produced by labour and wages paid" + ] + }, + ... +] + +Return ONLY valid JSON β€” an array with one entry per input note, in the same order: +[{ "id": "abc123", "statement": "..." }, ...] +``` + +**Output:** `DecontextualizedNode[]` + +```typescript +interface DecontextualizedNode { + id: string + statement: string +} +``` + +**Error handling:** +- Statement identical to original text β†’ valid, keep it (note was too sparse to expand). +- Parse failure β†’ fall back to `text + ". " + annotation` concatenated as the statement. + +--- + +## Phase 2b β€” Clustering (AI Call B, parallel with A) + +**File:** `plugin/src/ai-adapter.ts` β†’ `callCluster(plugin, edgeMap)` + +**Model:** Lightest/fastest model available for the current provider. This is the cheapest call in the pipeline β€” the output is only node ID groupings, no prose. + +**Task:** Group nodes into semantically coherent named sections. Nodes with no connections go into "General." + +**Prompt:** + +``` +Group the following research notes into semantically coherent sections for a synthesis document. + +RULES: +- Use the connection hints to guide grouping, but group by meaning β€” not just graph proximity +- Notes with no connections go into a section named "General" +- Aim for 3–7 sections; merge thin sections rather than leaving singletons +- Name each section descriptively (2–5 words) +- Every note ID must appear in exactly one section + +SOURCE MATERIAL: {sourceAnchors} + +Notes: +[ + { + "id": "abc123", + "text": "labour theory β€” socially necessary time", + "category": "Economics", + "contentType": "definition", + "connectedIds": ["def456", "ghi789"] + }, + ... +] + +Return ONLY valid JSON: +[{ "sectionName": "Labour Theory of Value", "nodeIds": ["abc123", "def456"] }, ...] +``` + +**Output:** `ClusterAssignment[]` + +```typescript +interface ClusterAssignment { + sectionName: string + nodeIds: string[] +} +``` + +**Post-processing (no AI):** +- Verify every node ID appears in exactly one section. +- Any missing IDs β†’ append to "General". +- Any duplicate assignments β†’ keep first occurrence. + +--- + +## Phase 2c β€” Synthesis (AI Call C, sequential) + +**File:** `plugin/src/ai-adapter.ts` β†’ `callSynthesize(plugin, mergedSections, sourceAnchors)` + +**Model:** Best available for the current provider. + +**Pre-merge (no AI):** Join Call A and Call B results by node ID before constructing the prompt: + +```typescript +const mergedSections = clusterAssignments.map(cluster => ({ + candidateHeading: cluster.sectionName, + statements: cluster.nodeIds + .map(id => decontextualizedNodes.find(n => n.id === id)) + .filter(Boolean), +})) +``` + +**Task:** Write section headings, intros, expounding prompts, gap markers, and an overall summary. Strictly grounded on the provided statements β€” no external knowledge fill. + +**Prompt:** + +``` +You are generating a Synthesis Document from research notes. + +A Synthesis Document consolidates sparse, fragmented notes into a coherent document +about the ideas and concepts in the nodespace. It is not a summary β€” it organizes notes +into their full meaning and pushes thinking outward through open questions. + +GROUNDING RULES β€” CRITICAL: +- Only use the provided statements and source material +- Do not supplement gaps with external knowledge +- Where notes are insufficient, flag as a gap explicitly + +SOURCE MATERIAL: {sourceAnchors} + +For each section: +1. Write a clear section heading (improve on the candidate name if needed) +2. Write a 1–2 sentence intro: what should the reader understand from this section? +3. Write 2–3 expounding prompts β€” open questions that push thinking into adjacent territory + the notes do NOT cover. These should not test recall; they should invite exploration. + Good example: "Are there economic systems that critique capital accumulation without + requiring collective ownership of production?" + Bad example: "What is the labour theory of value?" (that's just recall) +4. List any gaps: concepts implied by the notes but not explained within them + +Also write a 2–3 sentence overall summary of the entire nodespace. + +Sections: +[ + { + "candidateHeading": "Labour Theory of Value", + "statements": [ + { "id": "abc123", "statement": "In Das Kapital, Marx argues that a commodity's value..." }, + { "id": "def456", "statement": "Surplus value is the difference between..." } + ] + }, + ... +] + +Return ONLY valid JSON: +{ + "summary": "...", + "sections": [{ + "heading": "...", + "intro": "...", + "nodeIds": ["abc123", "def456"], + "expandingPrompts": ["...", "..."], + "gaps": ["..."] + }] +} +``` + +**Output:** `SynthesisOutline` + +```typescript +interface SynthesisOutline { + summary: string + sections: Array<{ + heading: string + intro: string + nodeIds: string[] + expandingPrompts: string[] + gaps: string[] + }> +} +``` + +--- + +## Phase 3 β€” Markdown Renderer (No AI) + +**File:** `lib/synthesis-export.ts` + +Pure function, no Obsidian dependencies. Usable from both the plugin and the web app. + +```typescript +export function renderSynthesisDocument( + canvasName: string, + outline: SynthesisOutline, + decontextualized: DecontextualizedNode[], + clusterAssignments: ClusterAssignment[], + date: string, +): string +``` + +**Output format:** + +```markdown +--- +title: "Synthesis: Canvas Name" +tags: [synthesis, nodepad] +created: 2026-05-11 +source: nodepad +--- + +# Canvas Name β€” Synthesis + +> 2–3 sentence summary of the entire nodespace. + +--- + +## Labour Theory of Value + +1–2 sentence intro explaining what the reader should understand. + +In Das Kapital, Marx argues that a commodity's value is determined by the socially +necessary labour time required to produce it β€” the foundation for his theory of +surplus value. +> *Source: nodes abc123, def456* + +Surplus value is the difference between the value workers produce and the wages +they receive, which Marx identifies as the mechanism of capitalist profit extraction. +> *Source: node def456* + +> [!example]- Expounding Prompts +> 1. Are there economic systems that critique capital accumulation without requiring collective ownership of production? +> 2. How does Marx's theory of value hold up against modern algorithmic pricing, where marginal cost approaches zero? + +> [!question]- Gaps in your notes +> - Rate of exploitation is referenced but not defined within your notes. + +--- + +## [Next section] +... + +--- +*Generated by [nodepad](https://nodepad.space) from 24 nodes Β· 11 May 2026* +``` + +**Wikilink injection (Obsidian plugin only):** +After rendering, scan the output text for terms that exactly match existing vault note titles via `app.vault.getMarkdownFiles()`. Wrap matched terms with `[[double brackets]]`. Case-insensitive, whole-word match only. Runs as a string post-pass before vault write. + +**Output path:** +- Default: vault root +- Filename: `{canvas-name}-synthesis.md` (slugified) +- If file exists: append `-2`, `-3`, etc. (never overwrite) + +--- + +## Progress UX + +The command shows incremental `Notice` updates during generation: + +``` +Generating synthesis… (1/3) Decontextualizing nodes +Generating synthesis… (2/3) Clustering sections +Generating synthesis… (3/3) Writing synthesis +Synthesis document created: my-canvas-synthesis.md +``` + +On completion, the generated file opens in a new Obsidian leaf. + +On any error, a `Notice` shows the error message and generation stops cleanly. + +--- + +## New Files + +| File | Purpose | +|---|---| +| `plugin/src/synthesis.ts` | Pipeline orchestration (Phases 1–3) | +| `lib/synthesis-export.ts` | Phase 3 markdown renderer (pure, no Obsidian deps) | + +## Modified Files + +| File | Change | +|---|---| +| `plugin/src/main.ts` | Register "Generate Synthesis Document" command | +| `plugin/src/view.tsx` | Expose `generateSynthesisDocument()` on `NodepadView` | +| `plugin/src/ai-adapter.ts` | Add `callDecontextualize`, `callCluster`, `callSynthesize` | + +--- + +## Open Questions + +1. **Model tiers:** Should Call B (clustering) explicitly use a lighter model, or always use the provider's primary? Recommendation: use primary for all three in v1; add per-call model override in a follow-up. +2. **Output folder:** Should there be a plugin setting for output path, or always write to vault root? Recommendation: vault root in v1, add setting later. +3. **Minimum block count:** Should the command be disabled when fewer than N blocks exist? Recommendation: warn at < 3 blocks, don't hard-block. +4. **Web app version:** A download variant (instead of vault write) would make this feature available outside Obsidian. Scoped out of this issue β€” separate backlog item if desired. From 8a605d26910cb3257167ec98445c5755a5f2c714 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 18:44:44 +0800 Subject: [PATCH 02/14] feat: implement Synthesis Document generation pipeline (#027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-phase AI pipeline that consolidates sparse nodespace notes into a structured, contextualized markdown document: - lib/synthesis.ts - buildEdgeMap(): Phase 1 β€” serialises block graph from influencedBy edges; separates reference nodes as source anchors - callDecontextualize(): Call A β€” rewrites each node into a self-contained statement using only its graph neighbours and source anchor as context (no external knowledge fill) - callCluster(): Call B β€” groups nodes into named sections by semantic meaning; post-processes to ensure every ID is assigned to exactly one section, missing nodes go to "General" - callSynthesize(): Call C β€” writes section headings, intros, expounding prompts (Socratic, not recall), and gap markers; nodeIds merged from clustering step by index - generateSynthesisDocument(): orchestrates Phases 1–3 with calls A+B in parallel then call C sequential - lib/synthesis-export.ts - renderSynthesisDocument(): Phase 3 renderer β€” YAML frontmatter, self-contained statements with source attribution, expounding prompts and gaps as Obsidian callout blocks - components/vim-input.tsx - Add "Synthesis document" action to command palette - pluginOnly: false (web-app only for now; plugin port later) - app/page.tsx - Handle "synthesis-doc" command: captures project snapshot synchronously then runs async pipeline, downloads result - synthesisStatus state drives a spinner toast during generation Co-Authored-By: Claude Sonnet 4.6 --- app/page.tsx | 55 +++++- components/vim-input.tsx | 7 +- lib/synthesis-export.ts | 84 +++++++++ lib/synthesis.ts | 382 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 524 insertions(+), 4 deletions(-) create mode 100644 lib/synthesis-export.ts create mode 100644 lib/synthesis.ts diff --git a/app/page.tsx b/app/page.tsx index ab4433f..ee8f321 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -19,6 +19,8 @@ import { generateGhostClient } from "@/lib/ai-ghost" import { exportToMarkdown, downloadMarkdown, copyToClipboard } from "@/lib/export" import { downloadNodepadFile, parseNodepadFile, NodepadParseError } from "@/lib/nodepad-format" import { detectContentType } from "@/lib/detect-content-type" +import { generateSynthesisDocument } from "@/lib/synthesis" +import { renderSynthesisDocument, slugifySynthesis } from "@/lib/synthesis-export" function generateId() { return Math.random().toString(36).substring(2, 10) @@ -58,6 +60,7 @@ export default function Page() { // ── Undo history ring (max 20 block snapshots per project) ─────────────── const blockHistoryRef = useRef>({}) const [undoToast, setUndoToast] = useState(null) + const [synthesisStatus, setSynthesisStatus] = useState(null) const undoToastTimer = useRef(null) const pushHistory = useCallback((projectId: string, currentBlocks: TextBlock[]) => { @@ -878,8 +881,34 @@ export default function Page() { } return prev }) + } else if (cmd === "synthesis-doc") { + // Capture current project snapshot synchronously, then run async pipeline + let blocks: TextBlock[] | null = null + let projName = "" + setProjects(prev => { + const proj = prev.find(p => p.id === activeProjectId) + if (proj) { blocks = proj.blocks; projName = proj.name } + return prev + }) + if (blocks) { + const captured = blocks as TextBlock[] + const name = projName + setSynthesisStatus("Building note graph…") + generateSynthesisDocument(captured, setSynthesisStatus) + .then(({ outline, decontextualized, clusters }) => { + const md = renderSynthesisDocument(name, outline, decontextualized, clusters) + const slug = slugifySynthesis(name) + downloadMarkdown(`${slug}-synthesis.md`, md) + setSynthesisStatus(null) + }) + .catch((err: Error) => { + console.error("[synthesis]", err) + setSynthesisStatus(`Error: ${err.message}`) + setTimeout(() => setSynthesisStatus(null), 5000) + }) + } } - + // Handle type overrides else if (cmd === "task" && text) addBlock(text, "task") else if (cmd === "thesis" && text) addBlock(text, "thesis") @@ -1028,6 +1057,30 @@ export default function Page() { /> + {/* Synthesis status toast */} + + {synthesisStatus && ( + +
+ {!synthesisStatus.startsWith("Error") && ( + + )} + {synthesisStatus} +
+ + )} + + {/* Undo toast */} {undoToast && ( diff --git a/components/vim-input.tsx b/components/vim-input.tsx index b0d2c3c..cd50ecf 100644 --- a/components/vim-input.tsx +++ b/components/vim-input.tsx @@ -5,7 +5,7 @@ import { motion, AnimatePresence } from "framer-motion" import { Trello, Grid, Trash2, Clipboard, Download, FolderOpen, FolderPlus, BookOpen, Sparkles, - FolderDown, FolderInput, GitFork + FolderDown, FolderInput, GitFork, ScrollText } from "lucide-react" import { Command } from "cmdk" import { useModKey } from "@/lib/utils" @@ -14,8 +14,9 @@ const ALL_ACTION_ITEMS = [ { id: "export-nodepad", icon: FolderDown, label: "Export", sub: ".nodepad", pluginOnly: false }, { id: "import-nodepad", icon: FolderInput, label: "Import", sub: ".nodepad", pluginOnly: false }, { id: "export-md", icon: Download, label: "Export", sub: "markdown", pluginOnly: true }, - { id: "copy-md", icon: Clipboard, label: "Copy", sub: "markdown", pluginOnly: true }, - { id: "clear", icon: Trash2, label: "Clear", sub: "canvas", pluginOnly: true }, + { id: "copy-md", icon: Clipboard, label: "Copy", sub: "markdown", pluginOnly: true }, + { id: "synthesis-doc", icon: ScrollText, label: "Synthesis", sub: "document", pluginOnly: false }, + { id: "clear", icon: Trash2, label: "Clear", sub: "canvas", pluginOnly: true }, ] // ─── Props ─────────────────────────────────────────────────────────────────── diff --git a/lib/synthesis-export.ts b/lib/synthesis-export.ts new file mode 100644 index 0000000..9b4ee83 --- /dev/null +++ b/lib/synthesis-export.ts @@ -0,0 +1,84 @@ +import type { DecontextualizedNode, ClusterAssignment, SynthesisOutline } from "@/lib/synthesis" + +function formatDate(): string { + return new Date().toLocaleDateString("en-GB", { day: "numeric", month: "long", year: "numeric" }) +} + +export function slugifySynthesis(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, "") + .trim() + .replace(/\s+/g, "-") + .slice(0, 50) +} + +export function renderSynthesisDocument( + canvasName: string, + outline: SynthesisOutline, + decontextualized: DecontextualizedNode[], + clusters: ClusterAssignment[], +): string { + const date = formatDate() + const totalNodes = clusters.reduce((sum, c) => sum + c.nodeIds.length, 0) + const stmtMap = new Map(decontextualized.map(n => [n.id, n.statement])) + + const lines: string[] = [] + + // ── YAML front matter ─────────────────────────────────────────────────────── + lines.push(`---`) + lines.push(`title: "Synthesis: ${canvasName}"`) + lines.push(`tags: [synthesis, nodepad]`) + lines.push(`created: ${new Date().toISOString().slice(0, 10)}`) + lines.push(`source: nodepad`) + lines.push(`---`) + lines.push(``) + + // ── Title + summary ───────────────────────────────────────────────────────── + lines.push(`# ${canvasName} β€” Synthesis`) + lines.push(``) + lines.push(`> ${outline.summary}`) + lines.push(``) + lines.push(`---`) + lines.push(``) + + // ── Sections ──────────────────────────────────────────────────────────────── + for (const section of outline.sections) { + lines.push(`## ${section.heading}`) + lines.push(``) + lines.push(section.intro) + lines.push(``) + + // Self-contained statements with source attribution + for (const nodeId of section.nodeIds) { + const stmt = stmtMap.get(nodeId) + if (!stmt) continue + lines.push(stmt) + lines.push(`> *Source: node \`${nodeId.slice(0, 8)}\`*`) + lines.push(``) + } + + // Expounding prompts as a collapsible callout + if (section.expandingPrompts.length > 0) { + lines.push(`> [!example]- Expounding Prompts`) + section.expandingPrompts.forEach((p, i) => lines.push(`> ${i + 1}. ${p}`)) + lines.push(``) + } + + // Gaps + if (section.gaps.length > 0) { + lines.push(`> [!question]- Gaps in your notes`) + section.gaps.forEach(g => lines.push(`> - ${g}`)) + lines.push(``) + } + + lines.push(`---`) + lines.push(``) + } + + // ── Footer ────────────────────────────────────────────────────────────────── + lines.push(`*Generated by [nodepad](https://nodepad.space) from ${totalNodes} node${totalNodes !== 1 ? "s" : ""} Β· ${date}*`) + lines.push(``) + + return lines.join("\n") +} diff --git a/lib/synthesis.ts b/lib/synthesis.ts new file mode 100644 index 0000000..6e9de5d --- /dev/null +++ b/lib/synthesis.ts @@ -0,0 +1,382 @@ +"use client" + +import { loadAIConfig, getBaseUrl, getProviderHeaders, type AIConfig } from "@/lib/ai-settings" +import type { TextBlock } from "@/components/tile-card" + +// ── Types ───────────────────────────────────────────────────────────────────── + +export interface NodeWithContext { + id: string + text: string + contentType: string + category?: string + annotation?: string + neighborIds: string[] + neighborTexts: string[] +} + +export interface EdgeMap { + nodes: NodeWithContext[] + sourceAnchors: NodeWithContext[] +} + +export interface DecontextualizedNode { + id: string + statement: string +} + +export interface ClusterAssignment { + sectionName: string + nodeIds: string[] +} + +export interface SynthesisSection { + heading: string + intro: string + nodeIds: string[] + expandingPrompts: string[] + gaps: string[] +} + +export interface SynthesisOutline { + summary: string + sections: SynthesisSection[] +} + +export interface SynthesisResult { + outline: SynthesisOutline + decontextualized: DecontextualizedNode[] + clusters: ClusterAssignment[] +} + +// ── Phase 1: Edge map builder ───────────────────────────────────────────────── + +export function buildEdgeMap(blocks: TextBlock[]): EdgeMap { + const byId = new Map(blocks.map(b => [b.id, b])) + const sourceAnchors: NodeWithContext[] = [] + const nodes: NodeWithContext[] = [] + + for (const block of blocks) { + const neighborIds = (block.influencedBy ?? []).filter(id => byId.has(id)) + const neighborTexts = neighborIds + .map(id => byId.get(id)!) + .map(n => [n.text, n.annotation].filter(Boolean).join(" β€” ")) + + const node: NodeWithContext = { + id: block.id, + text: block.text, + contentType: block.contentType, + category: block.category, + annotation: block.annotation, + neighborIds, + neighborTexts, + } + + if (block.contentType === "reference") { + sourceAnchors.push(node) + } else { + nodes.push(node) + } + } + + return { nodes, sourceAnchors } +} + +// ── Shared AI helpers ───────────────────────────────────────────────────────── + +function getTargetUrl(config: AIConfig): string { + const base = getBaseUrl(config) + return config.provider === "ollama" ? `${base}/api/chat` : `${base}/chat/completions` +} + +function buildPayload( + config: AIConfig, + messages: { role: string; content: string }[], + maxTokens: number, +) { + return config.provider === "ollama" + ? { model: config.modelId, messages, stream: false, options: { temperature: 0.2 } } + : { + model: config.modelId, + max_tokens: maxTokens, + messages, + response_format: { type: "json_object" }, + temperature: 0.2, + } +} + +async function callAI(config: AIConfig, targetUrl: string, payload: object): Promise { + const response = await fetch("/api/ai", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + url: targetUrl, + method: "POST", + headers: getProviderHeaders(config), + body: payload, + }), + }) + + if (!response.ok) { + const text = await response.text().catch(() => "") + throw new Error(`AI request failed (${response.status}): ${text.slice(0, 200)}`) + } + + const data = await response.json() + const content = config.provider === "ollama" + ? data.message?.content + : data.choices?.[0]?.message?.content + + if (!content) throw new Error("Empty response from AI provider") + return content as string +} + +function extractJson(text: string): string { + const fenced = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/) + if (fenced) return fenced[1].trim() + const arrStart = text.indexOf("[") + const arrEnd = text.lastIndexOf("]") + if (arrStart !== -1 && arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1) + const objStart = text.indexOf("{") + const objEnd = text.lastIndexOf("}") + if (objStart !== -1 && objEnd > objStart) return text.slice(objStart, objEnd + 1) + return text.trim() +} + +// ── Phase 2a: Decontextualization (Call A) ──────────────────────────────────── + +const DECONTEXTUALIZE_SYSTEM = `You rewrite sparse research notes into self-contained statements. + +RULES: +- Use ONLY the note's text, annotation, and provided neighboring notes as context +- Do NOT draw on external knowledge β€” if a note is too sparse to expand, return its text unchanged +- Resolve pronouns, abbreviations, and topic shortcuts using only the provided context +- One statement per note β€” a single clear sentence or short paragraph +- Preserve factual meaning exactly; do not add claims absent from the context + +OUTPUT: You MUST return a JSON array with one entry per input note, in the same order: +[{ "id": "...", "statement": "..." }, ...]` + +export async function callDecontextualize( + edgeMap: EdgeMap, + config: AIConfig, +): Promise { + const sourceCtx = edgeMap.sourceAnchors.length > 0 + ? `SOURCE MATERIAL:\n${edgeMap.sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` + : "" + + const notesJson = JSON.stringify( + edgeMap.nodes.map(n => ({ + id: n.id, + text: n.text, + annotation: n.annotation ?? "", + neighbors: n.neighborTexts, + })), + null, 2, + ) + + const targetUrl = getTargetUrl(config) + const payload = buildPayload(config, [ + { role: "system", content: DECONTEXTUALIZE_SYSTEM }, + { role: "user", content: `${sourceCtx}Notes to expand:\n${notesJson}` }, + ], 6000) + + const raw = await callAI(config, targetUrl, payload) + + let parsed: DecontextualizedNode[] = [] + try { + parsed = JSON.parse(extractJson(raw)) + } catch { + return edgeMap.nodes.map(n => ({ id: n.id, statement: n.text })) + } + + const resultMap = new Map(parsed.map(n => [n.id, n.statement])) + return edgeMap.nodes.map(n => ({ + id: n.id, + statement: resultMap.get(n.id) || n.text, + })) +} + +// ── Phase 2b: Clustering (Call B) ───────────────────────────────────────────── + +const CLUSTER_SYSTEM = `You group research notes into semantically coherent sections. + +RULES: +- Group by meaning and conceptual relationship β€” connection hints guide but do not dictate grouping +- Notes with no connections go into "General" unless they clearly fit an existing section +- Aim for 3–7 sections; merge thin sections rather than leaving singletons +- Every note ID must appear in exactly one section +- Name each section clearly (2–5 words) + +OUTPUT: You MUST return a JSON array: +[{ "sectionName": "...", "nodeIds": ["id1", "id2"] }, ...]` + +export async function callCluster( + edgeMap: EdgeMap, + config: AIConfig, +): Promise { + const sourceCtx = edgeMap.sourceAnchors.length > 0 + ? `SOURCE MATERIAL:\n${edgeMap.sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` + : "" + + const notesJson = JSON.stringify( + edgeMap.nodes.map(n => ({ + id: n.id, + text: n.text, + category: n.category ?? "", + contentType: n.contentType, + connectedIds: n.neighborIds, + })), + null, 2, + ) + + const targetUrl = getTargetUrl(config) + const payload = buildPayload(config, [ + { role: "system", content: CLUSTER_SYSTEM }, + { role: "user", content: `${sourceCtx}Notes to cluster:\n${notesJson}` }, + ], 2000) + + const raw = await callAI(config, targetUrl, payload) + + let parsed: ClusterAssignment[] = [] + try { + parsed = JSON.parse(extractJson(raw)) + } catch { + return [{ sectionName: "Notes", nodeIds: edgeMap.nodes.map(n => n.id) }] + } + + // Ensure every node appears in exactly one section + const seen = new Set() + const allIds = new Set(edgeMap.nodes.map(n => n.id)) + const cleaned: ClusterAssignment[] = [] + + for (const section of parsed) { + const unique = (section.nodeIds ?? []).filter(id => allIds.has(id) && !seen.has(id)) + unique.forEach(id => seen.add(id)) + if (unique.length > 0) cleaned.push({ sectionName: section.sectionName, nodeIds: unique }) + } + + const missing = edgeMap.nodes.map(n => n.id).filter(id => !seen.has(id)) + if (missing.length > 0) { + const gi = cleaned.findIndex(s => s.sectionName === "General") + if (gi >= 0) cleaned[gi].nodeIds.push(...missing) + else cleaned.push({ sectionName: "General", nodeIds: missing }) + } + + return cleaned +} + +// ── Phase 2c: Synthesis (Call C) ───────────────────────────────────────────── + +const SYNTHESIZE_SYSTEM = `You generate a Synthesis Document from research notes. + +A Synthesis Document consolidates fragmented notes into a coherent document about the ideas +and concepts captured in a nodespace. It is NOT a summary β€” it contextualises notes and +pushes thinking outward through open, exploratory questions. + +GROUNDING RULES β€” CRITICAL: +- Only use the provided statements and source material as your knowledge base +- Do NOT supplement with external knowledge +- Where notes are insufficient, flag as a gap explicitly + +For each section, produce: +1. A clear section heading (improve the candidate name if needed) +2. A 1–2 sentence intro: what should the reader understand from this section? +3. 2–3 expounding prompts β€” open questions that push thinking into adjacent territory the + notes do NOT cover. These invite exploration, not recall. + GOOD: "Are there economic systems that critique capital accumulation without requiring collective ownership?" + BAD: "What is the labour theory of value?" (that tests recall, not exploration) +4. A gaps list: concepts implied by the notes but not explained within them (empty array if none) + +Also produce a 2–3 sentence summary of the entire nodespace. + +OUTPUT: You MUST return ONLY valid JSON (no markdown, no explanation): +{ + "summary": "...", + "sections": [{ + "heading": "...", + "intro": "...", + "expandingPrompts": ["...", "..."], + "gaps": ["..."] + }] +}` + +export async function callSynthesize( + clusters: ClusterAssignment[], + decontextualized: DecontextualizedNode[], + sourceAnchors: NodeWithContext[], + config: AIConfig, +): Promise { + const stmtMap = new Map(decontextualized.map(n => [n.id, n.statement])) + const sourceCtx = sourceAnchors.length > 0 + ? `SOURCE MATERIAL:\n${sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` + : "" + + const sectionsJson = JSON.stringify( + clusters.map(c => ({ + candidateHeading: c.sectionName, + statements: c.nodeIds + .map(id => stmtMap.get(id)) + .filter(Boolean), + })), + null, 2, + ) + + const targetUrl = getTargetUrl(config) + const payload = buildPayload(config, [ + { role: "system", content: SYNTHESIZE_SYSTEM }, + { role: "user", content: `${sourceCtx}Sections to synthesise:\n${sectionsJson}` }, + ], 4000) + + const raw = await callAI(config, targetUrl, payload) + + let parsed: { summary: string; sections: Omit[] } + try { + parsed = JSON.parse(extractJson(raw)) + if (!parsed.summary || !Array.isArray(parsed.sections)) throw new Error("bad shape") + } catch { + throw new Error("Synthesis: could not parse AI response. Try again.") + } + + // Merge AI-generated prose with node IDs from the clustering step (by index) + const sections: SynthesisSection[] = parsed.sections.map((s, i) => ({ + heading: s.heading ?? clusters[i]?.sectionName ?? `Section ${i + 1}`, + intro: s.intro ?? "", + nodeIds: clusters[i]?.nodeIds ?? [], + expandingPrompts: s.expandingPrompts ?? [], + gaps: s.gaps ?? [], + })) + + return { summary: parsed.summary, sections } +} + +// ── Main orchestration ───────────────────────────────────────────────────────── + +export async function generateSynthesisDocument( + blocks: TextBlock[], + onProgress?: (step: string) => void, +): Promise { + const config = loadAIConfig() + if (!config) throw new Error("No AI provider configured. Add an API key in Settings.") + + const enrichedBlocks = blocks.filter(b => !b.isEnriching && !b.isError) + if (enrichedBlocks.length === 0) throw new Error("No notes to synthesise. Add some notes to the canvas first.") + + onProgress?.("Building note graph…") + const edgeMap = buildEdgeMap(enrichedBlocks) + + if (edgeMap.nodes.length === 0) { + throw new Error("All notes are reference nodes. Add content notes to the canvas.") + } + + onProgress?.("Decontextualising and clustering notes…") + const [decontextualized, clusters] = await Promise.all([ + callDecontextualize(edgeMap, config), + callCluster(edgeMap, config), + ]) + + onProgress?.("Writing synthesis…") + const outline = await callSynthesize(clusters, decontextualized, edgeMap.sourceAnchors, config) + + return { outline, decontextualized, clusters } +} From b0a2dbae2474c998c96b719db9cbd6653b896dbf Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 18:59:18 +0800 Subject: [PATCH 03/14] fix: read project via projectsRef in synthesis-doc handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous setProjects(prev => ...) trick for reading state is not reliable in React 18 concurrent mode β€” the updater is not guaranteed to run before the subsequent if (blocks) check. projectsRef.current is already the established pattern in this file for reading the latest project state synchronously inside callbacks (see enrichBlock, generateGhost, moveBlock, copyBlock). Use it here for consistency and correctness. Co-Authored-By: Claude Sonnet 4.6 --- app/page.tsx | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/app/page.tsx b/app/page.tsx index ee8f321..15574b6 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -882,19 +882,12 @@ export default function Page() { return prev }) } else if (cmd === "synthesis-doc") { - // Capture current project snapshot synchronously, then run async pipeline - let blocks: TextBlock[] | null = null - let projName = "" - setProjects(prev => { - const proj = prev.find(p => p.id === activeProjectId) - if (proj) { blocks = proj.blocks; projName = proj.name } - return prev - }) - if (blocks) { - const captured = blocks as TextBlock[] - const name = projName + const proj = projectsRef.current.find(p => p.id === activeProjectId) + if (proj) { + const blocks = proj.blocks + const name = proj.name setSynthesisStatus("Building note graph…") - generateSynthesisDocument(captured, setSynthesisStatus) + generateSynthesisDocument(blocks, setSynthesisStatus) .then(({ outline, decontextualized, clusters }) => { const md = renderSynthesisDocument(name, outline, decontextualized, clusters) const slug = slugifySynthesis(name) From 7f1dc3ca60df622f5dbb4e00156ce42ecaf3bfb5 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 19:12:32 +0800 Subject: [PATCH 04/14] fix(synthesis): log raw response and harden callSynthesize parsing - Log first 1000 chars of raw response before parsing so parse failures show what the AI actually returned - Wrap parse error to include the raw response start in the message - Normalise common alternative top-level keys (data, synthesis, overview, description) so models that wrap the output don't fail - Error message now includes the actual keys the AI returned, making the next debugging round self-diagnosing Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis.ts | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/synthesis.ts b/lib/synthesis.ts index 6e9de5d..8371c5d 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -329,17 +329,37 @@ export async function callSynthesize( ], 4000) const raw = await callAI(config, targetUrl, payload) + console.log("[synthesis] callSynthesize raw response:", raw.slice(0, 1000)) - let parsed: { summary: string; sections: Omit[] } + let parsed: { summary?: string; sections?: Omit[] } try { parsed = JSON.parse(extractJson(raw)) - if (!parsed.summary || !Array.isArray(parsed.sections)) throw new Error("bad shape") - } catch { - throw new Error("Synthesis: could not parse AI response. Try again.") + } catch (e) { + console.error("[synthesis] callSynthesize JSON parse failed. Raw:", raw.slice(0, 500)) + throw new Error(`Synthesis: AI returned non-JSON. Raw start: ${raw.slice(0, 120)}`) + } + + // Normalise: some models wrap the output under a top-level key + const root = (parsed as Record) + const normalisedSections = + Array.isArray(root.sections) ? root.sections as Omit[] : + Array.isArray(root.data) ? root.data as Omit[] : + Array.isArray(root.synthesis) ? root.synthesis as Omit[] : + null + + const normalisedSummary = + typeof root.summary === "string" ? root.summary : + typeof root.overview === "string" ? root.overview : + typeof root.description === "string" ? root.description : + null + + if (!normalisedSummary || !normalisedSections) { + console.error("[synthesis] callSynthesize unexpected shape. Keys:", Object.keys(root)) + throw new Error(`Synthesis: unexpected response shape. Keys found: ${Object.keys(root).join(", ")}`) } // Merge AI-generated prose with node IDs from the clustering step (by index) - const sections: SynthesisSection[] = parsed.sections.map((s, i) => ({ + const sections: SynthesisSection[] = normalisedSections.map((s, i) => ({ heading: s.heading ?? clusters[i]?.sectionName ?? `Section ${i + 1}`, intro: s.intro ?? "", nodeIds: clusters[i]?.nodeIds ?? [], @@ -347,7 +367,7 @@ export async function callSynthesize( gaps: s.gaps ?? [], })) - return { summary: parsed.summary, sections } + return { summary: normalisedSummary, sections } } // ── Main orchestration ───────────────────────────────────────────────────────── From 073566ea31b8416ea434e6ee63822c39781b4df7 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 19:13:57 +0800 Subject: [PATCH 05/14] fix(synthesis): dump raw AI response to file on parse failure On any JSON parse or shape error in callSynthesize, download the raw response as synthesis-call-c-raw.txt instead of logging to console. Makes the actual AI output inspectable without DevTools. Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/synthesis.ts b/lib/synthesis.ts index 8371c5d..3cfe00f 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -131,6 +131,16 @@ async function callAI(config: AIConfig, targetUrl: string, payload: object): Pro return content as string } +function dumpRawResponse(label: string, raw: string): void { + const blob = new Blob([raw], { type: "text/plain;charset=utf-8" }) + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = `${label}-raw.txt` + a.click() + URL.revokeObjectURL(url) +} + function extractJson(text: string): string { const fenced = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/) if (fenced) return fenced[1].trim() @@ -329,14 +339,13 @@ export async function callSynthesize( ], 4000) const raw = await callAI(config, targetUrl, payload) - console.log("[synthesis] callSynthesize raw response:", raw.slice(0, 1000)) let parsed: { summary?: string; sections?: Omit[] } try { parsed = JSON.parse(extractJson(raw)) - } catch (e) { - console.error("[synthesis] callSynthesize JSON parse failed. Raw:", raw.slice(0, 500)) - throw new Error(`Synthesis: AI returned non-JSON. Raw start: ${raw.slice(0, 120)}`) + } catch { + dumpRawResponse("synthesis-call-c", raw) + throw new Error("Synthesis: AI returned non-JSON. Raw response saved to synthesis-call-c-raw.txt") } // Normalise: some models wrap the output under a top-level key @@ -354,8 +363,8 @@ export async function callSynthesize( null if (!normalisedSummary || !normalisedSections) { - console.error("[synthesis] callSynthesize unexpected shape. Keys:", Object.keys(root)) - throw new Error(`Synthesis: unexpected response shape. Keys found: ${Object.keys(root).join(", ")}`) + dumpRawResponse("synthesis-call-c", raw) + throw new Error(`Synthesis: unexpected response shape (keys: ${Object.keys(root).join(", ")}). Raw response saved to synthesis-call-c-raw.txt`) } // Merge AI-generated prose with node IDs from the clustering step (by index) From fc2d059adb4f50b5eaa2acb157edb042bc7b9695 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 19:21:19 +0800 Subject: [PATCH 06/14] fix(synthesis): handle array-at-root response from callSynthesize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini returned the sections array directly at the top level instead of wrapping it in { summary, sections }. Keys 0,1,2,3,4 in the error confirm this β€” Object.keys on an array returns its indices. Now detects Array.isArray(parsed) and uses the array as sections directly with an empty summary, rather than failing the shape check. Also widens the object-path to check data/synthesis as section keys and overview/description as summary keys. Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis.ts | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/lib/synthesis.ts b/lib/synthesis.ts index 3cfe00f..a72fdde 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -348,23 +348,35 @@ export async function callSynthesize( throw new Error("Synthesis: AI returned non-JSON. Raw response saved to synthesis-call-c-raw.txt") } - // Normalise: some models wrap the output under a top-level key - const root = (parsed as Record) - const normalisedSections = - Array.isArray(root.sections) ? root.sections as Omit[] : - Array.isArray(root.data) ? root.data as Omit[] : - Array.isArray(root.synthesis) ? root.synthesis as Omit[] : - null - - const normalisedSummary = - typeof root.summary === "string" ? root.summary : - typeof root.overview === "string" ? root.overview : - typeof root.description === "string" ? root.description : - null - - if (!normalisedSummary || !normalisedSections) { + // Normalise: handle both array-at-root and object shapes + type RawSection = Omit + type RootObj = Record + + let normalisedSummary: string | null = null + let normalisedSections: RawSection[] | null = null + + if (Array.isArray(parsed)) { + // Model returned the sections array directly β€” no summary + normalisedSections = parsed as RawSection[] + normalisedSummary = "" + } else { + const root = parsed as RootObj + normalisedSections = + Array.isArray(root.sections) ? root.sections as RawSection[] : + Array.isArray(root.data) ? root.data as RawSection[] : + Array.isArray(root.synthesis) ? root.synthesis as RawSection[] : + null + + normalisedSummary = + typeof root.summary === "string" ? root.summary : + typeof root.overview === "string" ? root.overview : + typeof root.description === "string" ? root.description : + null + } + + if (!normalisedSections) { dumpRawResponse("synthesis-call-c", raw) - throw new Error(`Synthesis: unexpected response shape (keys: ${Object.keys(root).join(", ")}). Raw response saved to synthesis-call-c-raw.txt`) + throw new Error(`Synthesis: unexpected response shape (keys: ${Object.keys(parsed as object).join(", ")}). Raw response saved to synthesis-call-c-raw.txt`) } // Merge AI-generated prose with node IDs from the clustering step (by index) @@ -376,7 +388,7 @@ export async function callSynthesize( gaps: s.gaps ?? [], })) - return { summary: normalisedSummary, sections } + return { summary: normalisedSummary ?? "", sections } } // ── Main orchestration ───────────────────────────────────────────────────────── From 405324afe9b63845461f889cb2b9ddf9baa0ea74 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 20:27:59 +0800 Subject: [PATCH 07/14] feat(synthesis): full pipeline redesign with progress UI and polish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline changes: - Replace single callSynthesize with callSynthesizeCluster (Call CΓ—N): parallel per-cluster synthesis, each receiving full nodespace context (all statements + annotations + document structure) steered to its target section. Parallel processes instructed not to explain or prompt about other sections. - Add callPolish (Call D): optional sequential editorial pass over the rendered draft markdown, forbidden from touching source attributions or callout blocks - generateSynthesisDocument now emits structured ProgressEvents (phase_start, phase_done, clusters_known, error) and returns CallTiming[] for the footer; Call D handled by page.tsx - SynthesisSection gains sectionSynthesis (flowing prose paragraphs) - SynthesisOutline drops top-level summary (no Call D for that) Renderer changes (synthesis-export.ts): - sectionSynthesis prose leads each section - Source notes moved to collapsible [!note]- callout - Timing table in collapsible [!info]- footer on both files - renderPolishedDocument() appends timing footer to Call D output - isPolished flag tags frontmatter and title New components: - SynthesisConfirmDialog: pre-generation modal showing source anchors (or warning if none), node count, polish toggle, and warnings about node snapshot, token costs, and time estimates - SynthesisProgressPanel: clickable bottom pill that opens a progress dialog with live per-call status, elapsed timers for running calls, parallel/sequential grouping, and completion summary page.tsx wiring: - synthesis-doc command opens confirm dialog (no immediate generation) - startSynthesis(enablePolish) captures project snapshot, runs pipeline, downloads raw .md, optionally runs Call D and downloads polished .md - Synthesis state replaces old synthesisStatus string with structured CallTiming[] and isActive/isDialogOpen booleans Co-Authored-By: Claude Sonnet 4.6 --- app/page.tsx | 174 +++++++++--- components/synthesis-confirm-dialog.tsx | 171 +++++++++++ components/synthesis-progress-panel.tsx | 245 ++++++++++++++++ lib/synthesis-export.ts | 97 +++++-- lib/synthesis.ts | 359 +++++++++++++++--------- 5 files changed, 863 insertions(+), 183 deletions(-) create mode 100644 components/synthesis-confirm-dialog.tsx create mode 100644 components/synthesis-progress-panel.tsx diff --git a/app/page.tsx b/app/page.tsx index 15574b6..473ac4b 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -13,14 +13,26 @@ import { IntroModal } from "@/components/intro-modal" import type { TextBlock } from "@/components/tile-card" import type { ContentType } from "@/lib/content-types" import { INITIAL_PROJECTS } from "@/lib/initial-data" -import { useAISettings } from "@/lib/ai-settings" +import { useAISettings, loadAIConfig } from "@/lib/ai-settings" import { enrichBlockClient } from "@/lib/ai-enrich" import { generateGhostClient } from "@/lib/ai-ghost" import { exportToMarkdown, downloadMarkdown, copyToClipboard } from "@/lib/export" import { downloadNodepadFile, parseNodepadFile, NodepadParseError } from "@/lib/nodepad-format" import { detectContentType } from "@/lib/detect-content-type" -import { generateSynthesisDocument } from "@/lib/synthesis" -import { renderSynthesisDocument, slugifySynthesis } from "@/lib/synthesis-export" +import { + generateSynthesisDocument, + callPolish, + getSourceAnchors, + type CallTiming, + type ProgressEvent, +} from "@/lib/synthesis" +import { + renderSynthesisDocument, + renderPolishedDocument, + slugifySynthesis, +} from "@/lib/synthesis-export" +import { SynthesisConfirmDialog } from "@/components/synthesis-confirm-dialog" +import { SynthesisProgressPanel } from "@/components/synthesis-progress-panel" function generateId() { return Math.random().toString(36).substring(2, 10) @@ -60,9 +72,18 @@ export default function Page() { // ── Undo history ring (max 20 block snapshots per project) ─────────────── const blockHistoryRef = useRef>({}) const [undoToast, setUndoToast] = useState(null) - const [synthesisStatus, setSynthesisStatus] = useState(null) const undoToastTimer = useRef(null) + // ── Synthesis state ───────────────────────────────────────────────────────── + const [synthConfirmOpen, setSynthConfirmOpen] = useState(false) + const [synthProgressOpen, setSynthProgressOpen] = useState(false) + const [synthCalls, setSynthCalls] = useState([]) + const [synthActive, setSynthActive] = useState(false) + const [synthTotalStart, setSynthTotalStart] = useState(undefined) + const [synthSourceAnchors, setSynthSourceAnchors] = useState([]) + const [synthBlockCount, setSynthBlockCount] = useState(0) + const synthEnablePolish = useRef(false) + const pushHistory = useCallback((projectId: string, currentBlocks: TextBlock[]) => { if (!blockHistoryRef.current[projectId]) blockHistoryRef.current[projectId] = [] const stack = blockHistoryRef.current[projectId] @@ -884,21 +905,9 @@ export default function Page() { } else if (cmd === "synthesis-doc") { const proj = projectsRef.current.find(p => p.id === activeProjectId) if (proj) { - const blocks = proj.blocks - const name = proj.name - setSynthesisStatus("Building note graph…") - generateSynthesisDocument(blocks, setSynthesisStatus) - .then(({ outline, decontextualized, clusters }) => { - const md = renderSynthesisDocument(name, outline, decontextualized, clusters) - const slug = slugifySynthesis(name) - downloadMarkdown(`${slug}-synthesis.md`, md) - setSynthesisStatus(null) - }) - .catch((err: Error) => { - console.error("[synthesis]", err) - setSynthesisStatus(`Error: ${err.message}`) - setTimeout(() => setSynthesisStatus(null), 5000) - }) + setSynthSourceAnchors(getSourceAnchors(proj.blocks)) + setSynthBlockCount(proj.blocks.length) + setSynthConfirmOpen(true) } } @@ -909,6 +918,92 @@ export default function Page() { setIsCommandKOpen(false) }, [clearBlocks, addBlock, activeProjectId]) + // ── Synthesis: start generation after confirm dialog ───────────────────────── + const startSynthesis = useCallback((enablePolish: boolean) => { + const proj = projectsRef.current.find(p => p.id === activeProjectId) + if (!proj) return + + synthEnablePolish.current = enablePolish + setSynthConfirmOpen(false) + setSynthCalls([]) + setSynthActive(true) + setSynthTotalStart(Date.now()) + setSynthProgressOpen(false) + + const blocks = proj.blocks + const name = proj.name + const slug = slugifySynthesis(name) + + const onProgress = (event: ProgressEvent) => { + setSynthCalls(prev => { + const next = [...prev] + if (event.type === "phase_start") { + next.push({ + id: event.id, label: event.label, status: "running", + startTime: Date.now(), + isParallel: event.id.startsWith("callC-") || event.id === "callA" || event.id === "callB", + }) + } else if (event.type === "phase_done") { + const t = next.find(t => t.id === event.id) + if (t) { t.status = "done"; t.durationMs = event.durationMs } + } else if (event.type === "error") { + const t = next.find(t => t.id === event.id) + if (t) t.status = "error" + } + return next + }) + } + + generateSynthesisDocument(blocks, onProgress) + .then(async ({ outline, decontextualized, clusters, timings }) => { + // Render raw (with timing footer) + const rawMd = renderSynthesisDocument(name, outline, decontextualized, clusters, timings, false) + downloadMarkdown(`${slug}-synthesis.md`, rawMd) + + if (enablePolish) { + // Register Call D in progress + const dStart = Date.now() + setSynthCalls(prev => [...prev, { + id: "callD", label: "Final editorial polish", + status: "running", startTime: dStart, isParallel: false, + }]) + + try { + const config = loadAIConfig() + if (!config) throw new Error("No AI config for polish") + + // Draft without timing footer for cleaner polish input + const draftForPolish = renderSynthesisDocument(name, outline, decontextualized, clusters, [], false) + const polishedText = await callPolish(draftForPolish, config) + const polishDuration = Date.now() - dStart + + setSynthCalls(prev => prev.map(c => + c.id === "callD" ? { ...c, status: "done", durationMs: polishDuration } : c + )) + + // Add Call D to timings for footer + const fullTimings: CallTiming[] = [ + ...timings, + { id: "callD", label: "Final editorial polish", status: "done", durationMs: polishDuration }, + ] + const polishedMd = renderPolishedDocument(polishedText, fullTimings) + downloadMarkdown(`${slug}-synthesis-polished.md`, polishedMd) + } catch (e) { + setSynthCalls(prev => prev.map(c => + c.id === "callD" ? { ...c, status: "error" } : c + )) + console.error("[synthesis polish]", e) + } + } + + setSynthActive(false) + }) + .catch((err: Error) => { + console.error("[synthesis]", err) + setSynthActive(false) + }) + }, [activeProjectId]) + return (
{/* Hidden file input for .nodepad import */} @@ -1050,29 +1145,24 @@ export default function Page() { />
- {/* Synthesis status toast */} - - {synthesisStatus && ( - -
- {!synthesisStatus.startsWith("Error") && ( - - )} - {synthesisStatus} -
- - )} - + {/* Synthesis confirm dialog */} + setSynthConfirmOpen(false)} + /> + + {/* Synthesis progress pill + dialog */} + setSynthProgressOpen(true)} + onDialogClose={() => setSynthProgressOpen(false)} + /> {/* Undo toast */} diff --git a/components/synthesis-confirm-dialog.tsx b/components/synthesis-confirm-dialog.tsx new file mode 100644 index 0000000..5229911 --- /dev/null +++ b/components/synthesis-confirm-dialog.tsx @@ -0,0 +1,171 @@ +"use client" + +import * as React from "react" +import { motion, AnimatePresence } from "framer-motion" +import { AlertTriangle, BookOpen, CheckSquare, Square, X } from "lucide-react" +import type { TextBlock } from "@/components/tile-card" + +interface SynthesisConfirmDialogProps { + isOpen: boolean + sourceAnchors: TextBlock[] + blockCount: number + onConfirm: (enablePolish: boolean) => void + onCancel: () => void +} + +export function SynthesisConfirmDialog({ + isOpen, + sourceAnchors, + blockCount, + onConfirm, + onCancel, +}: SynthesisConfirmDialogProps) { + const [enablePolish, setEnablePolish] = React.useState(false) + + // Reset polish toggle when dialog re-opens + React.useEffect(() => { + if (isOpen) setEnablePolish(false) + }, [isOpen]) + + if (!isOpen) return null + + const hasAnchors = sourceAnchors.length > 0 + + return ( + + {isOpen && ( + <> + {/* Backdrop */} + + + {/* Dialog */} + +
e.stopPropagation()} + > + {/* Header */} +
+
+

+ Generate Synthesis Document +

+

+ {blockCount} note{blockCount !== 1 ? "s" : ""} Β· {hasAnchors ? `${sourceAnchors.length} source anchor${sourceAnchors.length !== 1 ? "s" : ""} detected` : "no source anchors"} +

+
+ +
+ +
+ + {/* Source anchors */} +
+
+ + + Source Anchors + +
+ {hasAnchors ? ( +
    + {sourceAnchors.map(a => ( +
  • + {a.text.length > 80 ? a.text.slice(0, 80) + "…" : a.text} +
    • + ))} +
+ ) : ( +
+ +

+ No source reference nodes detected. Adding a{" "} + reference node naming your + source material significantly improves output quality. You can proceed + without one or cancel to add one first. +

+
+ )} +
+ + {/* Polish toggle */} +
+ +
+ + {/* Warnings */} +
+

+ Β·{" "} + Notes added after clicking Generate will not be included in this synthesis. +

+

+ Β·{" "} + Multiple parallel and sequential AI calls will fire simultaneously β€” expect + a spike in provider activity and corresponding token costs. +

+

+ Β·{" "} + Estimated time varies by provider and model. Expect ~5 minutes for the + raw document{enablePolish ? ", longer with polish enabled" : ""}. +

+
+
+ + {/* Actions */} +
+ + +
+
+ + + )} + + ) +} diff --git a/components/synthesis-progress-panel.tsx b/components/synthesis-progress-panel.tsx new file mode 100644 index 0000000..ffc8b04 --- /dev/null +++ b/components/synthesis-progress-panel.tsx @@ -0,0 +1,245 @@ +"use client" + +import * as React from "react" +import { motion, AnimatePresence } from "framer-motion" +import { CheckCircle, AlertCircle, X, ChevronUp } from "lucide-react" +import type { CallTiming } from "@/lib/synthesis" +import { formatDuration } from "@/lib/synthesis" + +interface SynthesisProgressPanelProps { + calls: CallTiming[] + isActive: boolean + totalStartMs?: number + isDialogOpen: boolean + onPillClick: () => void + onDialogClose: () => void +} + +// ── Elapsed timer for running calls ────────────────────────────────────────── + +function useElapsed(active: boolean): number { + const [now, setNow] = React.useState(Date.now()) + React.useEffect(() => { + if (!active) return + const id = setInterval(() => setNow(Date.now()), 1000) + return () => clearInterval(id) + }, [active]) + return now +} + +// ── Single call row ─────────────────────────────────────────────────────────── + +function CallRow({ call, now }: { call: CallTiming; now: number }) { + const elapsed = call.startTime ? now - call.startTime : 0 + + return ( +
+ {/* Status icon */} +
+ {call.status === "done" && ( + + )} + {call.status === "error" && ( + + )} + {call.status === "running" && ( + + )} + {call.status === "pending" && ( +
+ )} +
+ + {/* Label */} + + {call.isParallel && βˆ₯} + {call.label} + + + {/* Duration */} + + {call.status === "done" && call.durationMs != null ? formatDuration(call.durationMs) : + call.status === "running" && call.startTime ? `${Math.round(elapsed / 1000)}s…` : + call.status === "error" ? "error" : + "β€”"} + +
+ ) +} + +// ── Progress dialog ─────────────────────────────────────────────────────────── + +function ProgressDialog({ + calls, + isActive, + totalStartMs, + onClose, +}: { + calls: CallTiming[] + isActive: boolean + totalStartMs?: number + onClose: () => void +}) { + const now = useElapsed(isActive) + const doneCount = calls.filter(c => c.status === "done").length + const totalCount = calls.length + const totalElapsed = totalStartMs ? now - totalStartMs : 0 + + // Group: parallel calls together, sequential separate + const parallelCalls = calls.filter(c => c.isParallel) + const sequentialCalls = calls.filter(c => !c.isParallel) + + return ( + <> + + +
e.stopPropagation()} + > + {/* Header */} +
+
+

+ {isActive ? "Synthesis in progress" : "Synthesis complete"} +

+

+ {isActive + ? `${doneCount} of ${totalCount} calls done Β· ${Math.round(totalElapsed / 1000)}s elapsed` + : `${doneCount} calls Β· ${formatDuration(totalElapsed)}` + } +

+
+ +
+ + {/* Call list */} +
+ {parallelCalls.length > 0 && ( + <> +

+ Parallel calls +

+ {parallelCalls.map(c => )} + + )} + {parallelCalls.length > 0 && sequentialCalls.length > 0 && ( +
+ )} + {sequentialCalls.length > 0 && ( + <> +

+ Sequential calls +

+ {sequentialCalls.map(c => )} + + )} +
+ + {/* Legend */} +
+ βˆ₯ parallel + Β· sequential +
+
+ + + ) +} + +// ── Bottom pill ─────────────────────────────────────────────────────────────── + +export function SynthesisProgressPanel({ + calls, + isActive, + totalStartMs, + isDialogOpen, + onPillClick, + onDialogClose, +}: SynthesisProgressPanelProps) { + const now = useElapsed(isActive) + const doneCount = calls.filter(c => c.status === "done").length + const totalCount = calls.length + const hasError = calls.some(c => c.status === "error") + + const pillLabel = hasError + ? "Synthesis error β€” click for details" + : isActive + ? totalCount > 0 + ? `Synthesising β€” ${doneCount}/${totalCount} calls done` + : "Synthesising…" + : `Synthesis done β€” ${totalStartMs ? formatDuration(now - totalStartMs) : ""}` + + const showPill = calls.length > 0 + + return ( + <> + {/* Bottom pill */} + + {showPill && ( + + {isActive && !hasError && ( + + )} + {hasError && } + {!isActive && !hasError && } + + {pillLabel} + + + + )} + + + {/* Progress dialog */} + + {isDialogOpen && ( + + )} + + + ) +} diff --git a/lib/synthesis-export.ts b/lib/synthesis-export.ts index 9b4ee83..8fc1171 100644 --- a/lib/synthesis-export.ts +++ b/lib/synthesis-export.ts @@ -1,4 +1,10 @@ -import type { DecontextualizedNode, ClusterAssignment, SynthesisOutline } from "@/lib/synthesis" +import type { + DecontextualizedNode, + ClusterAssignment, + SynthesisOutline, + CallTiming, +} from "@/lib/synthesis" +import { formatDuration } from "@/lib/synthesis" function formatDate(): string { return new Date().toLocaleDateString("en-GB", { day: "numeric", month: "long", year: "numeric" }) @@ -13,11 +19,36 @@ export function slugifySynthesis(name: string): string { .slice(0, 50) } +function renderTimingTable(timings: CallTiming[], isPolished: boolean): string { + if (timings.length === 0) return "" + + const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) + const lines: string[] = [] + + lines.push(`> [!info]- Generation Statistics`) + lines.push(`> | Step | Duration |`) + lines.push(`> |---|---|`) + + for (const t of timings) { + const dur = t.durationMs != null ? formatDuration(t.durationMs) : "β€”" + const prefix = t.isParallel ? "*(parallel)* " : "" + lines.push(`> | ${prefix}${t.label} | ${dur} |`) + } + + lines.push(`> | **Total** | **${formatDuration(totalMs)}** |`) + if (isPolished) lines.push(`> `) + if (isPolished) lines.push(`> *Includes final editorial polish (Call D).*`) + + return lines.join("\n") +} + export function renderSynthesisDocument( canvasName: string, outline: SynthesisOutline, decontextualized: DecontextualizedNode[], clusters: ClusterAssignment[], + timings: CallTiming[], + isPolished = false, ): string { const date = formatDate() const totalNodes = clusters.reduce((sum, c) => sum + c.nodeIds.length, 0) @@ -27,17 +58,15 @@ export function renderSynthesisDocument( // ── YAML front matter ─────────────────────────────────────────────────────── lines.push(`---`) - lines.push(`title: "Synthesis: ${canvasName}"`) - lines.push(`tags: [synthesis, nodepad]`) + lines.push(`title: "${isPolished ? "Synthesis (Polished)" : "Synthesis"}: ${canvasName}"`) + lines.push(`tags: [synthesis, nodepad${isPolished ? ", polished" : ""}]`) lines.push(`created: ${new Date().toISOString().slice(0, 10)}`) lines.push(`source: nodepad`) lines.push(`---`) lines.push(``) - // ── Title + summary ───────────────────────────────────────────────────────── - lines.push(`# ${canvasName} β€” Synthesis`) - lines.push(``) - lines.push(`> ${outline.summary}`) + // ── Title ─────────────────────────────────────────────────────────────────── + lines.push(`# ${canvasName} β€” Synthesis${isPolished ? " (Polished)" : ""}`) lines.push(``) lines.push(`---`) lines.push(``) @@ -46,19 +75,33 @@ export function renderSynthesisDocument( for (const section of outline.sections) { lines.push(`## ${section.heading}`) lines.push(``) - lines.push(section.intro) - lines.push(``) - // Self-contained statements with source attribution - for (const nodeId of section.nodeIds) { - const stmt = stmtMap.get(nodeId) - if (!stmt) continue - lines.push(stmt) - lines.push(`> *Source: node \`${nodeId.slice(0, 8)}\`*`) + if (section.intro) { + lines.push(section.intro) lines.push(``) } - // Expounding prompts as a collapsible callout + // Synthesis prose β€” the main output + if (section.sectionSynthesis) { + lines.push(section.sectionSynthesis) + lines.push(``) + } + + // Source notes β€” collapsible reference + const sectionStatements = section.nodeIds + .map(id => ({ id, stmt: stmtMap.get(id) })) + .filter((s): s is { id: string; stmt: string } => !!s.stmt) + + if (sectionStatements.length > 0) { + lines.push(`> [!note]- Source Notes (${sectionStatements.length})`) + for (const { id, stmt } of sectionStatements) { + const truncated = stmt.length > 120 ? stmt.slice(0, 120) + "…" : stmt + lines.push(`> - ${truncated} *(node \`${id.slice(0, 8)}\`)*`) + } + lines.push(``) + } + + // Expounding prompts if (section.expandingPrompts.length > 0) { lines.push(`> [!example]- Expounding Prompts`) section.expandingPrompts.forEach((p, i) => lines.push(`> ${i + 1}. ${p}`)) @@ -76,9 +119,31 @@ export function renderSynthesisDocument( lines.push(``) } + // ── Timing footer ─────────────────────────────────────────────────────────── + if (timings.length > 0) { + lines.push(renderTimingTable(timings, isPolished)) + lines.push(``) + } + // ── Footer ────────────────────────────────────────────────────────────────── lines.push(`*Generated by [nodepad](https://nodepad.space) from ${totalNodes} node${totalNodes !== 1 ? "s" : ""} Β· ${date}*`) lines.push(``) return lines.join("\n") } + +export function renderPolishedDocument( + polishedText: string, + timings: CallTiming[], +): string { + const timingBlock = renderTimingTable(timings, true) + const footer = `\n${timingBlock}\n\n*Generated by [nodepad](https://nodepad.space) Β· ${formatDate()}*\n` + + // Append timing + footer before the last line if it already has a footer, + // otherwise just append + const lastFooterMatch = polishedText.match(/\n\*Generated by \[nodepad\][\s\S]*$/) + if (lastFooterMatch) { + return polishedText.slice(0, polishedText.lastIndexOf(lastFooterMatch[0])) + footer + } + return polishedText.trimEnd() + "\n\n" + footer +} diff --git a/lib/synthesis.ts b/lib/synthesis.ts index a72fdde..ea60eac 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -3,7 +3,24 @@ import { loadAIConfig, getBaseUrl, getProviderHeaders, type AIConfig } from "@/lib/ai-settings" import type { TextBlock } from "@/components/tile-card" -// ── Types ───────────────────────────────────────────────────────────────────── +// ── Progress events ─────────────────────────────────────────────────────────── + +export type ProgressEvent = + | { type: "phase_start"; id: string; label: string } + | { type: "phase_done"; id: string; durationMs: number } + | { type: "clusters_known"; clusterNames: string[] } + | { type: "error"; id: string; message: string } + +export interface CallTiming { + id: string + label: string + status: "pending" | "running" | "done" | "error" + startTime?: number + durationMs?: number + isParallel?: boolean +} + +// ── Data types ──────────────────────────────────────────────────────────────── export interface NodeWithContext { id: string @@ -33,13 +50,13 @@ export interface ClusterAssignment { export interface SynthesisSection { heading: string intro: string + sectionSynthesis: string nodeIds: string[] expandingPrompts: string[] gaps: string[] } export interface SynthesisOutline { - summary: string sections: SynthesisSection[] } @@ -47,6 +64,7 @@ export interface SynthesisResult { outline: SynthesisOutline decontextualized: DecontextualizedNode[] clusters: ClusterAssignment[] + timings: CallTiming[] } // ── Phase 1: Edge map builder ───────────────────────────────────────────────── @@ -57,7 +75,7 @@ export function buildEdgeMap(blocks: TextBlock[]): EdgeMap { const nodes: NodeWithContext[] = [] for (const block of blocks) { - const neighborIds = (block.influencedBy ?? []).filter(id => byId.has(id)) + const neighborIds = (block.influencedBy ?? []).filter(id => byId.has(id)) const neighborTexts = neighborIds .map(id => byId.get(id)!) .map(n => [n.text, n.annotation].filter(Boolean).join(" β€” ")) @@ -93,15 +111,17 @@ function buildPayload( config: AIConfig, messages: { role: string; content: string }[], maxTokens: number, + wantJson = true, ) { - return config.provider === "ollama" + const isOllama = config.provider === "ollama" + return isOllama ? { model: config.modelId, messages, stream: false, options: { temperature: 0.2 } } : { model: config.modelId, max_tokens: maxTokens, messages, - response_format: { type: "json_object" }, temperature: 0.2, + ...(wantJson ? { response_format: { type: "json_object" } } : {}), } } @@ -153,12 +173,21 @@ function extractJson(text: string): string { return text.trim() } -// ── Phase 2a: Decontextualization (Call A) ──────────────────────────────────── +export function formatDuration(ms: number): string { + if (ms < 1000) return `<1s` + const s = ms / 1000 + if (s < 60) return `${s.toFixed(1)}s` + const m = Math.floor(s / 60) + const rem = Math.round(s % 60) + return `${m}m ${rem}s` +} + +// ── Call A: Decontextualisation ─────────────────────────────────────────────── const DECONTEXTUALIZE_SYSTEM = `You rewrite sparse research notes into self-contained statements. RULES: -- Use ONLY the note's text, annotation, and provided neighboring notes as context +- Use ONLY the note's text, annotation, and provided neighbouring notes as context - Do NOT draw on external knowledge β€” if a note is too sparse to expand, return its text unchanged - Resolve pronouns, abbreviations, and topic shortcuts using only the provided context - One statement per note β€” a single clear sentence or short paragraph @@ -186,7 +215,7 @@ export async function callDecontextualize( ) const targetUrl = getTargetUrl(config) - const payload = buildPayload(config, [ + const payload = buildPayload(config, [ { role: "system", content: DECONTEXTUALIZE_SYSTEM }, { role: "user", content: `${sourceCtx}Notes to expand:\n${notesJson}` }, ], 6000) @@ -194,20 +223,13 @@ export async function callDecontextualize( const raw = await callAI(config, targetUrl, payload) let parsed: DecontextualizedNode[] = [] - try { - parsed = JSON.parse(extractJson(raw)) - } catch { - return edgeMap.nodes.map(n => ({ id: n.id, statement: n.text })) - } + try { parsed = JSON.parse(extractJson(raw)) } catch { /* fall through */ } const resultMap = new Map(parsed.map(n => [n.id, n.statement])) - return edgeMap.nodes.map(n => ({ - id: n.id, - statement: resultMap.get(n.id) || n.text, - })) + return edgeMap.nodes.map(n => ({ id: n.id, statement: resultMap.get(n.id) || n.text })) } -// ── Phase 2b: Clustering (Call B) ───────────────────────────────────────────── +// ── Call B: Clustering ──────────────────────────────────────────────────────── const CLUSTER_SYSTEM = `You group research notes into semantically coherent sections. @@ -241,7 +263,7 @@ export async function callCluster( ) const targetUrl = getTargetUrl(config) - const payload = buildPayload(config, [ + const payload = buildPayload(config, [ { role: "system", content: CLUSTER_SYSTEM }, { role: "user", content: `${sourceCtx}Notes to cluster:\n${notesJson}` }, ], 2000) @@ -249,13 +271,10 @@ export async function callCluster( const raw = await callAI(config, targetUrl, payload) let parsed: ClusterAssignment[] = [] - try { - parsed = JSON.parse(extractJson(raw)) - } catch { + try { parsed = JSON.parse(extractJson(raw)) } catch { return [{ sectionName: "Notes", nodeIds: edgeMap.nodes.map(n => n.id) }] } - // Ensure every node appears in exactly one section const seen = new Set() const allIds = new Set(edgeMap.nodes.map(n => n.id)) const cleaned: ClusterAssignment[] = [] @@ -276,126 +295,152 @@ export async function callCluster( return cleaned } -// ── Phase 2c: Synthesis (Call C) ───────────────────────────────────────────── - -const SYNTHESIZE_SYSTEM = `You generate a Synthesis Document from research notes. - -A Synthesis Document consolidates fragmented notes into a coherent document about the ideas -and concepts captured in a nodespace. It is NOT a summary β€” it contextualises notes and -pushes thinking outward through open, exploratory questions. - -GROUNDING RULES β€” CRITICAL: -- Only use the provided statements and source material as your knowledge base -- Do NOT supplement with external knowledge -- Where notes are insufficient, flag as a gap explicitly - -For each section, produce: -1. A clear section heading (improve the candidate name if needed) -2. A 1–2 sentence intro: what should the reader understand from this section? -3. 2–3 expounding prompts β€” open questions that push thinking into adjacent territory the - notes do NOT cover. These invite exploration, not recall. - GOOD: "Are there economic systems that critique capital accumulation without requiring collective ownership?" - BAD: "What is the labour theory of value?" (that tests recall, not exploration) -4. A gaps list: concepts implied by the notes but not explained within them (empty array if none) - -Also produce a 2–3 sentence summary of the entire nodespace. - -OUTPUT: You MUST return ONLY valid JSON (no markdown, no explanation): +// ── Call CΓ—N: Per-cluster synthesis ────────────────────────────────────────── + +const CLUSTER_SYNTHESIZE_SYSTEM = `You are one of several parallel processes each generating the synthesis for one section of a multi-section document. + +IMPORTANT β€” PARALLEL PROCESSES: +Other independent processes are simultaneously generating synthesis for every other section. +You are responsible for your assigned target section ONLY. This means: +- Do NOT explain concepts from other sections, even if it would help clarify your own. + If a concept from another section is directly relevant, reference it by section name only + (e.g. "as explored in the Ideal Theory section") β€” never explain it yourself. +- Do NOT generate expounding prompts about topics already covered in other sections. +- Do NOT flag gaps that are addressed in any other section of this document. + +GROUNDING RULES: +- Use only the provided statements, annotations, and source material +- Do not draw on external knowledge +- Where notes are insufficient, flag as a gap β€” only if not addressed in another section + +For your target section, produce: +1. heading β€” a clear improved section heading (2–5 words) +2. intro β€” 1–2 sentences: what will the reader understand from this section? +3. sectionSynthesis β€” 2–4 cohesive paragraphs explaining the section as a whole. + Weave the statements AND their annotations into flowing prose. Do NOT list notes + individually β€” synthesise them into a unified explanation. Use annotations for depth. +4. expandingPrompts β€” 2–3 open questions pushing thinking into territory NOT covered + anywhere in this document. Do not ask about topics other sections address. +5. gaps β€” concepts implied by this section's notes but unexplained here AND not addressed + in any other section. Empty array if none. + +OUTPUT: Valid JSON only, no markdown, no explanation: { - "summary": "...", - "sections": [{ - "heading": "...", - "intro": "...", - "expandingPrompts": ["...", "..."], - "gaps": ["..."] - }] + "heading": "...", + "intro": "...", + "sectionSynthesis": "...", + "expandingPrompts": ["...", "..."], + "gaps": ["..."] }` -export async function callSynthesize( - clusters: ClusterAssignment[], +type RawClusterResult = Omit + +export async function callSynthesizeCluster( + targetCluster: ClusterAssignment, + allClusters: ClusterAssignment[], decontextualized: DecontextualizedNode[], - sourceAnchors: NodeWithContext[], + edgeMap: EdgeMap, config: AIConfig, -): Promise { - const stmtMap = new Map(decontextualized.map(n => [n.id, n.statement])) - const sourceCtx = sourceAnchors.length > 0 - ? `SOURCE MATERIAL:\n${sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` +): Promise { + const stmtMap = new Map(decontextualized.map(n => [n.id, n.statement])) + const annotMap = new Map(edgeMap.nodes.map(n => [n.id, n.annotation ?? ""])) + + const nodeSection = new Map() + for (const cluster of allClusters) { + for (const id of cluster.nodeIds) nodeSection.set(id, cluster.sectionName) + } + + const sourceCtx = edgeMap.sourceAnchors.length > 0 + ? `SOURCE MATERIAL:\n${edgeMap.sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` : "" - const sectionsJson = JSON.stringify( - clusters.map(c => ({ - candidateHeading: c.sectionName, - statements: c.nodeIds - .map(id => stmtMap.get(id)) - .filter(Boolean), + const documentStructure = allClusters + .map((c, i) => `Section ${i + 1}: "${c.sectionName}" (${c.nodeIds.length} notes)`) + .join("\n") + + const allNotesJson = JSON.stringify( + edgeMap.nodes.map(n => ({ + id: n.id, + section: nodeSection.get(n.id) ?? "General", + statement: stmtMap.get(n.id) ?? n.text, + annotation: annotMap.get(n.id) ?? "", })), null, 2, ) + const userMessage = [ + sourceCtx, + `FULL DOCUMENT STRUCTURE:\n${documentStructure}`, + `\nALL NOTES (statements + annotations):\n${allNotesJson}`, + `\n---\nYOUR TARGET SECTION: "${targetCluster.sectionName}"`, + `Node IDs assigned to this section: [${targetCluster.nodeIds.join(", ")}]`, + `\nGenerate the synthesis for this section only.`, + ].join("\n") + const targetUrl = getTargetUrl(config) - const payload = buildPayload(config, [ - { role: "system", content: SYNTHESIZE_SYSTEM }, - { role: "user", content: `${sourceCtx}Sections to synthesise:\n${sectionsJson}` }, + const payload = buildPayload(config, [ + { role: "system", content: CLUSTER_SYNTHESIZE_SYSTEM }, + { role: "user", content: userMessage }, ], 4000) const raw = await callAI(config, targetUrl, payload) - let parsed: { summary?: string; sections?: Omit[] } try { - parsed = JSON.parse(extractJson(raw)) + const parsed = JSON.parse(extractJson(raw)) as RawClusterResult + return { + heading: parsed.heading ?? targetCluster.sectionName, + intro: parsed.intro ?? "", + sectionSynthesis: parsed.sectionSynthesis ?? "", + expandingPrompts: Array.isArray(parsed.expandingPrompts) ? parsed.expandingPrompts : [], + gaps: Array.isArray(parsed.gaps) ? parsed.gaps : [], + } } catch { - dumpRawResponse("synthesis-call-c", raw) - throw new Error("Synthesis: AI returned non-JSON. Raw response saved to synthesis-call-c-raw.txt") + const safeLabel = targetCluster.sectionName.replace(/\s+/g, "-").replace(/[^a-z0-9-]/gi, "") + dumpRawResponse(`synthesis-callC-${safeLabel}`, raw) + throw new Error(`Synthesis (${targetCluster.sectionName}): could not parse AI response. Raw saved to file.`) } +} - // Normalise: handle both array-at-root and object shapes - type RawSection = Omit - type RootObj = Record - - let normalisedSummary: string | null = null - let normalisedSections: RawSection[] | null = null - - if (Array.isArray(parsed)) { - // Model returned the sections array directly β€” no summary - normalisedSections = parsed as RawSection[] - normalisedSummary = "" - } else { - const root = parsed as RootObj - normalisedSections = - Array.isArray(root.sections) ? root.sections as RawSection[] : - Array.isArray(root.data) ? root.data as RawSection[] : - Array.isArray(root.synthesis) ? root.synthesis as RawSection[] : - null - - normalisedSummary = - typeof root.summary === "string" ? root.summary : - typeof root.overview === "string" ? root.overview : - typeof root.description === "string" ? root.description : - null - } +// ── Call D: Final editorial polish ──────────────────────────────────────────── - if (!normalisedSections) { - dumpRawResponse("synthesis-call-c", raw) - throw new Error(`Synthesis: unexpected response shape (keys: ${Object.keys(parsed as object).join(", ")}). Raw response saved to synthesis-call-c-raw.txt`) - } +const POLISH_SYSTEM = `You are performing a final editorial polish on a synthesis document. - // Merge AI-generated prose with node IDs from the clustering step (by index) - const sections: SynthesisSection[] = normalisedSections.map((s, i) => ({ - heading: s.heading ?? clusters[i]?.sectionName ?? `Section ${i + 1}`, - intro: s.intro ?? "", - nodeIds: clusters[i]?.nodeIds ?? [], - expandingPrompts: s.expandingPrompts ?? [], - gaps: s.gaps ?? [], - })) +The document was generated section-by-section by independent parallel processes. Your job +is to refine it as a whole for coherence, flow, and consistency across sections. + +You MAY: +- Refine section headings to form a more coherent document sequence +- Adjust intro sentences to acknowledge adjacent sections where natural +- Add brief cross-references between sections where concepts connect +- Standardise terminology used inconsistently across sections +- Tighten synthesis paragraphs for clarity and cross-section flow + +You MUST NOT: +- Change any line starting with "> *Source: node" β€” these are factual attributions +- Rewrite or remove the [!example], [!question], [!note], or [!info] callout blocks +- Add information not present in the draft +- Alter factual claims in synthesis paragraphs β€” editorial refinement only - return { summary: normalisedSummary ?? "", sections } +Return the complete refined markdown document and nothing else.` + +export async function callPolish( + draftMarkdown: string, + config: AIConfig, +): Promise { + const targetUrl = getTargetUrl(config) + const payload = buildPayload(config, [ + { role: "system", content: POLISH_SYSTEM }, + { role: "user", content: `Polish the following synthesis document:\n\n${draftMarkdown}` }, + ], 8000, false) + + return callAI(config, targetUrl, payload) } -// ── Main orchestration ───────────────────────────────────────────────────────── +// ── Main orchestration ──────────────────────────────────────────────────────── export async function generateSynthesisDocument( blocks: TextBlock[], - onProgress?: (step: string) => void, + onProgress: (event: ProgressEvent) => void, ): Promise { const config = loadAIConfig() if (!config) throw new Error("No AI provider configured. Add an API key in Settings.") @@ -403,21 +448,85 @@ export async function generateSynthesisDocument( const enrichedBlocks = blocks.filter(b => !b.isEnriching && !b.isError) if (enrichedBlocks.length === 0) throw new Error("No notes to synthesise. Add some notes to the canvas first.") - onProgress?.("Building note graph…") - const edgeMap = buildEdgeMap(enrichedBlocks) + const timings: CallTiming[] = [] + + function startCall(id: string, label: string, isParallel = false): number { + const startTime = Date.now() + timings.push({ id, label, status: "running", startTime, isParallel }) + onProgress({ type: "phase_start", id, label }) + return startTime + } + function doneCall(id: string, startTime: number) { + const durationMs = Date.now() - startTime + const t = timings.find(t => t.id === id) + if (t) { t.status = "done"; t.durationMs = durationMs } + onProgress({ type: "phase_done", id, durationMs }) + } + + function errorCall(id: string, message: string) { + const t = timings.find(t => t.id === id) + if (t) t.status = "error" + onProgress({ type: "error", id, message }) + } + + // Phase 1 β€” edge map (instant) + const edgeMap = buildEdgeMap(enrichedBlocks) if (edgeMap.nodes.length === 0) { throw new Error("All notes are reference nodes. Add content notes to the canvas.") } - onProgress?.("Decontextualising and clustering notes…") - const [decontextualized, clusters] = await Promise.all([ - callDecontextualize(edgeMap, config), - callCluster(edgeMap, config), - ]) + // Calls A + B β€” parallel + const startA = startCall("callA", "Decontextualising notes", true) + const startB = startCall("callB", "Clustering sections", true) + + let decontextualized: DecontextualizedNode[] + let clusters: ClusterAssignment[] + + try { + ;[decontextualized, clusters] = await Promise.all([ + callDecontextualize(edgeMap, config) + .then(r => { doneCall("callA", startA); return r }) + .catch(e => { errorCall("callA", String(e)); throw e }), + callCluster(edgeMap, config) + .then(r => { + doneCall("callB", startB) + onProgress({ type: "clusters_known", clusterNames: r.map(c => c.sectionName) }) + return r + }) + .catch(e => { errorCall("callB", String(e)); throw e }), + ]) + } catch (e) { throw e } + + // Calls CΓ—N β€” parallel, one per cluster + const clusterStarts = clusters.map((cluster, i) => + startCall(`callC-${i}`, cluster.sectionName, true) + ) - onProgress?.("Writing synthesis…") - const outline = await callSynthesize(clusters, decontextualized, edgeMap.sourceAnchors, config) + let sectionResults: RawClusterResult[] + try { + sectionResults = await Promise.all( + clusters.map((cluster, i) => + callSynthesizeCluster(cluster, clusters, decontextualized, edgeMap, config) + .then(r => { doneCall(`callC-${i}`, clusterStarts[i]); return r }) + .catch(e => { errorCall(`callC-${i}`, String(e)); throw e }) + ) + ) + } catch (e) { throw e } + + const sections: SynthesisSection[] = clusters.map((cluster, i) => ({ + ...sectionResults[i], + nodeIds: cluster.nodeIds, + })) + + return { + outline: { sections }, + decontextualized, + clusters, + timings, + } +} - return { outline, decontextualized, clusters } +export function getSourceAnchors(blocks: TextBlock[]): TextBlock[] { + return blocks.filter(b => b.contentType === "reference") } From ff490ba580a5294d0ee5fcf90fad084d76d0f8d3 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 20:38:53 +0800 Subject: [PATCH 08/14] fix(synthesis): handle LaTeX backslashes in Call C JSON responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLMs generating LaTeX inside JSON strings sometimes emit single backslashes (\mathbb, \frac, \cong) which are invalid JSON escape sequences. JSON.parse throws on \m, \c etc. β€” only \", \, \/, \n, \r, \t, \b, \f, \uXXXX are legal. Fix: on first parse failure, apply a regex that escapes any backslash not already part of a valid JSON escape sequence, then retry. The outer catch still saves the raw dump if the fixed parse also fails. Regex: /\(?!["\/bfnrtu])/g β€” matches \ not followed by a JSON escape character, replaces with \. Verified against all four raw dump files from the test run. Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/synthesis.ts b/lib/synthesis.ts index ea60eac..83609a8 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -386,13 +386,25 @@ export async function callSynthesizeCluster( const raw = await callAI(config, targetUrl, payload) try { - const parsed = JSON.parse(extractJson(raw)) as RawClusterResult + const jsonStr = extractJson(raw) + + let obj: RawClusterResult + try { + obj = JSON.parse(jsonStr) + } catch { + // LaTeX backslash fix: LLMs often emit \mathbb, \frac etc. as single backslashes + // inside JSON strings, which are invalid escape sequences. Escape any \ not already + // part of a valid JSON escape sequence and retry once. + const fixed = jsonStr.replace(/\\(?!["\\/bfnrtu])/g, "\\\\") + obj = JSON.parse(fixed) + } + return { - heading: parsed.heading ?? targetCluster.sectionName, - intro: parsed.intro ?? "", - sectionSynthesis: parsed.sectionSynthesis ?? "", - expandingPrompts: Array.isArray(parsed.expandingPrompts) ? parsed.expandingPrompts : [], - gaps: Array.isArray(parsed.gaps) ? parsed.gaps : [], + heading: obj.heading ?? targetCluster.sectionName, + intro: obj.intro ?? "", + sectionSynthesis: obj.sectionSynthesis ?? "", + expandingPrompts: Array.isArray(obj.expandingPrompts) ? obj.expandingPrompts : [], + gaps: Array.isArray(obj.gaps) ? obj.gaps : [], } } catch { const safeLabel = targetCluster.sectionName.replace(/\s+/g, "-").replace(/[^a-z0-9-]/gi, "") From 4f6b5f219eb66ca0e019f36f0d95a28f8b46d969 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 20:55:30 +0800 Subject: [PATCH 09/14] fix(synthesis): extractJson must check object before array The previous implementation checked for '[' first. Synthesis responses are JSON objects that contain arrays (expandingPrompts, gaps). When Gemini returned without a fenced code block, extractJson found the '[' inside expandingPrompts and sliced from there to the last ']', producing a broken substring like '["p1","p2"],\n "gaps":["g1"]'. Both JSON.parse attempts then failed. The dump showed the original raw string (which was always valid) because dumpRawResponse is called with raw, not jsonStr. Fix: find which delimiter appears first in the text and use that. - Object responses (synthesis): '{' appears before any '[' in the JSON body - Array responses (decontextualize, cluster): '[' is the first char This correctly handles both without affecting the other callers. Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/synthesis.ts b/lib/synthesis.ts index 83609a8..c8d470c 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -162,14 +162,25 @@ function dumpRawResponse(label: string, raw: string): void { } function extractJson(text: string): string { + // Fenced code block takes priority const fenced = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/) if (fenced) return fenced[1].trim() - const arrStart = text.indexOf("[") - const arrEnd = text.lastIndexOf("]") - if (arrStart !== -1 && arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1) + const objStart = text.indexOf("{") - const objEnd = text.lastIndexOf("}") - if (objStart !== -1 && objEnd > objStart) return text.slice(objStart, objEnd + 1) + const arrStart = text.indexOf("[") + + // Use whichever opening delimiter appears first in the text. + // This correctly handles: + // - Object responses that contain arrays (synthesis): { comes before any [ + // - Array responses (decontextualize, cluster): [ comes before any { + if (objStart !== -1 && (arrStart === -1 || objStart <= arrStart)) { + const objEnd = text.lastIndexOf("}") + if (objEnd > objStart) return text.slice(objStart, objEnd + 1) + } + if (arrStart !== -1) { + const arrEnd = text.lastIndexOf("]") + if (arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1) + } return text.trim() } From b71dc525d683df8ad7947aa1fc3908ba91110d0a Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 21:10:32 +0800 Subject: [PATCH 10/14] fix(synthesis): add dismiss button to progress pill on completion The pill previously stayed visible indefinitely after synthesis completed. Now shows a small X on the right side of the pill when isActive=false, allowing the user to dismiss it. The pill is also reset (undismissed) when a new synthesis starts. Co-Authored-By: Claude Sonnet 4.6 --- components/synthesis-progress-panel.tsx | 55 +++++++++++++++++-------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/components/synthesis-progress-panel.tsx b/components/synthesis-progress-panel.tsx index ffc8b04..1a71c6e 100644 --- a/components/synthesis-progress-panel.tsx +++ b/components/synthesis-progress-panel.tsx @@ -189,6 +189,13 @@ export function SynthesisProgressPanel({ const totalCount = calls.length const hasError = calls.some(c => c.status === "error") + const [dismissed, setDismissed] = React.useState(false) + + // Reset dismissed state when a new synthesis starts + React.useEffect(() => { + if (isActive) setDismissed(false) + }, [isActive]) + const pillLabel = hasError ? "Synthesis error β€” click for details" : isActive @@ -197,35 +204,49 @@ export function SynthesisProgressPanel({ : "Synthesising…" : `Synthesis done β€” ${totalStartMs ? formatDuration(now - totalStartMs) : ""}` - const showPill = calls.length > 0 + const showPill = calls.length > 0 && !dismissed return ( <> {/* Bottom pill */} {showPill && ( - - {isActive && !hasError && ( - + + {/* Dismiss β€” only shown when not active */} + {!isActive && ( + )} - {hasError && } - {!isActive && !hasError && } - - {pillLabel} - - - + )} From 746fc469f91be67d3abddb2384fa8d5181b96702 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 21:31:51 +0800 Subject: [PATCH 11/14] feat(synthesis): add generated_in duration to YAML frontmatter Raw file: total of all Call A/B/C timings. Polished file: total including Call D, injected into the polished markdown's existing frontmatter (replaces if already present from a previous run, inserts before closing --- if absent). The two files now have different generated_in values reflecting when each actually completed. Co-Authored-By: Claude Sonnet 4.6 --- lib/synthesis-export.ts | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/synthesis-export.ts b/lib/synthesis-export.ts index 8fc1171..d2c29c9 100644 --- a/lib/synthesis-export.ts +++ b/lib/synthesis-export.ts @@ -57,11 +57,13 @@ export function renderSynthesisDocument( const lines: string[] = [] // ── YAML front matter ─────────────────────────────────────────────────────── + const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) lines.push(`---`) lines.push(`title: "${isPolished ? "Synthesis (Polished)" : "Synthesis"}: ${canvasName}"`) lines.push(`tags: [synthesis, nodepad${isPolished ? ", polished" : ""}]`) lines.push(`created: ${new Date().toISOString().slice(0, 10)}`) lines.push(`source: nodepad`) + if (totalMs > 0) lines.push(`generated_in: "${formatDuration(totalMs)}"`) lines.push(`---`) lines.push(``) @@ -136,14 +138,31 @@ export function renderPolishedDocument( polishedText: string, timings: CallTiming[], ): string { + const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) const timingBlock = renderTimingTable(timings, true) - const footer = `\n${timingBlock}\n\n*Generated by [nodepad](https://nodepad.space) Β· ${formatDate()}*\n` + const footer = `\n${timingBlock}\n\n*Generated by [nodepad](https://nodepad.space) Β· ${formatDate()}*\n` + + // Inject generated_in into the frontmatter (replace existing value if present, + // or insert before the closing --- of the first frontmatter block) + let text = polishedText + if (totalMs > 0) { + const frontmatterEnd = text.indexOf("\n---", 4) // skip opening --- + if (frontmatterEnd !== -1) { + const hasDuration = /^generated_in:/m.test(text.slice(0, frontmatterEnd)) + if (hasDuration) { + text = text.replace(/^generated_in:.*$/m, `generated_in: "${formatDuration(totalMs)}"`) + } else { + text = text.slice(0, frontmatterEnd) + + `\ngenerated_in: "${formatDuration(totalMs)}"` + + text.slice(frontmatterEnd) + } + } + } - // Append timing + footer before the last line if it already has a footer, - // otherwise just append - const lastFooterMatch = polishedText.match(/\n\*Generated by \[nodepad\][\s\S]*$/) + // Append timing + footer, replacing the raw footer if Call D preserved it + const lastFooterMatch = text.match(/\n\*Generated by \[nodepad\][\s\S]*$/) if (lastFooterMatch) { - return polishedText.slice(0, polishedText.lastIndexOf(lastFooterMatch[0])) + footer + return text.slice(0, text.lastIndexOf(lastFooterMatch[0])) + footer } - return polishedText.trimEnd() + "\n\n" + footer + return text.trimEnd() + "\n\n" + footer } From 13e725d01c87fbe0b0d6f6c2cb827f7ffa5508aa Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 21:46:23 +0800 Subject: [PATCH 12/14] fix(synthesis): use wall-clock time for generated_in, not parallel sum Summing all timing durations gives inflated numbers because parallel calls (A+B together, all Cs together) run simultaneously. The sum counted each worker's time independently, e.g. 5m 6s vs the actual ~2m 29s the user waited. Fix: capture generationStart = Date.now() at the top of startSynthesis and compute wall-clock elapsed at each download point: - raw file: Date.now() - generationStart (after all Cs complete) - polished file: Date.now() - generationStart (after Call D completes) Both renderers now take wallClockMs? as an explicit parameter rather than deriving from the timing array. Co-Authored-By: Claude Sonnet 4.6 --- app/page.tsx | 16 +++++++++------- lib/synthesis-export.ts | 17 ++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/app/page.tsx b/app/page.tsx index 473ac4b..b987939 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -927,9 +927,11 @@ export default function Page() { setSynthConfirmOpen(false) setSynthCalls([]) setSynthActive(true) - setSynthTotalStart(Date.now()) setSynthProgressOpen(false) + const generationStart = Date.now() // local β€” used for accurate wall-clock timing + setSynthTotalStart(generationStart) // state β€” used for the progress pill display + const blocks = proj.blocks const name = proj.name const slug = slugifySynthesis(name) @@ -956,12 +958,12 @@ export default function Page() { generateSynthesisDocument(blocks, onProgress) .then(async ({ outline, decontextualized, clusters, timings }) => { - // Render raw (with timing footer) - const rawMd = renderSynthesisDocument(name, outline, decontextualized, clusters, timings, false) + // Wall-clock time for the raw file = time from start to Call C completion + const rawWallClockMs = Date.now() - generationStart + const rawMd = renderSynthesisDocument(name, outline, decontextualized, clusters, timings, false, rawWallClockMs) downloadMarkdown(`${slug}-synthesis.md`, rawMd) if (enablePolish) { - // Register Call D in progress const dStart = Date.now() setSynthCalls(prev => [...prev, { id: "callD", label: "Final editorial polish", @@ -972,7 +974,6 @@ export default function Page() { const config = loadAIConfig() if (!config) throw new Error("No AI config for polish") - // Draft without timing footer for cleaner polish input const draftForPolish = renderSynthesisDocument(name, outline, decontextualized, clusters, [], false) const polishedText = await callPolish(draftForPolish, config) const polishDuration = Date.now() - dStart @@ -981,12 +982,13 @@ export default function Page() { c.id === "callD" ? { ...c, status: "done", durationMs: polishDuration } : c )) - // Add Call D to timings for footer const fullTimings: CallTiming[] = [ ...timings, { id: "callD", label: "Final editorial polish", status: "done", durationMs: polishDuration }, ] - const polishedMd = renderPolishedDocument(polishedText, fullTimings) + // Wall-clock for polished = full elapsed including Call D + const polishedWallClockMs = Date.now() - generationStart + const polishedMd = renderPolishedDocument(polishedText, fullTimings, polishedWallClockMs) downloadMarkdown(`${slug}-synthesis-polished.md`, polishedMd) } catch (e) { setSynthCalls(prev => prev.map(c => diff --git a/lib/synthesis-export.ts b/lib/synthesis-export.ts index d2c29c9..d15bb0f 100644 --- a/lib/synthesis-export.ts +++ b/lib/synthesis-export.ts @@ -49,6 +49,7 @@ export function renderSynthesisDocument( clusters: ClusterAssignment[], timings: CallTiming[], isPolished = false, + wallClockMs?: number, ): string { const date = formatDate() const totalNodes = clusters.reduce((sum, c) => sum + c.nodeIds.length, 0) @@ -57,13 +58,12 @@ export function renderSynthesisDocument( const lines: string[] = [] // ── YAML front matter ─────────────────────────────────────────────────────── - const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) lines.push(`---`) lines.push(`title: "${isPolished ? "Synthesis (Polished)" : "Synthesis"}: ${canvasName}"`) lines.push(`tags: [synthesis, nodepad${isPolished ? ", polished" : ""}]`) lines.push(`created: ${new Date().toISOString().slice(0, 10)}`) lines.push(`source: nodepad`) - if (totalMs > 0) lines.push(`generated_in: "${formatDuration(totalMs)}"`) + if (wallClockMs != null && wallClockMs > 0) lines.push(`generated_in: "${formatDuration(wallClockMs)}"`) lines.push(`---`) lines.push(``) @@ -137,23 +137,22 @@ export function renderSynthesisDocument( export function renderPolishedDocument( polishedText: string, timings: CallTiming[], + wallClockMs?: number, ): string { - const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) const timingBlock = renderTimingTable(timings, true) const footer = `\n${timingBlock}\n\n*Generated by [nodepad](https://nodepad.space) Β· ${formatDate()}*\n` - // Inject generated_in into the frontmatter (replace existing value if present, - // or insert before the closing --- of the first frontmatter block) + // Inject generated_in into the frontmatter using actual wall-clock time let text = polishedText - if (totalMs > 0) { - const frontmatterEnd = text.indexOf("\n---", 4) // skip opening --- + if (wallClockMs != null && wallClockMs > 0) { + const frontmatterEnd = text.indexOf("\n---", 4) if (frontmatterEnd !== -1) { const hasDuration = /^generated_in:/m.test(text.slice(0, frontmatterEnd)) if (hasDuration) { - text = text.replace(/^generated_in:.*$/m, `generated_in: "${formatDuration(totalMs)}"`) + text = text.replace(/^generated_in:.*$/m, `generated_in: "${formatDuration(wallClockMs)}"`) } else { text = text.slice(0, frontmatterEnd) + - `\ngenerated_in: "${formatDuration(totalMs)}"` + + `\ngenerated_in: "${formatDuration(wallClockMs)}"` + text.slice(frontmatterEnd) } } From afb8cacb7cddaec588f3e73ce628470c3870b679 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 22:04:50 +0800 Subject: [PATCH 13/14] fix(synthesis): timing table Total uses wall-clock; remove dead event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - renderTimingTable now takes wallClockMs? and uses it for the Total row label ("Total (wall-clock)"), consistent with the generated_in frontmatter field. Both renderSynthesisDocument and renderPolishedDocument pass their wallClockMs through. - Remove clusters_known from ProgressEvent type and its emission in callCluster β€” the event was never consumed by the UI handler since Call C phase_start events fire immediately after B completes, making the signal redundant. - Update BACKLOG #027 to reflect web-app implementation is complete; plugin port remains open as future work. Co-Authored-By: Claude Sonnet 4.6 --- BACKLOG.md | 6 +++--- lib/synthesis-export.ts | 13 ++++++++----- lib/synthesis.ts | 13 ++++--------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index d4f516f..e2d98df 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -194,9 +194,9 @@ ## πŸ”­ Upcoming Features -### [#027] Obsidian Plugin β€” Synthesis Document Generation -- **Status**: `Open` | **Priority**: `P2` | **Labels**: `Feature`, `Obsidian`, `AI` -- **Created**: 2026-05-11 +### [#027] Synthesis Document Generation +- **Status**: `Closed (web-app)` Β· `Open (plugin port)` | **Priority**: `P2` | **Labels**: `Feature`, `AI` +- **Created**: 2026-05-11 | **Web-app resolved**: 2026-05-11 - **Design spec**: [`docs/synthesis-document-plan.md`](docs/synthesis-document-plan.md) - **Description**: On-demand command ("Generate Synthesis Document") that consolidates enriched nodes from a `.nodepad` canvas into a structured, contextualized Obsidian markdown document. Unlike the raw markdown export which dumps nodes grouped by type, this pipeline expands sparse notes into self-contained statements, clusters them into coherent thematic sections by meaning, and adds expounding prompts that push thinking into adjacent territory the notes don't cover. Acts as the bridge from nodepad's raw idea staging area into Obsidian's permanent knowledge graph. - **Pipeline**: diff --git a/lib/synthesis-export.ts b/lib/synthesis-export.ts index d15bb0f..30f1eb2 100644 --- a/lib/synthesis-export.ts +++ b/lib/synthesis-export.ts @@ -19,10 +19,9 @@ export function slugifySynthesis(name: string): string { .slice(0, 50) } -function renderTimingTable(timings: CallTiming[], isPolished: boolean): string { +function renderTimingTable(timings: CallTiming[], isPolished: boolean, wallClockMs?: number): string { if (timings.length === 0) return "" - const totalMs = timings.reduce((sum, t) => sum + (t.durationMs ?? 0), 0) const lines: string[] = [] lines.push(`> [!info]- Generation Statistics`) @@ -35,7 +34,11 @@ function renderTimingTable(timings: CallTiming[], isPolished: boolean): string { lines.push(`> | ${prefix}${t.label} | ${dur} |`) } - lines.push(`> | **Total** | **${formatDuration(totalMs)}** |`) + // Use actual wall-clock time for Total β€” parallel calls run simultaneously + // so summing individual durations would give an inflated number + if (wallClockMs != null && wallClockMs > 0) { + lines.push(`> | **Total (wall-clock)** | **${formatDuration(wallClockMs)}** |`) + } if (isPolished) lines.push(`> `) if (isPolished) lines.push(`> *Includes final editorial polish (Call D).*`) @@ -123,7 +126,7 @@ export function renderSynthesisDocument( // ── Timing footer ─────────────────────────────────────────────────────────── if (timings.length > 0) { - lines.push(renderTimingTable(timings, isPolished)) + lines.push(renderTimingTable(timings, isPolished, wallClockMs)) lines.push(``) } @@ -139,7 +142,7 @@ export function renderPolishedDocument( timings: CallTiming[], wallClockMs?: number, ): string { - const timingBlock = renderTimingTable(timings, true) + const timingBlock = renderTimingTable(timings, true, wallClockMs) const footer = `\n${timingBlock}\n\n*Generated by [nodepad](https://nodepad.space) Β· ${formatDate()}*\n` // Inject generated_in into the frontmatter using actual wall-clock time diff --git a/lib/synthesis.ts b/lib/synthesis.ts index c8d470c..c431e86 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -6,10 +6,9 @@ import type { TextBlock } from "@/components/tile-card" // ── Progress events ─────────────────────────────────────────────────────────── export type ProgressEvent = - | { type: "phase_start"; id: string; label: string } - | { type: "phase_done"; id: string; durationMs: number } - | { type: "clusters_known"; clusterNames: string[] } - | { type: "error"; id: string; message: string } + | { type: "phase_start"; id: string; label: string } + | { type: "phase_done"; id: string; durationMs: number } + | { type: "error"; id: string; message: string } export interface CallTiming { id: string @@ -512,11 +511,7 @@ export async function generateSynthesisDocument( .then(r => { doneCall("callA", startA); return r }) .catch(e => { errorCall("callA", String(e)); throw e }), callCluster(edgeMap, config) - .then(r => { - doneCall("callB", startB) - onProgress({ type: "clusters_known", clusterNames: r.map(c => c.sectionName) }) - return r - }) + .then(r => { doneCall("callB", startB); return r }) .catch(e => { errorCall("callB", String(e)); throw e }), ]) } catch (e) { throw e } From 93a29819ade865a0db7aab1c446efb7e67fae4a8 Mon Sep 17 00:00:00 2001 From: Dev-020 Date: Mon, 11 May 2026 22:23:24 +0800 Subject: [PATCH 14/14] feat(plugin): port Synthesis Document generation to Obsidian plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transport layer: - lib/synthesis.ts: export SynthesisCallFn type; rename internal callAI to defaultCallAI; all call functions (callDecontextualize, callCluster, callSynthesizeCluster, callPolish, generateSynthesisDocument) accept optional callFn/injectedConfig so the plugin can swap fetch/api/ai for requestUrl/spawnCLI without duplicating any prompt or parsing logic - plugin/src/ai-adapter.ts: add makeSynthesisCallFn(plugin) which returns a SynthesisCallFn β€” routes Gemini CLI through spawnCLI("gemini", ["--policy","simple"], combinedPrompt), all other providers via requestUrl() directly to the provider (no /api/ai proxy needed) UI components: - SynthesisConfirmDialog: add container? prop; render via createPortal when provided so it scopes to the Obsidian leaf instead of document.body - SynthesisProgressPanel: same container?/portal treatment Plugin wiring (plugin/src/view.tsx): - NodepadView gains synthesisTriggerRef and triggerSynthesis() method (called from the command, shows Notice if no canvas is open) - NodepadApp gains full synthesis state (same as page.tsx) wired to synthesisTriggerRef so the command opens the confirm dialog - startSynthesis() runs the pipeline with makeSynthesisCallFn and writes output via app.vault.create() + workspace.getLeaf().openFile() β€” files appear in the same folder as the canvas, opened immediately in a new tab; no browser download - Error surfaced via Obsidian Notice instead of console-only Command (plugin/src/main.ts): - Register "Generate Synthesis Document" with checkCallback that requires an active NodepadView Co-Authored-By: Claude Sonnet 4.6 --- components/synthesis-confirm-dialog.tsx | 7 +- components/synthesis-progress-panel.tsx | 7 +- lib/synthesis.ts | 35 ++++-- plugin/src/ai-adapter.ts | 46 +++++++ plugin/src/main.ts | 11 ++ plugin/src/view.tsx | 161 +++++++++++++++++++++++- 6 files changed, 254 insertions(+), 13 deletions(-) diff --git a/components/synthesis-confirm-dialog.tsx b/components/synthesis-confirm-dialog.tsx index 5229911..86bedef 100644 --- a/components/synthesis-confirm-dialog.tsx +++ b/components/synthesis-confirm-dialog.tsx @@ -1,6 +1,7 @@ "use client" import * as React from "react" +import { createPortal } from "react-dom" import { motion, AnimatePresence } from "framer-motion" import { AlertTriangle, BookOpen, CheckSquare, Square, X } from "lucide-react" import type { TextBlock } from "@/components/tile-card" @@ -11,6 +12,7 @@ interface SynthesisConfirmDialogProps { blockCount: number onConfirm: (enablePolish: boolean) => void onCancel: () => void + container?: HTMLElement } export function SynthesisConfirmDialog({ @@ -19,6 +21,7 @@ export function SynthesisConfirmDialog({ blockCount, onConfirm, onCancel, + container, }: SynthesisConfirmDialogProps) { const [enablePolish, setEnablePolish] = React.useState(false) @@ -31,7 +34,7 @@ export function SynthesisConfirmDialog({ const hasAnchors = sourceAnchors.length > 0 - return ( + const content = ( {isOpen && ( <> @@ -168,4 +171,6 @@ export function SynthesisConfirmDialog({ )} ) + + return container ? createPortal(content, container) : content } diff --git a/components/synthesis-progress-panel.tsx b/components/synthesis-progress-panel.tsx index 1a71c6e..fe112f9 100644 --- a/components/synthesis-progress-panel.tsx +++ b/components/synthesis-progress-panel.tsx @@ -1,6 +1,7 @@ "use client" import * as React from "react" +import { createPortal } from "react-dom" import { motion, AnimatePresence } from "framer-motion" import { CheckCircle, AlertCircle, X, ChevronUp } from "lucide-react" import type { CallTiming } from "@/lib/synthesis" @@ -13,6 +14,7 @@ interface SynthesisProgressPanelProps { isDialogOpen: boolean onPillClick: () => void onDialogClose: () => void + container?: HTMLElement } // ── Elapsed timer for running calls ────────────────────────────────────────── @@ -183,6 +185,7 @@ export function SynthesisProgressPanel({ isDialogOpen, onPillClick, onDialogClose, + container, }: SynthesisProgressPanelProps) { const now = useElapsed(isActive) const doneCount = calls.filter(c => c.status === "done").length @@ -206,7 +209,7 @@ export function SynthesisProgressPanel({ const showPill = calls.length > 0 && !dismissed - return ( + const panel = ( <> {/* Bottom pill */} @@ -263,4 +266,6 @@ export function SynthesisProgressPanel({ ) + + return container ? createPortal(panel, container) : panel } diff --git a/lib/synthesis.ts b/lib/synthesis.ts index c431e86..c3184a7 100644 --- a/lib/synthesis.ts +++ b/lib/synthesis.ts @@ -124,7 +124,17 @@ function buildPayload( } } -async function callAI(config: AIConfig, targetUrl: string, payload: object): Promise { +// ── Injectable transport ────────────────────────────────────────────────────── +// The web-app routes through /api/ai; the plugin uses requestUrl/spawnCLI +// directly. Pass a SynthesisCallFn to swap the transport layer. + +export type SynthesisCallFn = ( + config: AIConfig, + targetUrl: string, + payload: object, +) => Promise + +async function defaultCallAI(config: AIConfig, targetUrl: string, payload: object): Promise { const response = await fetch("/api/ai", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -209,6 +219,7 @@ OUTPUT: You MUST return a JSON array with one entry per input note, in the same export async function callDecontextualize( edgeMap: EdgeMap, config: AIConfig, + callFn: SynthesisCallFn = defaultCallAI, ): Promise { const sourceCtx = edgeMap.sourceAnchors.length > 0 ? `SOURCE MATERIAL:\n${edgeMap.sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` @@ -230,7 +241,7 @@ export async function callDecontextualize( { role: "user", content: `${sourceCtx}Notes to expand:\n${notesJson}` }, ], 6000) - const raw = await callAI(config, targetUrl, payload) + const raw = await callFn(config, targetUrl, payload) let parsed: DecontextualizedNode[] = [] try { parsed = JSON.parse(extractJson(raw)) } catch { /* fall through */ } @@ -256,6 +267,7 @@ OUTPUT: You MUST return a JSON array: export async function callCluster( edgeMap: EdgeMap, config: AIConfig, + callFn: SynthesisCallFn = defaultCallAI, ): Promise { const sourceCtx = edgeMap.sourceAnchors.length > 0 ? `SOURCE MATERIAL:\n${edgeMap.sourceAnchors.map(a => `- ${a.text}`).join("\n")}\n\n` @@ -278,7 +290,7 @@ export async function callCluster( { role: "user", content: `${sourceCtx}Notes to cluster:\n${notesJson}` }, ], 2000) - const raw = await callAI(config, targetUrl, payload) + const raw = await callFn(config, targetUrl, payload) let parsed: ClusterAssignment[] = [] try { parsed = JSON.parse(extractJson(raw)) } catch { @@ -351,6 +363,7 @@ export async function callSynthesizeCluster( decontextualized: DecontextualizedNode[], edgeMap: EdgeMap, config: AIConfig, + callFn: SynthesisCallFn = defaultCallAI, ): Promise { const stmtMap = new Map(decontextualized.map(n => [n.id, n.statement])) const annotMap = new Map(edgeMap.nodes.map(n => [n.id, n.annotation ?? ""])) @@ -393,7 +406,7 @@ export async function callSynthesizeCluster( { role: "user", content: userMessage }, ], 4000) - const raw = await callAI(config, targetUrl, payload) + const raw = await callFn(config, targetUrl, payload) try { const jsonStr = extractJson(raw) @@ -448,6 +461,7 @@ Return the complete refined markdown document and nothing else.` export async function callPolish( draftMarkdown: string, config: AIConfig, + callFn: SynthesisCallFn = defaultCallAI, ): Promise { const targetUrl = getTargetUrl(config) const payload = buildPayload(config, [ @@ -455,7 +469,7 @@ export async function callPolish( { role: "user", content: `Polish the following synthesis document:\n\n${draftMarkdown}` }, ], 8000, false) - return callAI(config, targetUrl, payload) + return callFn(config, targetUrl, payload) } // ── Main orchestration ──────────────────────────────────────────────────────── @@ -463,9 +477,12 @@ export async function callPolish( export async function generateSynthesisDocument( blocks: TextBlock[], onProgress: (event: ProgressEvent) => void, + callFn?: SynthesisCallFn, + injectedConfig?: AIConfig, ): Promise { - const config = loadAIConfig() + const config = injectedConfig ?? loadAIConfig() if (!config) throw new Error("No AI provider configured. Add an API key in Settings.") + const fn = callFn ?? defaultCallAI const enrichedBlocks = blocks.filter(b => !b.isEnriching && !b.isError) if (enrichedBlocks.length === 0) throw new Error("No notes to synthesise. Add some notes to the canvas first.") @@ -507,10 +524,10 @@ export async function generateSynthesisDocument( try { ;[decontextualized, clusters] = await Promise.all([ - callDecontextualize(edgeMap, config) + callDecontextualize(edgeMap, config, fn) .then(r => { doneCall("callA", startA); return r }) .catch(e => { errorCall("callA", String(e)); throw e }), - callCluster(edgeMap, config) + callCluster(edgeMap, config, fn) .then(r => { doneCall("callB", startB); return r }) .catch(e => { errorCall("callB", String(e)); throw e }), ]) @@ -525,7 +542,7 @@ export async function generateSynthesisDocument( try { sectionResults = await Promise.all( clusters.map((cluster, i) => - callSynthesizeCluster(cluster, clusters, decontextualized, edgeMap, config) + callSynthesizeCluster(cluster, clusters, decontextualized, edgeMap, config, fn) .then(r => { doneCall(`callC-${i}`, clusterStarts[i]); return r }) .catch(e => { errorCall(`callC-${i}`, String(e)); throw e }) ) diff --git a/plugin/src/ai-adapter.ts b/plugin/src/ai-adapter.ts index fe125cc..67a24e4 100644 --- a/plugin/src/ai-adapter.ts +++ b/plugin/src/ai-adapter.ts @@ -9,6 +9,7 @@ import { } from "@/lib/ai-settings" import { CONTENT_TYPE_CONFIG, type ContentType } from "@/lib/content-types" import { detectContentType } from "@/lib/detect-content-type" +import { type SynthesisCallFn } from "@/lib/synthesis" // ── Types re-exported from shared lib (avoiding "use client" import issues) ─── @@ -715,3 +716,48 @@ Return ONLY valid JSON: throw new Error("Could not parse ghost response") } } + +// ── Synthesis transport (for plugin port of Synthesis Document generation) ──── + +export function makeSynthesisCallFn(plugin: NodepadPlugin): SynthesisCallFn { + return async (config: AIConfig, targetUrl: string, payload: object) => { + // Gemini CLI: combine messages into a single prompt and spawn CLI + if (config.provider === "geminicli") { + const messages = ((payload as Record).messages ?? []) as Array<{ role: string; content: string }> + const system = messages.find(m => m.role === "system")?.content ?? "" + const user = messages.find(m => m.role === "user")?.content ?? "" + const combined = [system, user].filter(Boolean).join("\n\n---\n\n") + + console.log("[Nodepad/Gemini CLI] Stage 2: Generating synthesis...") + const result = await spawnCLI("gemini", ["--policy", "simple"], combined, 480000) + if (!result.success) throw new Error(`Gemini CLI synthesis failed: ${result.error}`) + console.log(`[Nodepad/Gemini CLI] Synthesis done (${result.content.length} chars)`) + return result.content + } + + // All other providers: requestUrl directly to the provider + const isOllama = config.provider === "ollama" + const url = isOllama ? `${getBaseUrl(config)}/api/chat` : targetUrl + + const response = await requestUrl({ + url, + method: "POST", + headers: getProviderHeaders(config), + body: JSON.stringify(payload), + contentType: "application/json", + throw: false, + }) + + if (response.status >= 400) { + throw new Error(`Synthesis AI request failed (${response.status})`) + } + + const data = response.json as Record + const content = isOllama + ? (data.message as { content?: string } | undefined)?.content + : ((data.choices as Array<{ message?: { content?: string } }> | undefined)?.[0]?.message?.content) + + if (!content) throw new Error("Empty response from AI provider") + return content + } +} diff --git a/plugin/src/main.ts b/plugin/src/main.ts index 0b86700..4e64c4a 100644 --- a/plugin/src/main.ts +++ b/plugin/src/main.ts @@ -19,6 +19,17 @@ export default class NodepadPlugin extends Plugin { callback: () => this.createNewSpace(), }) + this.addCommand({ + id: "generate-synthesis-document", + name: "Generate Synthesis Document", + checkCallback: (checking) => { + const view = this.app.workspace.getActiveViewOfType(NodepadView) + if (!view) return false + if (!checking) view.triggerSynthesis() + return true + }, + }) + this.registerEvent( this.app.workspace.on("file-menu", (menu, item) => { if (!(item instanceof TFolder)) return diff --git a/plugin/src/view.tsx b/plugin/src/view.tsx index 251eae6..6d49790 100644 --- a/plugin/src/view.tsx +++ b/plugin/src/view.tsx @@ -3,7 +3,7 @@ import { createRoot, type Root } from "react-dom/client" import React, { useState, useCallback, useEffect, useRef, useMemo } from "react" import { motion, AnimatePresence } from "framer-motion" import type NodepadPlugin from "./main" -import { enrichBlock, generateGhost } from "./ai-adapter" +import { enrichBlock, generateGhost, getPluginAIConfig, makeSynthesisCallFn } from "./ai-adapter" import { TilingArea } from "@/components/tiling-area" import { KanbanArea } from "@/components/kanban-area" @@ -16,6 +16,20 @@ import type { TextBlock } from "@/components/tile-card" import type { ContentType } from "@/lib/content-types" import { detectContentType } from "@/lib/detect-content-type" import { exportToMarkdown, copyToClipboard } from "@/lib/export" +import { + generateSynthesisDocument, + callPolish, + getSourceAnchors, + type CallTiming, + type ProgressEvent, +} from "@/lib/synthesis" +import { + renderSynthesisDocument, + renderPolishedDocument, + slugifySynthesis, +} from "@/lib/synthesis-export" +import { SynthesisConfirmDialog } from "@/components/synthesis-confirm-dialog" +import { SynthesisProgressPanel } from "@/components/synthesis-progress-panel" export const VIEW_TYPE = "nodepad-view" @@ -105,12 +119,21 @@ export class NodepadView extends TextFileView { private root: Root | null = null readonly plugin: NodepadPlugin private fileData = "" + readonly synthesisTriggerRef: { current: (() => void) | null } = { current: null } constructor(leaf: WorkspaceLeaf, plugin: NodepadPlugin) { super(leaf) this.plugin = plugin } + triggerSynthesis() { + if (this.synthesisTriggerRef.current) { + this.synthesisTriggerRef.current() + } else { + new Notice("Nodepad: open a canvas first to generate a synthesis document.") + } + } + getViewType() { return VIEW_TYPE } getDisplayText() { return this.file?.basename ?? "Nodepad" } getIcon() { return "layout-dashboard" } @@ -157,6 +180,7 @@ export class NodepadView extends TextFileView { initialData={this.fileData} fileName={this.file?.basename} folderPath={this.file?.parent?.path} + synthesisTriggerRef={this.synthesisTriggerRef} onSave={(data) => { this.fileData = data this.requestSave() @@ -183,9 +207,10 @@ interface NodepadAppProps { onSave: (data: string) => void onMenuClick: () => void portalContainer?: HTMLElement + synthesisTriggerRef?: { current: (() => void) | null } } -function NodepadApp({ plugin, initialData, fileName, folderPath, onSave, onMenuClick, portalContainer }: NodepadAppProps) { +function NodepadApp({ plugin, initialData, fileName, folderPath, onSave, onMenuClick, portalContainer, synthesisTriggerRef }: NodepadAppProps) { const parsed = useMemo(() => parseFileData(initialData), [initialData]) const [blocks, setBlocks] = useState(parsed.blocks) @@ -197,6 +222,119 @@ function NodepadApp({ plugin, initialData, fileName, folderPath, onSave, onMenuC const [viewMode, setViewMode] = useState<"tiling" | "kanban" | "graph">(parsed.viewMode ?? "tiling") const [isGhostPanelOpen, setIsGhostPanelOpen] = useState(false) const [isIndexOpen, setIsIndexOpen] = useState(false) + + // ── Synthesis state ─────────────────────────────────────────────────────── + const [synthConfirmOpen, setSynthConfirmOpen] = useState(false) + const [synthProgressOpen, setSynthProgressOpen] = useState(false) + const [synthCalls, setSynthCalls] = useState([]) + const [synthActive, setSynthActive] = useState(false) + const [synthTotalStart, setSynthTotalStart] = useState(undefined) + const [synthSourceAnchors, setSynthSourceAnchors] = useState([]) + const synthEnablePolish = useRef(false) + + // Expose trigger to NodepadView so the command palette can open the dialog + useEffect(() => { + if (synthesisTriggerRef) { + synthesisTriggerRef.current = () => { + setSynthSourceAnchors(getSourceAnchors(blocks)) + setSynthConfirmOpen(true) + } + return () => { synthesisTriggerRef.current = null } + } + }, [synthesisTriggerRef, blocks]) + + const startSynthesis = useCallback((enablePolish: boolean) => { + synthEnablePolish.current = enablePolish + setSynthConfirmOpen(false) + setSynthCalls([]) + setSynthActive(true) + setSynthProgressOpen(false) + + const generationStart = Date.now() + setSynthTotalStart(generationStart) + + const canvasName = fileName ?? "Synthesis" + const slug = slugifySynthesis(canvasName) + const dir = folderPath ?? "" + + const config = getPluginAIConfig(plugin) + const callFn = makeSynthesisCallFn(plugin) + + const onProgress = (event: ProgressEvent) => { + setSynthCalls(prev => { + const next = [...prev] + if (event.type === "phase_start") { + next.push({ + id: event.id, label: event.label, status: "running", + startTime: Date.now(), + isParallel: event.id.startsWith("callC-") || event.id === "callA" || event.id === "callB", + }) + } else if (event.type === "phase_done") { + const t = next.find(t => t.id === event.id) + if (t) { t.status = "done"; t.durationMs = event.durationMs } + } else if (event.type === "error") { + const t = next.find(t => t.id === event.id) + if (t) t.status = "error" + } + return next + }) + } + + const writeToVault = async (filename: string, content: string) => { + let path = dir ? `${dir}/${filename}` : filename + let idx = 2 + while (plugin.app.vault.getAbstractFileByPath(path)) { + const dot = filename.lastIndexOf(".") + const base = dot >= 0 ? filename.slice(0, dot) : filename + const ext = dot >= 0 ? filename.slice(dot) : "" + path = dir ? `${dir}/${base}-${idx}${ext}` : `${base}-${idx}${ext}` + idx++ + } + const file = await plugin.app.vault.create(path, content) + await plugin.app.workspace.getLeaf("tab").openFile(file) + } + + generateSynthesisDocument(blocks, onProgress, callFn, config ?? undefined) + .then(async ({ outline, decontextualized, clusters, timings }) => { + const rawMs = Date.now() - generationStart + const rawMd = renderSynthesisDocument(canvasName, outline, decontextualized, clusters, timings, false, rawMs) + await writeToVault(`${slug}-synthesis.md`, rawMd) + + if (enablePolish && config) { + const dStart = Date.now() + setSynthCalls(prev => [...prev, { + id: "callD", label: "Final editorial polish", + status: "running", startTime: dStart, isParallel: false, + }]) + try { + const draftForPolish = renderSynthesisDocument(canvasName, outline, decontextualized, clusters, [], false) + const polishedText = await callPolish(draftForPolish, config, callFn) + const polishDuration = Date.now() - dStart + setSynthCalls(prev => prev.map(c => + c.id === "callD" ? { ...c, status: "done", durationMs: polishDuration } : c + )) + const fullTimings: CallTiming[] = [ + ...timings, + { id: "callD", label: "Final editorial polish", status: "done", durationMs: polishDuration }, + ] + const polishedMs = Date.now() - generationStart + const polishedMd = renderPolishedDocument(polishedText, fullTimings, polishedMs) + await writeToVault(`${slug}-synthesis-polished.md`, polishedMd) + } catch (e) { + setSynthCalls(prev => prev.map(c => + c.id === "callD" ? { ...c, status: "error" } : c + )) + console.error("[synthesis polish]", e) + } + } + setSynthActive(false) + }) + .catch((err: Error) => { + console.error("[synthesis]", err) + new Notice(`Synthesis failed: ${err.message}`) + setSynthActive(false) + }) + }, [blocks, fileName, folderPath, plugin]) const [isCommandKOpen, setIsCommandKOpen] = useState(false) const [highlightedBlockId, setHighlightedBlockId] = useState(null) const [undoToast, setUndoToast] = useState(null) @@ -688,6 +826,25 @@ function NodepadApp({ plugin, initialData, fileName, folderPath, onSave, onMenuC isOpen={isIndexOpen} viewMode={viewMode} /> + + {/* Synthesis dialogs β€” portalled into the plugin container */} + setSynthConfirmOpen(false)} + container={portalContainer} + /> + setSynthProgressOpen(true)} + onDialogClose={() => setSynthProgressOpen(false)} + container={portalContainer} + />
) }