From 29c8ee7aaa0ee19e05b1c1ebc40c45e7c2aafe16 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:33:39 -0500 Subject: [PATCH 01/38] feat(read): read any UTF-8 file outside the dedicated types as text vault_read_file now returns HTML pages, scripts and files with no extension as text. A file outside the dedicated types is refused when a NUL byte sits in its first 8,000 bytes (binary data or UTF-16 text) or when its bytes are not UTF-8. The dedicated text formats keep their path with no NUL check. Co-Authored-By: Claude Opus 5.5 --- .../server-error-contracts.test.ts | 23 +++ .../__tests__/decode-text-buffer.test.ts | 64 ++++++++ src/utils/decode-text-buffer.ts | 26 ++++ .../__tests__/tool-definitions.test.ts | 22 ++- .../__tests__/asset-operations.test.ts | 137 +++++++++++++++--- .../vault-operations/asset-operations.ts | 62 ++++++-- 6 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 src/utils/__tests__/decode-text-buffer.test.ts create mode 100644 src/utils/decode-text-buffer.ts diff --git a/src/__tests__/integration/server-error-contracts.test.ts b/src/__tests__/integration/server-error-contracts.test.ts index cbbb78bf5..59d63e498 100644 --- a/src/__tests__/integration/server-error-contracts.test.ts +++ b/src/__tests__/integration/server-error-contracts.test.ts @@ -2229,6 +2229,29 @@ describe("malformed canvas", () => { }) }) +// ── Binary file ────────────────────────────────────────────── + +describe("binary file", () => { + it("vault_read_file on a file holding a NUL byte", async () => { + const filePath = "Archive.zip" + const fileFullPath = join(serverVaultPath, filePath) + // A ZIP local file header, whose version and flag fields hold NUL bytes + await writeFile(fileFullPath, Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00])) + onTestFinished(() => rm(fileFullPath)) + + const result = await callTool({ client, name: "vault_read_file", args: { path: filePath } }) + expect(result).toEqual({ + content: [ + { + type: "text", + text: '[Error]: unsupported file type ".zip": "Archive.zip" exists (8 bytes) and holds a NUL byte in its first 8,000 bytes (binary data or UTF-16 text). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and UTF-8 text files', + }, + ], + isError: true, + }) + }) +}) + // ── Whitespace-only task text ──────────────────────────────── describe("whitespace-only task text", () => { diff --git a/src/utils/__tests__/decode-text-buffer.test.ts b/src/utils/__tests__/decode-text-buffer.test.ts new file mode 100644 index 000000000..25291e8c9 --- /dev/null +++ b/src/utils/__tests__/decode-text-buffer.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from "vitest" +import { decodeTextBuffer } from "../decode-text-buffer.js" + +/** `length` bytes of ASCII "a" with a NUL byte at `nulIndex`. */ +const buildBufferWithNulAt = (params: { length: number; nulIndex: number }): Buffer => { + const buffer = Buffer.alloc(params.length, "a") + buffer[params.nulIndex] = 0 + return buffer +} + +describe("decodeTextBuffer", () => { + it("decodes UTF-8 text, multibyte characters included", () => { + expect(decodeTextBuffer(Buffer.from("café — ✓\n", "utf8"))).toEqual({ + status: "text", + text: "café — ✓\n", + }) + }) + + it("decodes an empty buffer as empty text", () => { + expect(decodeTextBuffer(Buffer.alloc(0))).toEqual({ status: "text", text: "" }) + }) + + it("calls bytes binary when a NUL byte is the first byte", () => { + expect(decodeTextBuffer(buildBufferWithNulAt({ length: 10, nulIndex: 0 }))).toEqual({ + status: "binary", + }) + }) + + it("calls bytes binary when a NUL byte is the last byte of the first 8,000", () => { + expect(decodeTextBuffer(buildBufferWithNulAt({ length: 9_000, nulIndex: 7_999 }))).toEqual({ + status: "binary", + }) + }) + + it("decodes bytes whose first NUL byte sits just past the first 8,000", () => { + const buffer = buildBufferWithNulAt({ length: 8_001, nulIndex: 8_000 }) + + expect(decodeTextBuffer(buffer)).toEqual({ + status: "text", + text: `${"a".repeat(8_000)}\u0000`, + }) + }) + + it("calls BOM-less UTF-16 text binary, since its ASCII characters carry NUL bytes", () => { + expect(decodeTextBuffer(Buffer.from("hi", "utf16le"))).toEqual({ status: "binary" }) + }) + + it("calls bytes that are not UTF-8 invalid", () => { + // 0xE9 is "é" in Latin-1 and never a complete UTF-8 sequence + expect(decodeTextBuffer(Buffer.from([0x63, 0x61, 0x66, 0xe9]))).toEqual({ + status: "invalid-utf8", + }) + }) + + it("calls an encoded UTF-16 surrogate invalid, as a strict UTF-8 decoder does", () => { + expect(decodeTextBuffer(Buffer.from([0xed, 0xa0, 0x80]))).toEqual({ status: "invalid-utf8" }) + }) + + it("drops a leading byte-order mark", () => { + const withByteOrderMark = Buffer.from([0xef, 0xbb, 0xbf, 0x68, 0x69]) + + expect(decodeTextBuffer(withByteOrderMark)).toEqual({ status: "text", text: "hi" }) + }) +}) diff --git a/src/utils/decode-text-buffer.ts b/src/utils/decode-text-buffer.ts new file mode 100644 index 000000000..bf20b12c4 --- /dev/null +++ b/src/utils/decode-text-buffer.ts @@ -0,0 +1,26 @@ +import { isUtf8 } from "node:buffer" + +/** How many leading bytes are searched for a NUL byte. Git calls a file binary + * when its first 8,000 bytes hold one (FIRST_FEW_BYTES in xdiff-interface.c): + * https://github.com/git/git/blob/master/xdiff-interface.c */ +export const NUL_SEARCH_WINDOW_BYTES = 8_000 + +type DecodedTextBuffer = + | Readonly<{ status: "text"; text: string }> + | Readonly<{ status: "binary" }> + | Readonly<{ status: "invalid-utf8" }> + +// Decoding is stateless outside streaming mode, so one decoder serves every call. +const utf8Decoder = new TextDecoder("utf-8") + +/** Decodes bytes as UTF-8 text. + * - A NUL byte within the first NUL_SEARCH_WINDOW_BYTES marks them binary. + * - Any byte sequence that is not UTF-8 marks them invalid. + * - A leading byte-order mark is dropped, as TextDecoder does by default. */ +export const decodeTextBuffer = (buffer: Uint8Array): DecodedTextBuffer => { + const holdsEarlyNulByte = buffer.subarray(0, NUL_SEARCH_WINDOW_BYTES).includes(0) + + if (holdsEarlyNulByte) return { status: "binary" } + if (!isUtf8(buffer)) return { status: "invalid-utf8" } + return { status: "text", text: utf8Decoder.decode(buffer) } +} diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index e99a747d4..c09bf4c4d 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -2848,21 +2848,37 @@ describe("file tool handlers", () => { }) }) - it("rejects an unsupported extension naming the readable types", async () => { + it("rejects a file holding NUL bytes, naming the readable types", async () => { const { vault, readAsset } = await setupAssetHarness() - await writeFile(join(vault, "song.mp3"), "xxxx", "utf8") + // An ID3v2.4 tag header, which opens most MP3 files and holds NUL bytes + await writeFile(join(vault, "song.mp3"), Buffer.from([0x49, 0x44, 0x33, 0x04, 0x00, 0x00])) const result = await readAsset({ path: "song.mp3" }) expect(result).toEqual({ isError: true, content: [ { type: "text", - text: '[Error]: unsupported file type ".mp3": "song.mp3" exists (4 bytes). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats (.svg/.json/.txt/.csv/.xml/.log/.yaml/.yml/.base)', + text: '[Error]: unsupported file type ".mp3": "song.mp3" exists (6 bytes) and holds a NUL byte in its first 8,000 bytes (binary data or UTF-16 text). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and UTF-8 text files', }, ], }) }) + it("returns an HTML page's source as text", async () => { + const { vault, readAsset } = await setupAssetHarness() + const pageSource = "\n

Runbook

\n\n" + await writeFile(join(vault, "runbook.html"), pageSource, "utf8") + const result = await readAsset({ path: "runbook.html" }) + expect(result).toEqual({ content: [{ type: "text", text: pageSource }] }) + }) + + it("reads a note whose .MD extension is uppercase as text, until notes with that extension are supported", async () => { + const { vault, readAsset } = await setupAssetHarness() + await writeFile(join(vault, "Note.MD"), "# Upper\n", "utf8") + const result = await readAsset({ path: "Note.MD" }) + expect(result).toEqual({ content: [{ type: "text", text: "# Upper\n" }] }) + }) + it("rejects a .md path without touching the note", async () => { const { vault, readAsset } = await setupAssetHarness() await writeFile(join(vault, "note.md"), "# A note\n", "utf8") diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts index d9697e90f..d476c3f38 100644 --- a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -440,25 +440,6 @@ describe("readAssetContent — PDF extraction", () => { ) }) - it("includes .pdf in the unsupported-type error's readable types list", async () => { - mockedReadAsset.mockResolvedValue({ - buffer: Buffer.from("fake-audio"), - bytes: 10_000, - extension: ".mp3", - }) - - await expect( - assetOperations.readAssetContent({ ...defaultParams, path: "audio/song.mp3" }, logger), - ).rejects.toThrow( - new Error( - 'unsupported file type ".mp3": "audio/song.mp3" exists ' + - "(10000 bytes). Readable types: images " + - "(.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats " + - "(.svg/.json/.txt/.csv/.xml/.log/.yaml/.yml/.base)", - ), - ) - }) - it("passes a PDF load failure other than pdf.js refusing the file through unchanged", async () => { mockedReadAsset.mockResolvedValue({ buffer: Buffer.from("not-a-real-pdf"), @@ -1017,6 +998,124 @@ describe("readAssetContent — PDF page rendering (raw: true)", () => { }) }) +describe("readAssetContent — UTF-8 files outside the dedicated types", () => { + /** Stubs readAsset with the given bytes, reporting the extension lowercased + * as readAsset does. */ + const stubReadAsset = (params: { buffer: Buffer; extension: string }): void => { + mockedReadAsset.mockResolvedValue({ + buffer: params.buffer, + bytes: params.buffer.length, + extension: params.extension, + }) + } + + /** `length` bytes of ASCII "a" with a NUL byte at `nulIndex`. */ + const buildBufferWithNulAt = (params: { length: number; nulIndex: number }): Buffer => { + const buffer = Buffer.alloc(params.length, "a") + buffer[params.nulIndex] = 0 + return buffer + } + + it.each([ + { + label: "an HTML page's source", + path: "runbooks/restart.html", + extension: ".html", + source: '\n

Restart — then wait

\n', + }, + { + label: "a Python script", + path: "scripts/report.py", + extension: ".py", + source: "def get_user_name():\n return 'café'\n", + }, + { + label: "a PowerShell script", + path: "scripts/Backup.ps1", + extension: ".ps1", + source: "Get-ChildItem -Path $Home | Out-Null\r\n", + }, + { + label: "a file with no extension", + path: "scripts/Makefile", + extension: "", + source: "build:\n\tnpm run build\n", + }, + ])("returns $label byte-exact as text", async ({ path, extension, source }) => { + stubReadAsset({ buffer: Buffer.from(source, "utf8"), extension }) + + const result = await assetOperations.readAssetContent({ ...defaultParams, path }, logger) + + expect(result).toEqual({ kind: "text", text: source, path }) + }) + + it("pages a script by line like the dedicated text formats", async () => { + stubReadAsset({ buffer: Buffer.from("one\ntwo\nthree\nfour\n", "utf8"), extension: ".sh" }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "deploy.sh", startLine: 2, limit: 2 }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: "two\nthree", + path: "deploy.sh", + lineWindow: { startLine: 2, endLine: 3, totalLines: 4 }, + }) + }) + + it("refuses a file with a NUL byte in its first 8,000 bytes, naming the readable types", async () => { + stubReadAsset({ + buffer: buildBufferWithNulAt({ length: 9_000, nulIndex: 7_999 }), + extension: ".mp3", + }) + + await expect( + assetOperations.readAssetContent({ ...defaultParams, path: "audio/song.mp3" }, logger), + ).rejects.toThrow( + new Error( + 'unsupported file type ".mp3": "audio/song.mp3" exists (9000 bytes) and holds a NUL ' + + "byte in its first 8,000 bytes (binary data or UTF-16 text). Readable types: images " + + "(.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and UTF-8 text files", + ), + ) + }) + + it("reads an HTML page whose first NUL byte sits just past the first 8,000 bytes", async () => { + const buffer = buildBufferWithNulAt({ length: 8_001, nulIndex: 8_000 }) + stubReadAsset({ buffer, extension: ".html" }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "padded.html" }, + logger, + ) + + expect(result).toEqual({ kind: "text", text: buffer.toString("utf8"), path: "padded.html" }) + }) + + it("refuses an HTML page that is not UTF-8", async () => { + // 0xE9 is "é" in Latin-1 and never a complete UTF-8 sequence + stubReadAsset({ buffer: Buffer.from([0x3c, 0x70, 0x3e, 0xe9]), extension: ".html" }) + + await expect( + assetOperations.readAssetContent({ ...defaultParams, path: "latin1.html" }, logger), + ).rejects.toThrow(new Error('not valid UTF-8: "latin1.html" cannot be returned as text')) + }) + + it("keeps reading a log that opens with NUL bytes, since the dedicated text formats skip the NUL check", async () => { + const buffer = Buffer.concat([Buffer.alloc(4), Buffer.from("boot ok\n", "utf8")]) + stubReadAsset({ buffer, extension: ".log" }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "server.log" }, + logger, + ) + + expect(result).toEqual({ kind: "text", text: buffer.toString("utf8"), path: "server.log" }) + }) +}) + describe("readAssetContent — line paging", () => { /** Stubs readAsset with a UTF-8 text file of the given extension. */ const stubReadAsset = (content: string, extension: string): void => { diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 5f15be56c..14a49d24c 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -7,6 +7,7 @@ import { pageTextByLines } from "../obsidian-markdown/lines.js" import type { LineWindow } from "../obsidian-markdown/lines.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" import { canvasImport, createPdfDocumentProxy } from "../obsidian-markdown/pdf-engine.js" +import { decodeTextBuffer, NUL_SEARCH_WINDOW_BYTES } from "../../utils/decode-text-buffer.js" import { describeErrorForLog } from "../../utils/describe-error.js" import { fitImageToByteBudget } from "../../utils/fit-image-to-byte-budget.js" import type { FittedImage } from "../../utils/fit-image-to-byte-budget.js" @@ -19,9 +20,9 @@ import type { Logger } from "../../logger.js" * * - `readAssetContent` dispatches one file to its most useful representation: * images fitted to a byte budget, canvases linearized (or their raw JSON - * source), PDFs as extracted text (or rendered page images), text formats - * decoded verbatim — whole, or as a 1-based line window when paging params - * are supplied — structured errors for the rest. + * source), PDFs as extracted text (or rendered page images), and every other + * file decoded verbatim as UTF-8 text — whole, or as a 1-based line window + * when paging params are supplied. * - `buildAssetListing` browses many: extension-filtered, counted per * extension over the full filtered set, and capped to a statted slice. * There is no pagination — `limit` caps the returned entries, and @@ -36,8 +37,9 @@ import type { Logger } from "../../logger.js" /** Extensions dispatched to the image pipeline (model-visible image blocks). */ const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp"]) -/** Extensions returned verbatim as text — plain-text formats an agent can - * read directly. .svg is XML source; .base is Obsidian Bases YAML. */ +/** Extensions returned verbatim as text with no NUL byte check, so a + * NUL-padded log still reads. .svg is XML source; .base is Obsidian Bases + * YAML. */ const TEXT_PASSTHROUGH_EXTENSIONS = new Set([ ".svg", ".json", @@ -167,6 +169,36 @@ const decodeUtf8Strict = (params: { buffer: Buffer; path: string }): string => { } } +/** Builds the text result for a file outside the dedicated types (HTML pages, + * scripts, files with no extension), refusing binary data and bytes that are + * not UTF-8. */ +const buildUtf8SourceResult = (params: { + buffer: Buffer + bytes: number + extension: string + path: string + startLine?: number | undefined + limit?: number | undefined +}): AssetReadResult => { + const { path, startLine, limit } = params + const decodedSource = decodeTextBuffer(params.buffer) + + switch (decodedSource.status) { + case "text": + return buildTextResult({ text: decodedSource.text, path, startLine, limit }) + case "binary": + throw new Error( + `unsupported file type "${params.extension}": "${path}" exists ` + + `(${params.bytes} bytes) and holds a NUL byte in its first ` + + `${NUL_SEARCH_WINDOW_BYTES.toLocaleString("en-US")} bytes (binary data or UTF-16 text). ` + + `Readable types: images (${[...IMAGE_EXTENSIONS].join("/")}), .canvas, .pdf, ` + + `and UTF-8 text files`, + ) + case "invalid-utf8": + throw new Error(`not valid UTF-8: "${path}" cannot be returned as text`) + } +} + // ── PDF reading ─────────────────────────────────────────────── /** The server's message for pdf.js refusing to open a vault file, worded like @@ -337,9 +369,9 @@ const readPdfPages = async ( * outline or raw JSON, PDF-extracted text), so every path that can hit the * text cap can also be paged. * - Throws structured errors for images with `raw`, non-text reads with - * paging params, PDFs that can't be opened or yield nothing, and - * unsupported types; the PDF and unsupported-type errors state the file's - * existence and size. + * paging params, PDFs that can't be opened or yield nothing, binary data + * outside the dedicated types, and text that is not UTF-8; the PDF and + * binary-data errors state the file's existence and size. */ const readAssetContent = async ( params: { @@ -438,12 +470,14 @@ const readAssetContent = async ( limit, }) } - throw new Error( - `unsupported file type "${asset.extension}": "${path}" exists ` + - `(${asset.bytes} bytes). Readable types: images ` + - `(${[...IMAGE_EXTENSIONS].join("/")}), .canvas, .pdf, and text formats ` + - `(${[...TEXT_PASSTHROUGH_EXTENSIONS].join("/")})`, - ) + return buildUtf8SourceResult({ + buffer: asset.buffer, + bytes: asset.bytes, + extension: asset.extension, + path, + startLine, + limit, + }) } // ── Browsing ──────────────────────────────────────────────────── From 76c8cd0d3f7c0121b3cdcc4e1a0e55259dedf448 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:05:32 -0500 Subject: [PATCH 02/38] fix(search): index and link files whatever the letter case of their extension One file-kind helper (obsidian-markdown/file-kinds.ts) maps a lowercased extension to a kind, and every extension check in reading, indexing, the note mover and the link tools uses it. A scanner's .PDF, an .TXT or a .CSV is now searchable, a .Canvas's file cards count as links and are rewritten when a note moves, and search results report the extension lowercased, as vault_list_files does. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/file-kinds.test.ts | 96 ++++++++ src/vault-mcp/obsidian-markdown/file-kinds.ts | 48 ++++ .../search/__tests__/file-watcher.test.ts | 107 +++++++++ .../search/__tests__/search-index.test.ts | 129 ++++++++-- src/vault-mcp/search/file-watcher.ts | 23 +- src/vault-mcp/search/hybrid-search.ts | 22 +- src/vault-mcp/search/search-helpers.ts | 3 +- src/vault-mcp/search/search-index.ts | 35 +-- src/vault-mcp/search/search-queries.ts | 12 +- .../__tests__/note-mover.test.ts | 59 +++-- .../vault-operations/asset-operations.ts | 225 ++++++++++-------- src/vault-mcp/vault-operations/note-mover.ts | 5 +- 12 files changed, 589 insertions(+), 175 deletions(-) create mode 100644 src/vault-mcp/obsidian-markdown/__tests__/file-kinds.test.ts create mode 100644 src/vault-mcp/obsidian-markdown/file-kinds.ts diff --git a/src/vault-mcp/obsidian-markdown/__tests__/file-kinds.test.ts b/src/vault-mcp/obsidian-markdown/__tests__/file-kinds.test.ts new file mode 100644 index 000000000..433687bd3 --- /dev/null +++ b/src/vault-mcp/obsidian-markdown/__tests__/file-kinds.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect } from "vitest" +import { getExtensionsOfKind, getFileKind, isCanvasPath } from "../file-kinds.js" + +// Test-owned, so a change to the production map has to change this table too +const EXPECTED_KINDS = [ + { extension: ".png", kind: "image" }, + { extension: ".jpg", kind: "image" }, + { extension: ".jpeg", kind: "image" }, + { extension: ".gif", kind: "image" }, + { extension: ".webp", kind: "image" }, + { extension: ".canvas", kind: "canvas" }, + { extension: ".pdf", kind: "pdf" }, + { extension: ".txt", kind: "text" }, + { extension: ".csv", kind: "text" }, + { extension: ".json", kind: "text" }, + { extension: ".xml", kind: "text" }, + { extension: ".svg", kind: "text" }, + { extension: ".log", kind: "text" }, + { extension: ".yaml", kind: "text" }, + { extension: ".yml", kind: "text" }, + { extension: ".base", kind: "text" }, + { extension: ".html", kind: "html" }, + { extension: ".htm", kind: "html" }, + { extension: ".py", kind: "script" }, + { extension: ".sh", kind: "script" }, + { extension: ".ps1", kind: "script" }, +] as const + +/** Uppercases every second letter of an extension: ".canvas" becomes ".cAnVaS". */ +const toMixedCase = (extension: string): string => { + return [...extension] + .map((character, index) => (index % 2 === 0 ? character.toUpperCase() : character)) + .join("") +} + +describe("getFileKind", () => { + it.each(EXPECTED_KINDS)("maps $extension to $kind", ({ extension, kind }) => { + expect(getFileKind(`Folder/file${extension}`)).toBe(kind) + }) + + it.each(EXPECTED_KINDS)("maps an uppercase $extension to $kind", ({ extension, kind }) => { + expect(getFileKind(`Folder/FILE${extension.toUpperCase()}`)).toBe(kind) + }) + + it.each(EXPECTED_KINDS)("maps a mixed-case $extension to $kind", ({ extension, kind }) => { + expect(getFileKind(`Folder/file${toMixedCase(extension)}`)).toBe(kind) + }) + + it("maps a file with no extension to other", () => { + expect(getFileKind("scripts/Makefile")).toBe("other") + }) + + it("maps an extension the map does not list to other", () => { + expect(getFileKind("audio/song.mp3")).toBe("other") + }) + + it("maps a markdown note to other, since notes are not files", () => { + expect(getFileKind("Projects/plan.md")).toBe("other") + }) + + it("reads the kind from the last extension only", () => { + expect(getFileKind("exports/report.pdf.txt")).toBe("text") + }) + + it("maps a dot-prefixed name with no other dot to other", () => { + expect(getFileKind("config/.pdf")).toBe("other") + }) +}) + +describe("isCanvasPath", () => { + it("accepts a canvas in any letter case", () => { + expect(["Board.canvas", "Board.Canvas", "Board.CANVAS"].map(isCanvasPath)).toEqual([ + true, + true, + true, + ]) + }) + + it("rejects a note and other files", () => { + expect(["Board.md", "Board.canvas.md", "Board.json"].map(isCanvasPath)).toEqual([ + false, + false, + false, + ]) + }) +}) + +describe("getExtensionsOfKind", () => { + it("lists the image extensions in order", () => { + expect(getExtensionsOfKind("image")).toEqual([".png", ".jpg", ".jpeg", ".gif", ".webp"]) + }) + + it("lists nothing for other, which covers unlisted extensions", () => { + expect(getExtensionsOfKind("other")).toEqual([]) + }) +}) diff --git a/src/vault-mcp/obsidian-markdown/file-kinds.ts b/src/vault-mcp/obsidian-markdown/file-kinds.ts new file mode 100644 index 000000000..74c64ca7b --- /dev/null +++ b/src/vault-mcp/obsidian-markdown/file-kinds.ts @@ -0,0 +1,48 @@ +import { posix } from "node:path" + +/** What a non-markdown vault file holds, which decides how it is read and + * indexed. "other" covers every extension the map below does not list, and + * files with no extension. */ +export type FileKind = "image" | "canvas" | "pdf" | "text" | "html" | "script" | "other" + +/** Keys are lowercase, and lookups lowercase the extension first, so a + * scanner's ".PDF" is a PDF. .svg is XML source; .base is Obsidian Bases + * YAML. */ +const FILE_KINDS_BY_EXTENSION: ReadonlyMap = new Map([ + [".png", "image"], + [".jpg", "image"], + [".jpeg", "image"], + [".gif", "image"], + [".webp", "image"], + [".canvas", "canvas"], + [".pdf", "pdf"], + [".txt", "text"], + [".csv", "text"], + [".json", "text"], + [".xml", "text"], + [".svg", "text"], + [".log", "text"], + [".yaml", "text"], + [".yml", "text"], + [".base", "text"], + [".html", "html"], + [".htm", "html"], + [".py", "script"], + [".sh", "script"], + [".ps1", "script"], +]) + +export const getFileKind = (filePath: string): FileKind => { + return FILE_KINDS_BY_EXTENSION.get(posix.extname(filePath).toLowerCase()) ?? "other" +} + +export const isCanvasPath = (filePath: string): boolean => { + return getFileKind(filePath) === "canvas" +} + +/** The lowercase extensions mapped to `kind`, in the map's order. */ +export const getExtensionsOfKind = (kind: FileKind): string[] => { + return [...FILE_KINDS_BY_EXTENSION] + .filter(([, mappedKind]) => mappedKind === kind) + .map(([extension]) => extension) +} diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 35f81e143..6d6ef662c 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -501,6 +501,113 @@ describe("file-watcher — file content indexing", () => { ]) }) + it.each([ + { + label: ".TXT", + fileName: "upper.TXT", + content: "crimson memo\n", + query: "crimson", + extension: ".txt", + }, + { + label: ".CSV", + fileName: "Prices.CSV", + content: "id,item\n1,saffron\n", + query: "saffron", + extension: ".csv", + }, + ])( + "indexes a file with an uppercase $label extension and removes it on delete", + { timeout: 15000 }, + async ({ fileName, content, query, extension }) => { + const fileIndex = createSearchIndex(":memory:", undefined, undefined, { + fileToolsEnabled: true, + }) + const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") + const removeSpy = vi.spyOn(fileIndex, "removeFileContent") + + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + REAL_WATCHER_OPTIONS, + ) + await writeFile(join(vault, fileName), content, "utf8") + await waitFor(() => upsertSpy.mock.calls.length > 0) + + const indexed = await fileIndex.hybridSearch({ query }, logger) + expect( + indexed.results.map((result) => ({ path: result.path, extension: result.extension })), + ).toEqual([{ path: fileName, extension }]) + + await unlink(join(vault, fileName)) + await waitFor(() => removeSpy.mock.calls.length > 0) + + const removed = await fileIndex.hybridSearch({ query }, logger) + expect(removed.results).toEqual([]) + }, + ) + + it( + "indexes a PDF with an uppercase extension and removes it on delete", + { timeout: 15000 }, + async () => { + const { buildMinimalPdf } = await import("../../obsidian-markdown/__tests__/pdf-fixture.js") + const fileIndex = createSearchIndex(":memory:", undefined, undefined, { + fileToolsEnabled: true, + }) + const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") + const removeSpy = vi.spyOn(fileIndex, "removeFileContent") + + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + REAL_WATCHER_OPTIONS, + ) + await writeFile(join(vault, "Report.PDF"), buildMinimalPdf()) + await waitFor(() => upsertSpy.mock.calls.length > 0) + + const indexed = await fileIndex.hybridSearch({ query: "Hello PDF" }, logger) + expect( + indexed.results.map((result) => ({ path: result.path, extension: result.extension })), + ).toEqual([{ path: "Report.PDF", extension: ".pdf" }]) + + await unlink(join(vault, "Report.PDF")) + await waitFor(() => removeSpy.mock.calls.length > 0) + + const removed = await fileIndex.hybridSearch({ query: "Hello PDF" }, logger) + expect(removed.results).toEqual([]) + }, + ) + + it( + "links the file cards of a canvas with an uppercase extension and drops them on delete", + { timeout: 15000 }, + async () => { + const fileIndex = createSearchIndex(":memory:", undefined, undefined, { + fileToolsEnabled: true, + }) + const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") + const removeSpy = vi.spyOn(fileIndex, "removeFileContent") + const canvasSource = JSON.stringify({ + nodes: [{ id: "f1", type: "file", x: 0, y: 0, width: 200, height: 100, file: "Plan.md" }], + edges: [], + }) + + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + REAL_WATCHER_OPTIONS, + ) + await writeFile(join(vault, "Board.Canvas"), canvasSource, "utf8") + await waitFor(() => upsertSpy.mock.calls.length > 0) + + const linkedTargets = fileIndex.getOutgoingLinks({ path: "Board.Canvas" }, logger) + expect(linkedTargets.map((link) => link.path)).toEqual(["Plan.md"]) + + await unlink(join(vault, "Board.Canvas")) + await waitFor(() => removeSpy.mock.calls.length > 0) + + expect(fileIndex.getOutgoingLinks({ path: "Board.Canvas" }, logger)).toEqual([]) + }, + ) + it("calls embedFileContent when indexing a non-md file", { timeout: 15000 }, async () => { const fileIndex = createSearchIndex(":memory:", undefined, undefined, { fileToolsEnabled: true, diff --git a/src/vault-mcp/search/__tests__/search-index.test.ts b/src/vault-mcp/search/__tests__/search-index.test.ts index c0a0ee355..550741ce7 100644 --- a/src/vault-mcp/search/__tests__/search-index.test.ts +++ b/src/vault-mcp/search/__tests__/search-index.test.ts @@ -7,12 +7,13 @@ import Database from "better-sqlite3" import { DateTime } from "luxon" import * as sqliteVec from "sqlite-vec" vi.mock("sqlite-vec", { spy: true }) -import { createSearchIndex, INDEXABLE_TEXT_EXTENSIONS } from "../search-index.js" +import { createSearchIndex, CONTENT_INDEXED_FILE_KINDS } from "../search-index.js" import type { NoteMetadata, OutgoingLinkEntry, SearchIndex, TaskEntry } from "../search-index.js" import type { StatusClassification } from "../../obsidian-markdown/tasks.js" import { logger } from "../../../logger.js" import { statOrNull } from "../../../utils/fs.js" import { extractPdfText } from "../../obsidian-markdown/pdf.js" +import { buildPdf } from "../../obsidian-markdown/__tests__/pdf-fixture.js" import type { BodyTagParser, BodyTagParseResult } from "../body-tag-parser.js" import { inProcessBodyTagParser, withBodyTags } from "./body-tag-test-helpers.js" @@ -7038,20 +7039,9 @@ Shared datefilter content for mtime boundary tests. // ── File content indexing constants ────────────────────────── -describe("INDEXABLE_TEXT_EXTENSIONS", () => { - it("contains the expected set of extensions for FTS indexing", () => { - expect([...INDEXABLE_TEXT_EXTENSIONS].sort()).toEqual([ - ".base", - ".csv", - ".json", - ".log", - ".pdf", - ".svg", - ".txt", - ".xml", - ".yaml", - ".yml", - ]) +describe("CONTENT_INDEXED_FILE_KINDS", () => { + it("indexes the content of PDF and text files", () => { + expect([...CONTENT_INDEXED_FILE_KINDS].toSorted()).toEqual(["pdf", "text"]) }) }) @@ -7847,6 +7837,115 @@ describe("canvas file content and links", () => { // ── File content vector embeddings ──────────────────────────── +// Each test rebuilds a vault holding two PDFs, whose text extraction can pass +// the default 5-second limit on a loaded machine +describe("files with an uppercase extension", { timeout: 15_000 }, () => { + /** A canvas holding one text card and one file card naming `cardPath`. */ + const buildCanvasWithFileCard = (params: { cardPath: string; text: string }): string => { + return JSON.stringify({ + nodes: [ + { id: "t1", type: "text", x: 0, y: 0, width: 200, height: 100, text: params.text }, + { id: "f1", type: "file", x: 300, y: 0, width: 200, height: 100, file: params.cardPath }, + ], + edges: [], + }) + } + + /** Each file holds one word no other file holds, so a search for that word + * finds only that file. Every uppercase file has a lowercase control. */ + const writeMixedCaseVault = async (): Promise => { + const vaultDir = await mkdtemp(join(tmpdir(), "uppercase-extensions-")) + onTestFinished(() => rm(vaultDir, { recursive: true })) + await mkdir(join(vaultDir, "Notes")) + await mkdir(join(vaultDir, "files")) + await writeFile(join(vaultDir, "Notes/Plan.md"), "# Plan\n", "utf8") + await writeFile(join(vaultDir, "Notes/Route.md"), "# Route\n", "utf8") + await writeFile(join(vaultDir, "Notes/Index.md"), "See [[files/Board.Canvas]].\n", "utf8") + await writeFile(join(vaultDir, "files/upper.TXT"), "crimson memo\n", "utf8") + await writeFile(join(vaultDir, "files/lower.txt"), "teal memo\n", "utf8") + await writeFile(join(vaultDir, "files/Prices.CSV"), "id,item\n1,saffron\n", "utf8") + await writeFile(join(vaultDir, "files/costs.csv"), "id,item\n1,umber\n", "utf8") + await writeFile( + join(vaultDir, "files/Report.PDF"), + buildPdf([{ text: "amber ledger", x: 100, y: 700, fontSize: 12 }]), + ) + await writeFile( + join(vaultDir, "files/control.pdf"), + buildPdf([{ text: "violet ledger", x: 100, y: 700, fontSize: 12 }]), + ) + await writeFile( + join(vaultDir, "files/Board.Canvas"), + buildCanvasWithFileCard({ cardPath: "Notes/Plan.md", text: "indigo board" }), + "utf8", + ) + await writeFile( + join(vaultDir, "files/control.canvas"), + buildCanvasWithFileCard({ cardPath: "Notes/Route.md", text: "olive board" }), + "utf8", + ) + return vaultDir + } + + const rebuildMixedCaseIndex = async (): Promise => { + const vaultDir = await writeMixedCaseVault() + const fileIndex = createSearchIndex(":memory:", undefined, undefined, { + fileToolsEnabled: true, + }) + await fileIndex.rebuildFromVault( + { vaultPath: vaultDir, bodyTagParser: inProcessBodyTagParser }, + logger, + ) + return fileIndex + } + + it("finds every file by its content after a rebuild, reporting its extension lowercased", async () => { + const fileIndex = await rebuildMixedCaseIndex() + const queries = ["crimson", "teal", "saffron", "umber", "amber", "violet", "indigo", "olive"] + + const searchFor = async (query: string) => { + const { results } = await fileIndex.hybridSearch({ query }, logger) + return results.map(({ path, extension }) => ({ path, extension })) + } + + expect(await Promise.all(queries.map(searchFor))).toEqual([ + [{ path: "files/upper.TXT", extension: ".txt" }], + [{ path: "files/lower.txt", extension: ".txt" }], + [{ path: "files/Prices.CSV", extension: ".csv" }], + [{ path: "files/costs.csv", extension: ".csv" }], + [{ path: "files/Report.PDF", extension: ".pdf" }], + [{ path: "files/control.pdf", extension: ".pdf" }], + [{ path: "files/Board.Canvas", extension: ".canvas" }], + [{ path: "files/control.canvas", extension: ".canvas" }], + ]) + }) + + it("lists the file cards of a canvas with an uppercase extension as its outgoing links", async () => { + const fileIndex = await rebuildMixedCaseIndex() + + const outgoingLinks = fileIndex.getOutgoingLinks({ path: "files/Board.Canvas" }, logger) + + expect(outgoingLinks.map(({ path, exists }) => ({ path, exists }))).toEqual([ + { path: "Notes/Plan.md", exists: true }, + ]) + }) + + it("lists a canvas with an uppercase extension among a note's backlinks", async () => { + const fileIndex = await rebuildMixedCaseIndex() + + const backlinks = fileIndex.getBacklinks({ path: "Notes/Plan.md" }, logger) + + expect(backlinks.map((backlink) => backlink.path)).toEqual(["files/Board.Canvas"]) + }) + + it("lists the notes linking to a canvas with an uppercase extension", async () => { + const fileIndex = await rebuildMixedCaseIndex() + + const backlinks = fileIndex.getBacklinks({ path: "files/Board.Canvas" }, logger) + + expect(backlinks.map((backlink) => backlink.path)).toEqual(["Notes/Index.md"]) + }) +}) + describe("committed embedding source versions", () => { it.each(["note", "file"] as const)( "rejects a %s model result after source deletion", diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 9fd6007f8..a339e1d1a 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -7,9 +7,9 @@ import { watch } from "chokidar" import { DateTime } from "luxon" import { readFile, readdir, stat } from "node:fs/promises" import type { Stats } from "node:fs" -import { extname, join, relative, resolve as resolvePath, sep } from "node:path" +import { join, relative, resolve as resolvePath, sep } from "node:path" import { - INDEXABLE_TEXT_EXTENSIONS, + CONTENT_INDEXED_FILE_KINDS, INDEXING_FILE_READ_CONCURRENCY, PDF_EXTRACTION_CONCURRENCY, } from "./search-index.js" @@ -21,6 +21,7 @@ import { } from "./body-tag-parser.js" import { splitPropertiesBlock } from "../obsidian-markdown/frontmatter.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" +import { getFileKind, isCanvasPath } from "../obsidian-markdown/file-kinds.js" import type { Logger } from "../../logger.js" import { describeError, describeErrorForLog } from "../../utils/describe-error.js" import { isErrnoException } from "../../utils/is-errno-exception.js" @@ -225,17 +226,16 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } // Canvas files are always read — link extraction is unconditional. - // PDF and text files (INDEXABLE_TEXT_EXTENSIONS, which includes .pdf) are - // only read when file content FTS is enabled. - const extension = extname(filePath) - const isCanvas = extension === ".canvas" + // PDF and text files are only read when file content FTS is enabled. + const fileKind = getFileKind(filePath) + const isCanvas = fileKind === "canvas" const isIndexableNonCanvas = - search.fileContentIndexingEnabled && INDEXABLE_TEXT_EXTENSIONS.has(extension) + search.fileContentIndexingEnabled && CONTENT_INDEXED_FILE_KINDS.has(fileKind) if (isCanvas || isIndexableNonCanvas) { try { const readContentToIndex = async (): Promise => { - if (extension !== ".pdf") return await readFile(filePath, "utf8") + if (fileKind !== "pdf") return await readFile(filePath, "utf8") const buffer = await readFile(filePath) // pdfjs throws on a Node Buffer, so it gets a plain Uint8Array view @@ -518,10 +518,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW // The content rows live in their own tables, so they are removed even // when the asset-table removal above failed. - const deletedExtension = extname(filePath) - const isDeletedCanvas = deletedExtension === ".canvas" + const isDeletedCanvas = isCanvasPath(filePath) const isDeletedIndexable = - search.fileContentIndexingEnabled && INDEXABLE_TEXT_EXTENSIONS.has(deletedExtension) + search.fileContentIndexingEnabled && CONTENT_INDEXED_FILE_KINDS.has(getFileKind(filePath)) if (isDeletedCanvas || isDeletedIndexable) { try { @@ -914,7 +913,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW // indexChangedNonMdFile extracts a PDF's text only when file content // indexing is on; otherwise a PDF is only statted and keeps the general bound. const needsPdfExtraction = ({ filePath }: ReservedPath): boolean => { - return search.fileContentIndexingEnabled && extname(filePath) === ".pdf" + return search.fileContentIndexingEnabled && getFileKind(filePath) === "pdf" } const pdfReads = reservedPaths.filter(needsPdfExtraction) const otherReads = reservedPaths.filter((reservedPath) => !needsPdfExtraction(reservedPath)) diff --git a/src/vault-mcp/search/hybrid-search.ts b/src/vault-mcp/search/hybrid-search.ts index 27f43c318..63fe95447 100644 --- a/src/vault-mcp/search/hybrid-search.ts +++ b/src/vault-mcp/search/hybrid-search.ts @@ -4,6 +4,7 @@ import type Database from "better-sqlite3" import type { Logger } from "../../logger.js" import { describeError } from "../../utils/describe-error.js" import { compareByUtf8Bytes } from "../../utils/compare-utf8-bytes.js" +import { getFileKind } from "../obsidian-markdown/file-kinds.js" import { sanitizeFtsQuery } from "./fts-query.js" import { computeRrfScores } from "./rrf.js" import { blendScores } from "./reranker.js" @@ -46,11 +47,20 @@ const DEFAULT_FILE_LEG_WEIGHT = 0.5 * kind that is off-topic for the query. Overridable like the leg weight. */ const DEFAULT_RERANK_KIND_PREFIX = false -/** Kind label for the reranker prefix, from the file's extension. */ -const fileKindLabel = (extension: string | undefined): string => { - if (extension === ".pdf") return "PDF file" - if (extension === ".canvas") return "Canvas file" - return "File" +/** Kind label for the reranker prefix. */ +const fileKindLabel = (filePath: string): string => { + switch (getFileKind(filePath)) { + case "pdf": + return "PDF file" + case "canvas": + return "Canvas file" + case "image": + case "text": + case "html": + case "script": + case "other": + return "File" + } } // ── Vector search (internal) ─────────────────────────────────── @@ -299,7 +309,7 @@ const tryRerank = async (params: { // Kind marker only — the chunk text already opens with the file's // title, so prefixing the filename again would just repeat it. if (params.rerankKindPrefix && result.kind === "file") { - return `${fileKindLabel(result.extension)}: ${documentText}` + return `${fileKindLabel(result.path)}: ${documentText}` } return documentText }) diff --git a/src/vault-mcp/search/search-helpers.ts b/src/vault-mcp/search/search-helpers.ts index 361c07e34..20306be71 100644 --- a/src/vault-mcp/search/search-helpers.ts +++ b/src/vault-mcp/search/search-helpers.ts @@ -321,7 +321,8 @@ export const fileContentRowToSearchResult = ( folder: row.folder, type: null, kind: "file", - extension: posix.extname(row.path) || undefined, + // Lowercased, as vault_list_files reports it + extension: posix.extname(row.path).toLowerCase() || undefined, modified: storedMtimeToIso(row.mtime), bytes: row.bytes, }) diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index d9dafc936..031eb7337 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -9,6 +9,8 @@ import type { Logger } from "../../logger.js" import { parseLeadingCallout } from "../obsidian-markdown/callouts.js" import type { LeadingCallout } from "../obsidian-markdown/callouts.js" import { linearizeCanvas, extractCanvasFileLinks } from "../obsidian-markdown/canvas.js" +import { getFileKind, isCanvasPath } from "../obsidian-markdown/file-kinds.js" +import type { FileKind } from "../obsidian-markdown/file-kinds.js" import { links } from "../obsidian-markdown/links.js" import { splitIntoLines } from "../obsidian-markdown/lines.js" import { parseHeadings } from "../obsidian-markdown/headings.js" @@ -62,20 +64,10 @@ export const INDEXING_FILE_READ_CONCURRENCY = 16 * CPU-heavy, so fewer run together than other file reads. */ export const PDF_EXTRACTION_CONCURRENCY = 4 -/** Extensions whose content is indexed into file_content_fts for vault_search. - * Excludes .canvas (own path with link extraction) and .md (notes_fts). */ -export const INDEXABLE_TEXT_EXTENSIONS = new Set([ - ".pdf", - ".txt", - ".csv", - ".json", - ".xml", - ".svg", - ".log", - ".yaml", - ".yml", - ".base", -]) +/** File kinds whose content is indexed into file_content_fts when file tools + * are on. A canvas is left out because it is read whatever that setting is, + * since its file cards are graph links. */ +export const CONTENT_INDEXED_FILE_KINDS: ReadonlySet = new Set(["pdf", "text"]) /** Cap on rendered content indexed into file_content_fts — prevents * pathological FTS rows from large CSVs or verbose JSONs. */ @@ -1331,10 +1323,10 @@ export const createSearchIndex = ( logger: Logger, ): symbol => { const extension = posix.extname(params.filePath) - const isCanvas = extension === ".canvas" + const isCanvas = isCanvasPath(params.filePath) - // Canvas: linearize for FTS, raw JSON for link extraction. - // Other types: rawContent is already the rendered text. + // A canvas is linearized for FTS, and its raw JSON is read for links. + // Every other kind arrives as its rendered text. const contentToIndex = isCanvas ? linearizeCanvas(params.rawContent) : params.rawContent // Truncate content exceeding the FTS cap — iterates Unicode code points @@ -2316,17 +2308,14 @@ export const createSearchIndex = ( mapper: readFileSize, }) const nonMarkdownFileSizes = nonMarkdownFileSizeResults.filter((entry) => entry !== null) - const canvasFiles = allNonMdFiles.filter((file) => file.relativePath.endsWith(".canvas")) + const canvasFiles = allNonMdFiles.filter((file) => isCanvasPath(file.relativePath)) // PDF and text files are only read when file content FTS is enabled — // without the tables, the extraction is wasted I/O. const pdfFiles = fileToolsEnabled - ? allNonMdFiles.filter((file) => file.relativePath.endsWith(".pdf")) + ? allNonMdFiles.filter((file) => getFileKind(file.relativePath) === "pdf") : [] const textFiles = fileToolsEnabled - ? allNonMdFiles.filter((file) => { - const extension = posix.extname(file.relativePath) - return extension !== ".pdf" && INDEXABLE_TEXT_EXTENSIONS.has(extension) - }) + ? allNonMdFiles.filter((file) => getFileKind(file.relativePath) === "text") : [] const readRebuildFileContent = async (params: { diff --git a/src/vault-mcp/search/search-queries.ts b/src/vault-mcp/search/search-queries.ts index d561ee337..09d8cf526 100644 --- a/src/vault-mcp/search/search-queries.ts +++ b/src/vault-mcp/search/search-queries.ts @@ -6,6 +6,7 @@ import { classifyIsoDate } from "../../utils/classify-iso-date.js" import { describeError } from "../../utils/describe-error.js" import { compareByUtf8Bytes } from "../../utils/compare-utf8-bytes.js" import { assertPathHasExtension } from "../../utils/assert-path-has-extension.js" +import { isCanvasPath } from "../obsidian-markdown/file-kinds.js" import { sanitizeFtsQuery, sanitizeFtsQueryAnyTerm } from "./fts-query.js" import { computeRrfScores } from "./rrf.js" import { sigmoid } from "./reranker.js" @@ -1467,6 +1468,13 @@ export const searchByProperty = ( // ── Link queries ─────────────────────────────────────────────── +/** A link query takes a note, whose ".md" must be lowercase, or a canvas in + * any letter case. */ +const assertLinkQueryPath = (path: string): void => { + if (isCanvasPath(path)) return + assertPathHasExtension(path, [".md", ".canvas"]) +} + /** Returns notes and files that link TO the given path (incoming links / * backlinks). Canvas file-node references are included — a canvas that * embeds a note via a `file`-type node appears as a backlink source. */ @@ -1475,7 +1483,7 @@ export const getBacklinks = ( params: { path: string }, logger: Logger, ): BacklinkEntry[] => { - assertPathHasExtension(params.path, [".md", ".canvas"]) + assertLinkQueryPath(params.path) const sql = ` SELECT l.source as path, COALESCE(n.title, f.basename) as title, @@ -1516,7 +1524,7 @@ export const getOutgoingLinks = ( params: { path: string; dailyNotesFolder?: string | null }, logger: Logger, ): OutgoingLinkEntry[] => { - assertPathHasExtension(params.path, [".md", ".canvas"]) + assertLinkQueryPath(params.path) // Left-join against both notes and non_md_files to classify each link target: // notes → kind "note", non_md_files → kind "file", neither → broken (defaults to "note"). const sql = ` diff --git a/src/vault-mcp/vault-operations/__tests__/note-mover.test.ts b/src/vault-mcp/vault-operations/__tests__/note-mover.test.ts index 1cef3f302..21113552a 100644 --- a/src/vault-mcp/vault-operations/__tests__/note-mover.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/note-mover.test.ts @@ -1266,29 +1266,58 @@ describe("moveNote — Canvas sources", () => { expect(vi.mocked(logger.warn)).not.toHaveBeenCalled() }) - it("treats only a lowercase .canvas extension as a Canvas", async () => { - const { writeFixture, moveNote, readNote, logger } = setupVault() + it("points the file card of a Canvas with an uppercase extension at the moved note's new path", async () => { + const { writeFixture, moveNote, readNote } = setupVault() + await writeFixture("Plan.md", "# Plan\n") + await writeFixture( + "Board.Canvas", + obsidianCanvas([ + '{"id":"f1","type":"file","file":"Plan.md","x":0,"y":0,"width":400,"height":400}', + ]), + ) + + const result = await moveNote({ + oldPath: "Plan.md", + newPath: "Roadmap.md", + backlinkSources: ["Board.Canvas"], + }) + + expect(await readNote("Board.Canvas")).toBe( + obsidianCanvas([ + '{"id":"f1","type":"file","file":"Roadmap.md","x":0,"y":0,"width":400,"height":400}', + ]), + ) + expect(result).toEqual({ + moved_to: "Roadmap.md", + links_updated: 1, + updated_notes: ["Board.Canvas"], + pruned_empty_folders: 0, + }) + }) + + it("finds and rewrites a text-card link in a Canvas with an uppercase extension that the index missed", async () => { + const { writeFixture, moveNote, readNote } = setupVault() await writeFixture("Foo.md", "# Foo\n") - const upperCaseCanvas = obsidianCanvas([ - '{"id":"t1","type":"text","text":"Start from [[Foo]]","x":0,"y":0,"width":250,"height":60}', - ]) - await writeFixture("Board.CANVAS", upperCaseCanvas) + await writeFixture( + "Board.CANVAS", + obsidianCanvas([ + '{"id":"t1","type":"text","text":"Start from [[Foo]]","x":0,"y":0,"width":250,"height":60}', + ]), + ) const result = await moveNote({ oldPath: "Foo.md", newPath: "Bar.md" }) + expect(await readNote("Board.CANVAS")).toBe( + obsidianCanvas([ + '{"id":"t1","type":"text","text":"Start from [[Bar]]","x":0,"y":0,"width":250,"height":60}', + ]), + ) expect(result).toEqual({ moved_to: "Bar.md", - links_updated: 0, - updated_notes: [], + links_updated: 1, + updated_notes: ["Board.CANVAS"], pruned_empty_folders: 0, }) - expect(await readNote("Board.CANVAS")).toBe(upperCaseCanvas) - // The scan skips Board.CANVAS and no other file links Foo.md, so the log - // must never fire, whatever its context. - expect(vi.mocked(logger.info)).not.toHaveBeenCalledWith( - "backlink verification discovered sources the index missed", - expect.anything(), - ) }) it("aborts without writing when a Canvas backlink source is not valid JSON", async () => { diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 14a49d24c..94ec860e5 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -3,6 +3,7 @@ import { getMeta, renderPageAsImage } from "unpdf" import type { PDFDocumentProxy } from "unpdf/pdfjs" import { vaultFs } from "./vault-filesystem.js" import { linearizeCanvas } from "../obsidian-markdown/canvas.js" +import { getExtensionsOfKind, getFileKind } from "../obsidian-markdown/file-kinds.js" import { pageTextByLines } from "../obsidian-markdown/lines.js" import type { LineWindow } from "../obsidian-markdown/lines.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" @@ -34,25 +35,7 @@ import type { Logger } from "../../logger.js" // ── Reading ───────────────────────────────────────────────────── -/** Extensions dispatched to the image pipeline (model-visible image blocks). */ -const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp"]) - -/** Extensions returned verbatim as text with no NUL byte check, so a - * NUL-padded log still reads. .svg is XML source; .base is Obsidian Bases - * YAML. */ -const TEXT_PASSTHROUGH_EXTENSIONS = new Set([ - ".svg", - ".json", - ".txt", - ".csv", - ".xml", - ".log", - ".yaml", - ".yml", - ".base", -]) - -/** Fixed cap on text output (passthrough files, canvas renditions and +/** Fixed cap on text output (text files, canvas renditions and * PDF-extracted text) so a huge text asset can't blow a client's response * limit. Deliberately not an env var — the image budget is the tunable * surface; text past this size is read in line windows, not raised into a @@ -191,7 +174,7 @@ const buildUtf8SourceResult = (params: { `unsupported file type "${params.extension}": "${path}" exists ` + `(${params.bytes} bytes) and holds a NUL byte in its first ` + `${NUL_SEARCH_WINDOW_BYTES.toLocaleString("en-US")} bytes (binary data or UTF-16 text). ` + - `Readable types: images (${[...IMAGE_EXTENSIONS].join("/")}), .canvas, .pdf, ` + + `Readable types: images (${getExtensionsOfKind("image").join("/")}), .canvas, .pdf, ` + `and UTF-8 text files`, ) case "invalid-utf8": @@ -359,13 +342,103 @@ const readPdfPages = async ( } } +type ReadAsset = Readonly<{ buffer: Buffer; bytes: number }> + +/** Images have no raw source or line range, because the fitted image block is + * the only form delivered. */ +const readImageAsset = async (params: { + asset: ReadAsset + path: string + raw: boolean | undefined + isPagedRead: boolean + maxImageOutputBytes: number +}): Promise => { + const { asset, path } = params + + if (params.raw) { + throw new Error( + `raw source is not available for images: "${path}" is ` + + `binary — its image block is the delivered form`, + ) + } + if (params.isPagedRead) { + throw new Error( + `line range is not available for images: "${path}" is ` + + `binary — its image block is the delivered form`, + ) + } + + const fitted = await fitImageToByteBudget({ + buffer: asset.buffer, + budgetBytes: params.maxImageOutputBytes, + }) + return { kind: "image", fitted, originalBytes: asset.bytes, path } +} + +/** Reads a PDF as its extracted text, or with `raw` as rendered page images, + * which cannot be paged by line. */ +const readPdfAsset = async ( + params: { + asset: ReadAsset + path: string + raw: boolean | undefined + startLine: number | undefined + limit: number | undefined + maxPdfRenderPages: number + maxImageOutputBytes: number + }, + logger: Logger, +): Promise => { + const { asset, path, raw, startLine, limit } = params + + if (raw && requestsLineWindow({ startLine, limit })) { + throw new Error( + `line range is not available for rendered PDF pages: ` + + `"${path}" delivers page images, not text`, + ) + } + + // Buffer → Uint8Array view: Buffer.buffer may be Node's shared pool, + // so byteOffset/byteLength carve out this buffer's portion. + const pdfData = new Uint8Array( + asset.buffer.buffer, + asset.buffer.byteOffset, + asset.buffer.byteLength, + ) + + if (raw) { + return readPdfPages( + { + pdfData, + path, + bytes: asset.bytes, + maxPdfRenderPages: params.maxPdfRenderPages, + maxImageOutputBytes: params.maxImageOutputBytes, + }, + logger, + ) + } + + const pdfResult = await readPdfText({ pdfData, path, bytes: asset.bytes }) + + if (!pdfResult.text) { + throw new Error( + `PDF has no extractable text: "${path}" exists ` + + `(${asset.bytes} bytes, ${pdfResult.totalPages} pages) but ` + + `contains no text content — it may be a scanned document or ` + + `image-only PDF`, + ) + } + return buildTextResult({ text: pdfResult.text, path, startLine, limit }) +} + /** * Reads a non-markdown vault file and returns its most useful representation * per type. * - `raw` returns a canvas's exact JSON source instead of its rendition, and * a PDF's rendered page images instead of its text. * - `startLine`/`limit` page any text rendition as a 1-based line window. - * They apply after the rendition is produced (passthrough source, canvas + * They apply after the rendition is produced (a text file's source, canvas * outline or raw JSON, PDF-extracted text), so every path that can hit the * text cap can also be paged. * - Throws structured errors for images with `raw`, non-text reads with @@ -387,97 +460,55 @@ const readAssetContent = async ( logger: Logger, ): Promise => { const { path, raw, startLine, limit } = params - const isPagedRead = requestsLineWindow({ startLine, limit }) const asset = await vaultFs.readAsset( { vaultPath: params.vaultPath, path, maxBytes: params.maxFileBytes }, logger, ) - // readAsset reports the extension lowercased, as these lookups expect - const isImage = IMAGE_EXTENSIONS.has(asset.extension) - - if (isImage && raw) { - throw new Error( - `raw source is not available for images: "${path}" is ` + - `binary — its image block is the delivered form`, - ) - } - if (isImage && isPagedRead) { - throw new Error( - `line range is not available for images: "${path}" is ` + - `binary — its image block is the delivered form`, - ) - } - if (isImage) { - const fitted = await fitImageToByteBudget({ - buffer: asset.buffer, - budgetBytes: params.maxImageOutputBytes, - }) - return { kind: "image", fitted, originalBytes: asset.bytes, path } - } - if (asset.extension === ".canvas") { - const canvasSource = decodeUtf8Strict({ buffer: asset.buffer, path }) - const text = raw ? canvasSource : linearizeCanvas(canvasSource) - return buildTextResult({ text, path, startLine, limit }) - } - if (TEXT_PASSTHROUGH_EXTENSIONS.has(asset.extension)) { - const text = decodeUtf8Strict({ buffer: asset.buffer, path }) - return buildTextResult({ text, path, startLine, limit }) - } - if (asset.extension === ".pdf") { - if (raw && isPagedRead) { - throw new Error( - `line range is not available for rendered PDF pages: ` + - `"${path}" delivers page images, not text`, - ) + switch (getFileKind(path)) { + case "image": + return readImageAsset({ + asset, + path, + raw, + isPagedRead: requestsLineWindow({ startLine, limit }), + maxImageOutputBytes: params.maxImageOutputBytes, + }) + case "canvas": { + const canvasSource = decodeUtf8Strict({ buffer: asset.buffer, path }) + const text = raw ? canvasSource : linearizeCanvas(canvasSource) + return buildTextResult({ text, path, startLine, limit }) } - - // Buffer → Uint8Array view: Buffer.buffer may be Node's shared pool, - // so byteOffset/byteLength carve out this buffer's portion. - const pdfData = new Uint8Array( - asset.buffer.buffer, - asset.buffer.byteOffset, - asset.buffer.byteLength, - ) - - if (raw) { - return readPdfPages( + case "pdf": + return readPdfAsset( { - pdfData, + asset, path, - bytes: asset.bytes, + raw, + startLine, + limit, maxPdfRenderPages: params.maxPdfRenderPages, maxImageOutputBytes: params.maxImageOutputBytes, }, logger, ) + case "text": { + // No NUL byte check here, so a NUL-padded log keeps reading + const text = decodeUtf8Strict({ buffer: asset.buffer, path }) + return buildTextResult({ text, path, startLine, limit }) } - - const pdfResult = await readPdfText({ pdfData, path, bytes: asset.bytes }) - - if (!pdfResult.text) { - throw new Error( - `PDF has no extractable text: "${path}" exists ` + - `(${asset.bytes} bytes, ${pdfResult.totalPages} pages) but ` + - `contains no text content — it may be a scanned document or ` + - `image-only PDF`, - ) - } - return buildTextResult({ - text: pdfResult.text, - path, - startLine, - limit, - }) + case "html": + case "script": + case "other": + return buildUtf8SourceResult({ + buffer: asset.buffer, + bytes: asset.bytes, + extension: asset.extension, + path, + startLine, + limit, + }) } - return buildUtf8SourceResult({ - buffer: asset.buffer, - bytes: asset.bytes, - extension: asset.extension, - path, - startLine, - limit, - }) } // ── Browsing ──────────────────────────────────────────────────── diff --git a/src/vault-mcp/vault-operations/note-mover.ts b/src/vault-mcp/vault-operations/note-mover.ts index b121c274b..7b13604c5 100644 --- a/src/vault-mcp/vault-operations/note-mover.ts +++ b/src/vault-mcp/vault-operations/note-mover.ts @@ -47,6 +47,7 @@ import { extractCanvasFileLinks, rewriteCanvasLinks, } from "../obsidian-markdown/canvas.js" +import { isCanvasPath } from "../obsidian-markdown/file-kinds.js" import { withExclusiveMultiFileLock } from "../../utils/file-write-lock.js" import { caseFoldPath } from "../../utils/case-fold-path.js" import { mapWithConcurrency } from "../../utils/map-with-concurrency.js" @@ -467,10 +468,6 @@ const rewriteNoteContent = ( return { content, linksRewritten } } -/** Case-sensitive, like the search index: only a lowercase ".canvas" - * extension marks a file as a Canvas. */ -const isCanvasPath = (path: string): boolean => posix.extname(path) === ".canvas" - /** True when a file card's path names a note. Obsidian opens a file card by * its exact path, so a card for Plan.pdf opens the PDF even though the * resolver's ".md" fallback would match a note named Plan.pdf.md. */ From 372d4ddf68d6a627b9745586fde6ecdd3077f64e Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:09:24 -0500 Subject: [PATCH 03/38] feat(search): extract an HTML page's visible text extractHtmlVisibleText runs htmlparser2's stackless tokenizer, so unclosed tags cost no more than closed ones. Hidden elements end only at a closing tag with their own name, block elements become line breaks, whitespace collapses, and reading stops once the text reaches the cap. Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 91 ++++++++- package.json | 1 + .../obsidian-markdown/__tests__/html.test.ts | 121 ++++++++++++ src/vault-mcp/obsidian-markdown/html.ts | 175 ++++++++++++++++++ 4 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 src/vault-mcp/obsidian-markdown/__tests__/html.test.ts create mode 100644 src/vault-mcp/obsidian-markdown/html.ts diff --git a/package-lock.json b/package-lock.json index d466e88c9..424b95fef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "express": "5.3.0", "express-rate-limit": "8.7.3", "gray-matter": "4.0.3", + "htmlparser2": "12.0.0", "luxon": "3.7.2", "pdfjs-dist": "6.5.438", "picomatch": "4.0.7", @@ -5890,6 +5891,73 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/dom-serializer": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-3.1.1.tgz", + "integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==", + "license": "MIT", + "dependencies": { + "domelementtype": "^3.0.0", + "domhandler": "^6.0.0", + "entities": "^8.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-3.0.0.tgz", + "integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/domhandler": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-6.0.1.tgz", + "integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^3.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-4.0.2.tgz", + "integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^3.0.0", + "domelementtype": "^3.0.0", + "domhandler": "^6.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -5930,7 +5998,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=20.19.0" @@ -6936,6 +7003,28 @@ "dev": true, "license": "MIT" }, + "node_modules/htmlparser2": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-12.0.0.tgz", + "integrity": "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^3.0.0", + "domhandler": "^6.0.0", + "domutils": "^4.0.2", + "entities": "^8.0.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", diff --git a/package.json b/package.json index 39d792976..9fd75183c 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "express": "5.3.0", "express-rate-limit": "8.7.3", "gray-matter": "4.0.3", + "htmlparser2": "12.0.0", "luxon": "3.7.2", "pdfjs-dist": "6.5.438", "picomatch": "4.0.7", diff --git a/src/vault-mcp/obsidian-markdown/__tests__/html.test.ts b/src/vault-mcp/obsidian-markdown/__tests__/html.test.ts new file mode 100644 index 000000000..e068de862 --- /dev/null +++ b/src/vault-mcp/obsidian-markdown/__tests__/html.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect } from "vitest" +import { DateTime } from "luxon" +import { extractHtmlVisibleText } from "../html.js" + +const LARGE_CAP = 1_000_000 + +const extractAll = (html: string): string => extractHtmlVisibleText({ html, maxBytes: LARGE_CAP }) + +describe("extractHtmlVisibleText", () => { + it("returns the visible text of a page, one line per block element", () => { + const page = + "Restart runbook" + + "

Restart

Stop the service first.

" + + "
  • Check logs
  • Start it
" + + expect(extractAll(page)).toBe( + "Restart runbook\nRestart\nStop the service first.\nCheck logs\nStart it", + ) + }) + + it("drops the text of script, style, noscript and template elements", () => { + const page = + "

before

" + + "

after

" + + expect(extractAll(page)).toBe("before\nafter") + }) + + it("drops the raw text of iframe, noembed and noframes fallback content", () => { + const page = + "

shown

" + + "embed fallbackframes fallback" + + expect(extractAll(page)).toBe("shown") + }) + + it("drops an uppercase SCRIPT element's text", () => { + expect(extractAll("

shown

")).toBe("shown") + }) + + it("ignores a stray closing tag for a hidden element that never opened", () => { + expect(extractAll("

VISIBLE

")).toBe("VISIBLE") + }) + + it("keeps a hidden element's text hidden through a closing tag with another name", () => { + expect(extractAll("

VISIBLE

")).toBe("VISIBLE") + }) + + it("keeps text hidden until the outer element of nested hidden elements closes", () => { + expect(extractAll("outer")).toBe("outer") + }) + + it("treats a self-closing script tag as open, as HTML does", () => { + expect(extractAll('

shown

after

')).toBe( + "shown\nafter", + ) + }) + + it("hides everything after a template that never closes", () => { + expect(extractAll("

shown