From feeb9f376678844529c3b2f98d18f4ea5e916185 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 01:16:29 +0000 Subject: [PATCH 01/19] feat(assets): PDF text extraction via unpdf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vault_read_asset now extracts text from PDFs using unpdf's content-stream parser instead of returning a "not yet supported" error. Scanned or image-only PDFs (no extractable text) get a descriptive error with page count. Extracted text is subject to the same MAX_TEXT_OUTPUT_BYTES cap as other text assets. - Add unpdf dependency for serverless-first PDF.js text extraction - Replace PDF error block in asset-operations.ts with extractText dispatch - Convert Buffer → Uint8Array for unpdf compatibility - Update vault_read_asset tool description (PDF section, errors, examples) - Update vault_list_assets description (remove stale ".pdf unsupported" note) - Fix "MCP attachment reads" → "MCP asset reads" comment in config.ts - Add asset-operations unit tests (PDF happy path, scanned PDF, whitespace- only, text cap, unsupported type includes .pdf) - Add integration test with minimal valid PDF fixture - Update existing tool-definitions tests for new error message format Co-Authored-By: Claude Opus 4.6 Claude-Session: https://claude.ai/code/session_01PajwhEzt5cjqQjBPVBuXHn --- package-lock.json | 93 ++---------- package.json | 1 + src/vault-mcp/config.ts | 2 +- .../mcp-core/__tests__/pdf-fixture.ts | 61 ++++++++ .../__tests__/tool-definitions.test.ts | 14 +- src/vault-mcp/mcp-core/tools/asset-tools.ts | 10 +- .../__tests__/asset-operations.test.ts | 134 ++++++++++++++++++ .../vault-operations/asset-operations.ts | 26 +++- 8 files changed, 245 insertions(+), 96 deletions(-) create mode 100644 src/vault-mcp/mcp-core/__tests__/pdf-fixture.ts create mode 100644 src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts diff --git a/package-lock.json b/package-lock.json index 1639abcc..1ee42c44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "picomatch": "4.0.5", "sharp": "0.35.3", "sqlite-vec": "0.1.9", + "unpdf": "1.6.2", "yaml": "2.9.0", "zod": "4.4.3" }, @@ -1639,9 +1640,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1658,9 +1656,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1677,9 +1672,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1696,9 +1688,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1715,9 +1704,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1734,9 +1720,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1753,9 +1736,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1772,9 +1752,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1791,9 +1768,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1816,9 +1790,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1841,9 +1812,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1866,9 +1834,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1891,9 +1856,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1916,9 +1878,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1941,9 +1900,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1966,9 +1922,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2287,9 +2240,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2307,9 +2257,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2327,9 +2274,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2347,9 +2291,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2367,9 +2308,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2387,9 +2325,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -5738,9 +5673,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5762,9 +5694,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5786,9 +5715,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5810,9 +5736,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7797,6 +7720,20 @@ "dev": true, "license": "MIT" }, + "node_modules/unpdf": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/unpdf/-/unpdf-1.6.2.tgz", + "integrity": "sha512-zQ80ySoPuPHOsvIoRp/nJyQt8TOUoTh1+WBCGcBvlddQNgKDLRwm0AY3x8Q35I7+kIiRSgqMx+Ma2pl9McIp7A==", + "license": "MIT", + "peerDependencies": { + "@napi-rs/canvas": "^0.1.69" + }, + "peerDependenciesMeta": { + "@napi-rs/canvas": { + "optional": true + } + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index c7dc716d..79938a44 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "picomatch": "4.0.5", "sharp": "0.35.3", "sqlite-vec": "0.1.9", + "unpdf": "1.6.2", "yaml": "2.9.0", "zod": "4.4.3" }, diff --git a/src/vault-mcp/config.ts b/src/vault-mcp/config.ts index 1e97412a..e0cab0fa 100644 --- a/src/vault-mcp/config.ts +++ b/src/vault-mcp/config.ts @@ -128,7 +128,7 @@ export const loadConfig = ( return value } - // 50 MiB — matches the most permissive prior art for MCP attachment reads. + // 50 MiB — matches the most permissive prior art for MCP asset reads. const maxAssetBytes = requireNonZeroBytes( "MAX_ASSET_BYTES", envVar.from(env).get("MAX_ASSET_BYTES").default("52428800").asIntPositive(), diff --git a/src/vault-mcp/mcp-core/__tests__/pdf-fixture.ts b/src/vault-mcp/mcp-core/__tests__/pdf-fixture.ts new file mode 100644 index 00000000..3005fa66 --- /dev/null +++ b/src/vault-mcp/mcp-core/__tests__/pdf-fixture.ts @@ -0,0 +1,61 @@ +/** Builds a minimal valid PDF 1.4 buffer containing "Hello PDF" — just enough + * structure for unpdf's text extraction to return readable content. */ +export const buildMinimalPdf = (): Buffer => { + const textContent = "Hello PDF" + const stream = `BT /F1 12 Tf 100 700 Td (${textContent}) Tj ET` + const streamBytes = Buffer.byteLength(stream, "ascii") + + const lines = [ + "%PDF-1.4", + "", + "1 0 obj", + "<< /Type /Catalog /Pages 2 0 R >>", + "endobj", + "", + "2 0 obj", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "endobj", + "", + "3 0 obj", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792]", + " /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", + "endobj", + "", + "4 0 obj", + `<< /Length ${streamBytes} >>`, + "stream", + stream, + "endstream", + "endobj", + "", + "5 0 obj", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "endobj", + "", + ] + + const body = lines.join("\n") + const xrefOffset = Buffer.byteLength(body, "ascii") + + const xrefEntries = [ + "0000000000 65535 f ", + `${String(body.indexOf("1 0 obj")).padStart(10, "0")} 00000 n `, + `${String(body.indexOf("2 0 obj")).padStart(10, "0")} 00000 n `, + `${String(body.indexOf("3 0 obj")).padStart(10, "0")} 00000 n `, + `${String(body.indexOf("4 0 obj")).padStart(10, "0")} 00000 n `, + `${String(body.indexOf("5 0 obj")).padStart(10, "0")} 00000 n `, + ] + + const trailer = [ + "xref", + `0 ${xrefEntries.length}`, + ...xrefEntries, + "trailer", + `<< /Size ${xrefEntries.length} /Root 1 0 R >>`, + "startxref", + String(xrefOffset), + "%%EOF", + ].join("\n") + + return Buffer.from(body + trailer, "ascii") +} diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index 6aac6409..cb5aa2ad 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -1109,14 +1109,14 @@ describe("asset tool handlers", () => { expect(result.content).toEqual([{ type: "text", text: '{"key": "value"}' }]) }) - it("rejects a .pdf with a not-yet-supported error carrying the file size", async () => { + it("returns extracted text from a valid PDF", async () => { const { vault, readAsset } = await setupAssetHarness() - await writeFile(join(vault, "doc.pdf"), "0123456789", "utf8") + const { buildMinimalPdf } = await import("./pdf-fixture.js") + await writeFile(join(vault, "doc.pdf"), buildMinimalPdf()) const result = await readAsset({ path: "doc.pdf" }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: PDF reading is not yet supported: "doc.pdf" exists (10 bytes) but text extraction is not available yet', - ) + expect(result.isError).toBeUndefined() + expect(result.content[0]?.type).toBe("text") + expect(result.content[0]?.text).toContain("Hello PDF") }) it("rejects an unsupported extension naming the readable types", async () => { @@ -1125,7 +1125,7 @@ describe("asset tool handlers", () => { const result = await readAsset({ path: "song.mp3" }) expect(result.isError).toBe(true) expect(result.content[0]?.text).toBe( - '[Error]: unsupported asset type ".mp3": "song.mp3" exists (4 bytes). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, and text formats (.svg/.json/.txt/.csv/.xml/.log/.base)', + '[Error]: unsupported asset type ".mp3": "song.mp3" exists (4 bytes). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats (.svg/.json/.txt/.csv/.xml/.log/.base)', ) }) diff --git a/src/vault-mcp/mcp-core/tools/asset-tools.ts b/src/vault-mcp/mcp-core/tools/asset-tools.ts index 5ec4bb1d..9de392a7 100644 --- a/src/vault-mcp/mcp-core/tools/asset-tools.ts +++ b/src/vault-mcp/mcp-core/tools/asset-tools.ts @@ -43,21 +43,23 @@ Example: vault_read_asset({ path: "attachments/diagram.png" }) — the image its Example: vault_read_asset({ path: "Boards/Roadmap.canvas" }) — a readable outline of the canvas Example: vault_read_asset({ path: "Boards/Roadmap.canvas", raw: true }) — the canvas's exact JSON source Example: vault_read_asset({ path: "exports/data.json" }) — the file content as text +Example: vault_read_asset({ path: "papers/research.pdf" }) — the extracted text content What each type returns: - Images (.png/.jpg/.jpeg/.gif/.webp): the image as a viewable image block — downscaled and recompressed server-side when it exceeds client response limits, delivered untouched otherwise — plus a text line stating the path, delivered format/dimensions/bytes, and the original dimensions when shrunk. Animated GIFs are reduced to their first frame when recompressed to fit the budget. - Canvas (.canvas): a readable markdown outline per JSON Canvas 1.0 — groups (by visual containment), node content in reading order, and a connections list with edge labels. Set raw: true for the exact JSON source instead (geometry, ids, colors — full fidelity). +- PDFs (.pdf): the extracted text content. Text is extracted from the PDF's content streams — layout order follows the stream, not visual position, so columns or complex layouts may interleave. Scanned or image-only PDFs with no embedded text return an error stating the page count. - Text formats (.svg/.json/.txt/.csv/.xml/.log/.base): the file content verbatim as text. .svg is returned as its XML source; .base as its YAML source. -- PDFs (.pdf): not yet readable — returns an error that confirms the file exists and its size; text extraction is planned. -When to use: whenever a note references an asset you need to actually see or read — an embedded diagram, a linked canvas or data file. Find the assets a note links to (with byte sizes) via vault_get_outgoing_links; browse a folder's assets via vault_list_assets. For .md notes use vault_read_note — this tool rejects them. +When to use: whenever a note references an asset you need to actually see or read — an embedded diagram, a linked canvas, data file, or PDF. Find the assets a note links to (with byte sizes) via vault_get_outgoing_links; browse a folder's assets via vault_list_assets. For .md notes use vault_read_note — this tool rejects them. Errors: - "not an asset" — the path ends in .md; read notes with vault_read_note - "asset not found" — nothing exists at that path; discover valid paths via vault_list_assets - "asset too large" — the file exceeds the server's read cap (MAX_ASSET_BYTES, default 50 MiB) -- "text output too large" — a text asset renders past the output cap; only smaller files can be returned whole +- "text output too large" — a text asset or PDF renders past the output cap; only smaller files can be returned whole - "not valid UTF-8" — the file's bytes aren't UTF-8 text; returning them would silently corrupt the content +- "PDF has no extractable text" — the PDF exists but contains no text content (scanned or image-only); states the page count - "image cannot be fitted" — the image could not be compressed under the output budget (MAX_IMAGE_OUTPUT_BYTES) - "raw source is not available for images" — raw applies to text-representable files; an image's delivered form is its image block - unsupported types (audio, archives, …) return an error naming the readable types plus the file's existence and size @@ -158,7 +160,7 @@ Errors: - A folder containing no assets — or a folder that doesn't exist — returns an empty listing, not an error. - A folder path escaping the vault (e.g. "../elsewhere") is rejected with a path-traversal error. -Returns: JSON with assets (array of { path, extension, bytes }, sorted by path), extension_counts (per-extension totals over the full filtered set), total (full filtered count), and truncated (true when total exceeds limit). bytes is the on-disk file size, not the delivery cost: reading an image via vault_read_asset returns a copy shrunk to fit when needed, so a large listed image is still cheap to read. Text formats return verbatim, so their listed size is what a read delivers. Assets of supported types are readable via vault_read_asset (unsupported types like .pdf return a descriptive error there); vault_search covers markdown notes.`, +Returns: JSON with assets (array of { path, extension, bytes }, sorted by path), extension_counts (per-extension totals over the full filtered set), total (full filtered count), and truncated (true when total exceeds limit). bytes is the on-disk file size, not the delivery cost: reading an image via vault_read_asset returns a copy shrunk to fit when needed, so a large listed image is still cheap to read. Text formats return verbatim, so their listed size is what a read delivers. Assets of supported types are readable via vault_read_asset; vault_search covers markdown notes.`, inputSchema: { folder: z .string() diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts new file mode 100644 index 00000000..f6cc8fbb --- /dev/null +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, vi } from "vitest" +import { assetOperations } from "../asset-operations.js" +import { logger } from "../../../logger.js" + +vi.mock("../vault-filesystem.js", () => ({ + vaultFs: { + readAsset: vi.fn(), + }, +})) + +vi.mock("unpdf", () => ({ + extractText: vi.fn(), +})) + +vi.mock("../../../utils/fit-image-to-byte-budget.js", () => ({ + fitImageToByteBudget: vi.fn(), +})) + +import { vaultFs } from "../vault-filesystem.js" +import { extractText } from "unpdf" + +const mockedReadAsset = vi.mocked(vaultFs.readAsset) +const mockedExtractText = vi.mocked(extractText) + +const defaultParams = { + vaultPath: "/vault", + maxAssetBytes: 52_428_800, + maxImageOutputBytes: 49_152, +} + +describe("readAssetContent — PDF extraction", () => { + it("extracts text from a PDF and returns it as a text result", async () => { + const pdfBuffer = Buffer.from("fake-pdf-bytes") + mockedReadAsset.mockResolvedValue({ + buffer: pdfBuffer, + bytes: 12_345, + extension: ".pdf", + }) + mockedExtractText.mockResolvedValue({ + totalPages: 3, + text: "Page one content.\nPage two content.\nPage three content.", + }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "papers/research.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: "Page one content.\nPage two content.\nPage three content.", + }) + expect(mockedExtractText).toHaveBeenCalledWith( + expect.any(Uint8Array), + { mergePages: true }, + ) + }) + + it("throws a descriptive error for scanned PDFs with no extractable text", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-scanned-pdf"), + bytes: 5_000_000, + extension: ".pdf", + }) + mockedExtractText.mockResolvedValue({ + totalPages: 12, + text: "", + }) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "scans/receipt.pdf" }, + logger, + ), + ).rejects.toThrow( + 'PDF has no extractable text: "scans/receipt.pdf" exists ' + + "(5000000 bytes, 12 pages)", + ) + }) + + it("throws for PDFs with only whitespace content", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 1_000, + extension: ".pdf", + }) + mockedExtractText.mockResolvedValue({ + totalPages: 1, + text: " \n\t \n ", + }) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "empty.pdf" }, + logger, + ), + ).rejects.toThrow("PDF has no extractable text") + }) + + it("rejects PDF text exceeding the output cap", async () => { + const largeText = "x".repeat(200_000) + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 500_000, + extension: ".pdf", + }) + mockedExtractText.mockResolvedValue({ + totalPages: 50, + text: largeText, + }) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "huge.pdf" }, + logger, + ), + ).rejects.toThrow("text output too large") + }) + + it("includes .pdf in the unsupported-type error's readable types list", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-audio"), + bytes: 10_000, + extension: ".mp3", + }) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "audio/song.mp3" }, + logger, + ), + ).rejects.toThrow(".pdf") + }) +}) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 044b8b97..12bb3d77 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -1,3 +1,4 @@ +import { extractText } from "unpdf" import { vaultFs } from "./vault-filesystem.js" import { linearizeCanvas } from "../obsidian-markdown/canvas.js" import { links } from "../obsidian-markdown/links.js" @@ -84,8 +85,8 @@ const decodeUtf8Strict = (params: { buffer: Buffer; path: string }): string => { /** * Reads a non-markdown vault file and returns its most useful representation * per type; `raw` skips the canvas rendition for the exact JSON source. - * Throws structured errors for images with `raw`, PDFs, and unsupported - * types — each stating the file's existence and size. + * Throws structured errors for images with `raw` and unsupported types — + * each stating the file's existence and size. */ const readAssetContent = async ( params: { @@ -128,15 +129,28 @@ const readAssetContent = async ( return { kind: "text", text } } if (asset.extension === ".pdf") { - throw new Error( - `PDF reading is not yet supported: "${path}" exists ` + - `(${asset.bytes} bytes) but text extraction is not available yet`, + const pdfData = new Uint8Array( + asset.buffer.buffer, + asset.buffer.byteOffset, + asset.buffer.byteLength, ) + const extracted = await extractText(pdfData, { mergePages: true }) + const text = typeof extracted.text === "string" ? extracted.text : "" + if (!text.trim()) { + throw new Error( + `PDF has no extractable text: "${path}" exists ` + + `(${asset.bytes} bytes, ${extracted.totalPages} pages) but ` + + `contains no text content — it may be a scanned document or ` + + `image-only PDF`, + ) + } + assertTextWithinCap({ text, path }) + return { kind: "text", text } } throw new Error( `unsupported asset type "${asset.extension}": "${path}" exists ` + `(${asset.bytes} bytes). Readable types: images ` + - `(.png/.jpg/.jpeg/.gif/.webp), .canvas, and text formats ` + + `(.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats ` + `(.svg/.json/.txt/.csv/.xml/.log/.base)`, ) } From df3b95e3bd86a9ba1a105b6774a04f0ed94acacb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 17:25:39 +0000 Subject: [PATCH 02/19] style: fix prettier formatting in asset-operations tests Co-Authored-By: Claude Opus 4.6 Claude-Session: https://claude.ai/code/session_01PajwhEzt5cjqQjBPVBuXHn --- .../vault-operations/__tests__/asset-operations.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts index f6cc8fbb..cd141d75 100644 --- a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -50,10 +50,9 @@ describe("readAssetContent — PDF extraction", () => { kind: "text", text: "Page one content.\nPage two content.\nPage three content.", }) - expect(mockedExtractText).toHaveBeenCalledWith( - expect.any(Uint8Array), - { mergePages: true }, - ) + expect(mockedExtractText).toHaveBeenCalledWith(expect.any(Uint8Array), { + mergePages: true, + }) }) it("throws a descriptive error for scanned PDFs with no extractable text", async () => { From 94167e71335cbbb3b23fa205af1fadd02bc3a3ee Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:58:54 -0400 Subject: [PATCH 03/19] fix(review): update feature surface docs for PDF text extraction ARCHITECTURE.md still described PDFs as "not-yet-supported" after the feature landed. README.md and DOCKERHUB.md listed supported asset types without mentioning PDFs. Updated all three to reflect the new capability. Co-Authored-By: Claude Opus 4.6 (1M context) --- ARCHITECTURE.md | 3 ++- DOCKERHUB.md | 1 + README.md | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 90cac7ff..1840329f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -296,7 +296,8 @@ Link queries use a `links` table populated during indexing: 1. **Images** (`.png`/`.jpg`/`.jpeg`/`.gif`/`.webp`) return an MCP `image` content block plus a one-line metadata text block. A shared fit-to-byte-budget pipeline (`utils/fit-image-to-byte-budget.ts`, built on sharp) makes oversized images deliverable: EXIF auto-orient → resize long edge to ≤1568px → walk a fixed quality ladder (JPEG via mozjpeg for opaque images, WebP for alpha — PNG has no quality knob) → shrink dimensions by √(budget/actual) if the ladder floor still exceeds the budget. Deterministic and terminating (bounded attempts, 64px floor); sharp's default `limitInputPixels` stays active as the decompression-bomb guard. The budget (`MAX_IMAGE_OUTPUT_BYTES`, default 48 KiB binary) is sized for the tightest mainstream client cap. 2. **Canvas** (`.canvas`) linearizes to markdown via the pure `obsidian-markdown/canvas.ts` parser ([JSON Canvas 1.0](https://jsoncanvas.org)): group membership by spatial rect containment (innermost group wins; equal rects tiebreak deterministically by id), nodes in reading order (y, then x), and an edge list with node ids resolved to display names. Lenient parsing — unknown properties ignored, malformed entries skipped. `raw: true` skips the linearizer and returns the JSON source verbatim for full structural fidelity. 3. **Text formats** (`.svg`/`.json`/`.txt`/`.csv`/`.xml`/`.log`/`.base`) pass through verbatim as text, capped at a fixed 100 KiB output size (explicit error over silent truncation). -4. **PDFs** return a structured not-yet-supported error carrying the file's existence and size (text extraction is a planned follow-up); unknown types return an error naming the readable set. +4. **PDFs** (`.pdf`) extract text content from the document's content streams (`unpdf`, based on Mozilla's PDF.js). Layout order follows the stream, not visual position, so columns or complex layouts may interleave. Scanned or image-only PDFs with no extractable text return an error stating the file's existence, size, and page count. +5. **Unknown types** return an error naming the readable set. The extension-to-representation routing above is implemented by the `vault-operations/asset-operations.ts` use-case. Beneath it, every read goes through `vaultFs.readAsset`, which applies the same `resolveSafePath` traversal guard as notes, rejects `.md` paths (notes belong to `vault_read_note`), and enforces a stat-before-read size cap (`MAX_ASSET_BYTES`, default 50 MiB). diff --git a/DOCKERHUB.md b/DOCKERHUB.md index e10730b1..ff0d6853 100644 --- a/DOCKERHUB.md +++ b/DOCKERHUB.md @@ -65,6 +65,7 @@ Your notes embed screenshots, reference architecture diagrams, and link out to c - **Images** — the image itself, not the filename. Screenshots and diagrams are downscaled and recompressed server-side when they exceed what MCP clients accept, so even a phone session can look at a 5MB architecture diagram - **Canvases** — a [Canvas](https://help.obsidian.md/canvas) board arrives as a readable outline: its groups, each card's content in reading order, and the connections between them. The exact JSON source is one flag away when full fidelity matters +- **PDFs** — text is extracted from the document's content streams; scanned or image-only PDFs return an error stating the page count - **Text and data files** — SVG, JSON, CSV, logs, and [Bases](https://help.obsidian.md/bases) files return exactly as written - **Browse** — list any folder's assets with per-extension counts and file sizes; assets a note links to report their size in the link graph too diff --git a/README.md b/README.md index 75d96fd3..7ff7560a 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,7 @@ Your notes embed screenshots, reference architecture diagrams, and link out to c - **Images** — the image itself, not the filename. Screenshots and diagrams are downscaled and recompressed server-side when they exceed what MCP clients accept, so even a phone session can look at a 5MB architecture diagram - **Canvases** — a [Canvas](https://help.obsidian.md/canvas) board arrives as a readable outline: its groups, each card's content in reading order, and the connections between them. The exact JSON source is one flag away when full fidelity matters +- **PDFs** — text is extracted from the document's content streams; scanned or image-only PDFs return an error stating the page count - **Text and data files** — SVG, JSON, CSV, logs, and [Bases](https://help.obsidian.md/bases) files return exactly as written - **Browse** — list any folder's assets with per-extension counts and file sizes; assets a note links to report their size in the link graph too From 6b65f943866e24791258a7e1d592fc41c1b0d3f5 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:04:36 -0400 Subject: [PATCH 04/19] style: remove redundant typeof guard and add Buffer conversion comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unpdf overload for mergePages:true already narrows text to string — the typeof check was dead code and the "" fallback masked the invariant. Added a why-comment on the non-obvious Buffer→Uint8Array pool conversion. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 12bb3d77..1c9120a7 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -129,13 +129,15 @@ const readAssetContent = async ( return { kind: "text", text } } if (asset.extension === ".pdf") { + // Buffer → Uint8Array view: Buffer.buffer may be Node's shared pool, + // so byteOffset/byteLength carve out this buffer's portion. const pdfData = new Uint8Array( asset.buffer.buffer, asset.buffer.byteOffset, asset.buffer.byteLength, ) const extracted = await extractText(pdfData, { mergePages: true }) - const text = typeof extracted.text === "string" ? extracted.text : "" + const text = extracted.text if (!text.trim()) { throw new Error( `PDF has no extractable text: "${path}" exists ` + From 0cb5ba6d91024de55e1388704611943c72bd22e1 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:11:35 -0400 Subject: [PATCH 05/19] test: exact error assertions + corrupt PDF coverage gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tighten 4 substring error assertions to exact message matches (deterministic errors should not use substring matchers — catches message drift). Add content-length assertion to PDF integration test. Write coverage gap test for extractText rejection propagation (corrupt/unparseable PDFs). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/tool-definitions.test.ts | 1 + .../__tests__/asset-operations.test.ts | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index cb5aa2ad..7e9c932e 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -1115,6 +1115,7 @@ describe("asset tool handlers", () => { await writeFile(join(vault, "doc.pdf"), buildMinimalPdf()) const result = await readAsset({ path: "doc.pdf" }) expect(result.isError).toBeUndefined() + expect(result.content).toHaveLength(1) expect(result.content[0]?.type).toBe("text") expect(result.content[0]?.text).toContain("Hello PDF") }) diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts index cd141d75..90a58944 100644 --- a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -73,7 +73,8 @@ describe("readAssetContent — PDF extraction", () => { ), ).rejects.toThrow( 'PDF has no extractable text: "scans/receipt.pdf" exists ' + - "(5000000 bytes, 12 pages)", + "(5000000 bytes, 12 pages) but contains no text content " + + "— it may be a scanned document or image-only PDF", ) }) @@ -93,7 +94,11 @@ describe("readAssetContent — PDF extraction", () => { { ...defaultParams, path: "empty.pdf" }, logger, ), - ).rejects.toThrow("PDF has no extractable text") + ).rejects.toThrow( + 'PDF has no extractable text: "empty.pdf" exists ' + + "(1000 bytes, 1 pages) but contains no text content " + + "— it may be a scanned document or image-only PDF", + ) }) it("rejects PDF text exceeding the output cap", async () => { @@ -113,7 +118,10 @@ describe("readAssetContent — PDF extraction", () => { { ...defaultParams, path: "huge.pdf" }, logger, ), - ).rejects.toThrow("text output too large") + ).rejects.toThrow( + 'text output too large: "huge.pdf" renders to 200000 bytes ' + + "(cap 102400 bytes)", + ) }) it("includes .pdf in the unsupported-type error's readable types list", async () => { @@ -128,6 +136,27 @@ describe("readAssetContent — PDF extraction", () => { { ...defaultParams, path: "audio/song.mp3" }, logger, ), - ).rejects.toThrow(".pdf") + ).rejects.toThrow( + 'unsupported asset type ".mp3": "audio/song.mp3" exists ' + + "(10000 bytes). Readable types: images " + + "(.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats " + + "(.svg/.json/.txt/.csv/.xml/.log/.base)", + ) + }) + + it("propagates extractText errors for corrupt PDFs", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("not-a-real-pdf"), + bytes: 14, + extension: ".pdf", + }) + mockedExtractText.mockRejectedValue(new Error("Invalid PDF structure")) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "corrupt.pdf" }, + logger, + ), + ).rejects.toThrow("Invalid PDF structure") }) }) From 10bbce0b4ca2225aad438136ec7b7722efa4d454 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:21:03 -0400 Subject: [PATCH 06/19] fix: add MAX_ASSET_BYTES and MAX_IMAGE_OUTPUT_BYTES to config tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both env vars were documented in every deploy surface (docker-compose, .env.example, cli/src/env.ts) and referenced in the vault_read_asset error descriptions, but missing from the README and DOCKERHUB config tables — a pre-existing gap from PR #351. Co-Authored-By: Claude Opus 4.6 (1M context) --- DOCKERHUB.md | 2 ++ README.md | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/DOCKERHUB.md b/DOCKERHUB.md index ff0d6853..8979836e 100644 --- a/DOCKERHUB.md +++ b/DOCKERHUB.md @@ -153,6 +153,8 @@ All settings are environment variables with sensible defaults. | `LOG_DIR` | — | `/data/logs` (remote), unset (local) | Directory for persistent log files. When set, logs are written to date-stamped files there alongside stdout. Unset means stdout only. | | `LOG_RETENTION_DAYS` | — | `30` | Days to keep log files before automatic cleanup on startup | | `WINDOWS_MODE` | — | `false` | On Windows? Set `true`. Switches the file watcher to polling and note moves to rename-based writes so a vault on a `C:` drive works through Docker Desktop. Safe to leave on for any Windows setup; unneeded on macOS/Linux/WSL2. | +| `MAX_ASSET_BYTES` | — | `52428800` (50 MiB) | Maximum file size `vault_read_asset` will read (in bytes). Files exceeding this are rejected before reading. Raise for vaults with very large individual files. | +| `MAX_IMAGE_OUTPUT_BYTES` | — | `49152` (48 KiB) | Byte budget for images delivered by `vault_read_asset`, in binary bytes before base64 encoding. Images exceeding this are downscaled and recompressed to fit. Sized for the tightest mainstream MCP client cap; raise for clients that accept larger responses. | ## Deployment Options diff --git a/README.md b/README.md index 7ff7560a..2c13eb58 100644 --- a/README.md +++ b/README.md @@ -296,23 +296,25 @@ These are conventions, not requirements — Vault Cortex works with any property All settings are environment variables with sensible defaults. -| Variable | Required? | Default | Description | -| --------------------------- | ----------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MCP_AUTH_TOKEN` | Yes | — | Bearer token for authentication (also the JWT signing key) | -| `VAULT_PATH` | Local only | — | Host path to your vault (bind mount source; remote uses a named volume) | -| `PUBLIC_URL` | Remote only | — | Public URL for OAuth discovery metadata | -| `EMBEDDING_ENABLED` | — | `true` | Set `false` to disable the embedding pipeline — skips model download, vector tables, embedding passes, and hybrid search. Search falls back to FTS5 keyword matching. | -| `RERANK_MODE` | — | `blended` | Cross-encoder reranking mode: `blended` applies position-aware score blending after RRF fusion (~200ms added latency), `none` skips reranking. Only takes effect when `EMBEDDING_ENABLED` is true. | -| `MEMORY_ENABLED` | — | `true` | Set `false` to fully disable the memory layer — hides memory tools, skips bootstrap, omits memory from server metadata. `MEMORY_DIR` is ignored when `false`. | -| `MEMORY_DIR` | — | `About Me` | Vault folder for structured memory files | -| `PROTECTED_PATHS` | — | `MEMORY_DIR, Daily Notes` | Folders that `vault_delete_note` refuses to touch | -| `ORPHAN_EXCLUDE_FOLDERS` | — | `Daily Notes, Templates, MEMORY_DIR` | Folders excluded from orphan detection | -| `TZ` | — | `UTC` | IANA timezone for timestamps and daily note resolution | -| `SERVICE_DOCUMENTATION_URL` | — | GitHub repo URL | URL returned in OAuth discovery metadata | -| `LOG_LEVEL` | — | `info` | Logging verbosity: `debug`, `info`, `warn`, `error` | -| `LOG_DIR` | — | `/data/logs` (remote), unset (local) | Directory for persistent log files. When set, logs are written to date-stamped files there alongside stdout. Unset means stdout only. | -| `LOG_RETENTION_DAYS` | — | `30` | Days to keep log files before automatic cleanup on startup | -| `WINDOWS_MODE` | — | `false` | On Windows? Set `true`. Switches the file watcher to polling and note moves to rename-based writes so a vault on a `C:` drive works through Docker Desktop. Safe to leave on for any Windows setup; unneeded on macOS/Linux/WSL2. | +| Variable | Required? | Default | Description | +| --------------------------- | ----------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MCP_AUTH_TOKEN` | Yes | — | Bearer token for authentication (also the JWT signing key) | +| `VAULT_PATH` | Local only | — | Host path to your vault (bind mount source; remote uses a named volume) | +| `PUBLIC_URL` | Remote only | — | Public URL for OAuth discovery metadata | +| `EMBEDDING_ENABLED` | — | `true` | Set `false` to disable the embedding pipeline — skips model download, vector tables, embedding passes, and hybrid search. Search falls back to FTS5 keyword matching. | +| `RERANK_MODE` | — | `blended` | Cross-encoder reranking mode: `blended` applies position-aware score blending after RRF fusion (~200ms added latency), `none` skips reranking. Only takes effect when `EMBEDDING_ENABLED` is true. | +| `MEMORY_ENABLED` | — | `true` | Set `false` to fully disable the memory layer — hides memory tools, skips bootstrap, omits memory from server metadata. `MEMORY_DIR` is ignored when `false`. | +| `MEMORY_DIR` | — | `About Me` | Vault folder for structured memory files | +| `PROTECTED_PATHS` | — | `MEMORY_DIR, Daily Notes` | Folders that `vault_delete_note` refuses to touch | +| `ORPHAN_EXCLUDE_FOLDERS` | — | `Daily Notes, Templates, MEMORY_DIR` | Folders excluded from orphan detection | +| `TZ` | — | `UTC` | IANA timezone for timestamps and daily note resolution | +| `SERVICE_DOCUMENTATION_URL` | — | GitHub repo URL | URL returned in OAuth discovery metadata | +| `LOG_LEVEL` | — | `info` | Logging verbosity: `debug`, `info`, `warn`, `error` | +| `LOG_DIR` | — | `/data/logs` (remote), unset (local) | Directory for persistent log files. When set, logs are written to date-stamped files there alongside stdout. Unset means stdout only. | +| `LOG_RETENTION_DAYS` | — | `30` | Days to keep log files before automatic cleanup on startup | +| `WINDOWS_MODE` | — | `false` | On Windows? Set `true`. Switches the file watcher to polling and note moves to rename-based writes so a vault on a `C:` drive works through Docker Desktop. Safe to leave on for any Windows setup; unneeded on macOS/Linux/WSL2. | +| `MAX_ASSET_BYTES` | — | `52428800` (50 MiB) | Maximum file size `vault_read_asset` will read (in bytes). Files exceeding this are rejected before reading. Raise for vaults with very large individual files. | +| `MAX_IMAGE_OUTPUT_BYTES` | — | `49152` (48 KiB) | Byte budget for images delivered by `vault_read_asset`, in binary bytes before base64 encoding. Images exceeding this are downscaled and recompressed to fit. Sized for the tightest mainstream MCP client cap; raise for clients that accept larger responses. | **Smart defaults:** Setting `MEMORY_DIR` automatically updates the defaults for `PROTECTED_PATHS` and `ORPHAN_EXCLUDE_FOLDERS`. You only set those explicitly for a fully custom list. When `MEMORY_ENABLED` is `false`, the memory layer is fully disabled — memory tools are hidden and the memory folder is not auto-created. From ad128b045ed7c8d710237cf727b94b9640407594 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:50:49 -0400 Subject: [PATCH 07/19] test: collapse decomposed PDF assertions into single toEqual Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index 7e9c932e..e8f0ce2c 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -1114,10 +1114,9 @@ describe("asset tool handlers", () => { const { buildMinimalPdf } = await import("./pdf-fixture.js") await writeFile(join(vault, "doc.pdf"), buildMinimalPdf()) const result = await readAsset({ path: "doc.pdf" }) - expect(result.isError).toBeUndefined() - expect(result.content).toHaveLength(1) - expect(result.content[0]?.type).toBe("text") - expect(result.content[0]?.text).toContain("Hello PDF") + expect(result).toEqual({ + content: [{ type: "text", text: expect.stringContaining("Hello PDF") }], + }) }) it("rejects an unsupported extension naming the readable types", async () => { From 79d0674effc90c34a1b8e3836796f02845dae67a Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:49:40 -0400 Subject: [PATCH 08/19] feat(assets): structured PDF extraction via extractTextItems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace flat extractText with extractTextItems + getMeta + extractLinks for richer PDF output at near-identical token cost (~7% overhead): - Document header: title, page count, links count - Heading hierarchy: relative font-size detection (top N-1 distinct sizes map to H1-H3, smallest is always body text) - Code fences: monospace fontFamily → fenced code blocks - Page separators: --- Page N --- - Links footer: deduplicated hyperlinks from the document - Proxy lifecycle: getDocumentProxy with try/finally cleanup The structured items are an intermediate processing step — the raw coordinates are never exposed; the delivery format is reconstructed markdown that a model reads naturally. Co-Authored-By: Claude Opus 4.6 (1M context) --- ARCHITECTURE.md | 2 +- .../__tests__/tool-definitions.test.ts | 12 +- src/vault-mcp/mcp-core/tools/asset-tools.ts | 2 +- .../__tests__/asset-operations.test.ts | 344 ++++++++++++++++-- .../vault-operations/asset-operations.ts | 167 ++++++++- 5 files changed, 486 insertions(+), 41 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1840329f..3924a8fb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -296,7 +296,7 @@ Link queries use a `links` table populated during indexing: 1. **Images** (`.png`/`.jpg`/`.jpeg`/`.gif`/`.webp`) return an MCP `image` content block plus a one-line metadata text block. A shared fit-to-byte-budget pipeline (`utils/fit-image-to-byte-budget.ts`, built on sharp) makes oversized images deliverable: EXIF auto-orient → resize long edge to ≤1568px → walk a fixed quality ladder (JPEG via mozjpeg for opaque images, WebP for alpha — PNG has no quality knob) → shrink dimensions by √(budget/actual) if the ladder floor still exceeds the budget. Deterministic and terminating (bounded attempts, 64px floor); sharp's default `limitInputPixels` stays active as the decompression-bomb guard. The budget (`MAX_IMAGE_OUTPUT_BYTES`, default 48 KiB binary) is sized for the tightest mainstream client cap. 2. **Canvas** (`.canvas`) linearizes to markdown via the pure `obsidian-markdown/canvas.ts` parser ([JSON Canvas 1.0](https://jsoncanvas.org)): group membership by spatial rect containment (innermost group wins; equal rects tiebreak deterministically by id), nodes in reading order (y, then x), and an edge list with node ids resolved to display names. Lenient parsing — unknown properties ignored, malformed entries skipped. `raw: true` skips the linearizer and returns the JSON source verbatim for full structural fidelity. 3. **Text formats** (`.svg`/`.json`/`.txt`/`.csv`/`.xml`/`.log`/`.base`) pass through verbatim as text, capped at a fixed 100 KiB output size (explicit error over silent truncation). -4. **PDFs** (`.pdf`) extract text content from the document's content streams (`unpdf`, based on Mozilla's PDF.js). Layout order follows the stream, not visual position, so columns or complex layouts may interleave. Scanned or image-only PDFs with no extractable text return an error stating the file's existence, size, and page count. +4. **PDFs** (`.pdf`) return structured markdown reconstructed from layout-aware extraction (`unpdf`, based on Mozilla's PDF.js). A document metadata header (title, page count, link count), heading hierarchy inferred from relative font sizes, fenced code blocks detected via monospace fonts, page separators, and a deduplicated links footer. Scanned or image-only PDFs with no extractable text return an error stating the file's existence, size, and page count. 5. **Unknown types** return an error naming the readable set. The extension-to-representation routing above is implemented by the `vault-operations/asset-operations.ts` use-case. Beneath it, every read goes through `vaultFs.readAsset`, which applies the same `resolveSafePath` traversal guard as notes, rejects `.md` paths (notes belong to `vault_read_note`), and enforces a stat-before-read size cap (`MAX_ASSET_BYTES`, default 50 MiB). diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index e8f0ce2c..6f75df75 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -1109,14 +1109,22 @@ describe("asset tool handlers", () => { expect(result.content).toEqual([{ type: "text", text: '{"key": "value"}' }]) }) - it("returns extracted text from a valid PDF", async () => { + it("returns structured markdown from a valid PDF", async () => { const { vault, readAsset } = await setupAssetHarness() const { buildMinimalPdf } = await import("./pdf-fixture.js") await writeFile(join(vault, "doc.pdf"), buildMinimalPdf()) const result = await readAsset({ path: "doc.pdf" }) expect(result).toEqual({ - content: [{ type: "text", text: expect.stringContaining("Hello PDF") }], + content: [ + { + type: "text", + text: expect.stringContaining("Hello PDF"), + }, + ], }) + const text = (result.content[0] as { text: string }).text + expect(text).toContain("Title:") + expect(text).toContain("Pages: 1") }) it("rejects an unsupported extension naming the readable types", async () => { diff --git a/src/vault-mcp/mcp-core/tools/asset-tools.ts b/src/vault-mcp/mcp-core/tools/asset-tools.ts index 9de392a7..0cd6046e 100644 --- a/src/vault-mcp/mcp-core/tools/asset-tools.ts +++ b/src/vault-mcp/mcp-core/tools/asset-tools.ts @@ -48,7 +48,7 @@ Example: vault_read_asset({ path: "papers/research.pdf" }) — the extracted tex What each type returns: - Images (.png/.jpg/.jpeg/.gif/.webp): the image as a viewable image block — downscaled and recompressed server-side when it exceeds client response limits, delivered untouched otherwise — plus a text line stating the path, delivered format/dimensions/bytes, and the original dimensions when shrunk. Animated GIFs are reduced to their first frame when recompressed to fit the budget. - Canvas (.canvas): a readable markdown outline per JSON Canvas 1.0 — groups (by visual containment), node content in reading order, and a connections list with edge labels. Set raw: true for the exact JSON source instead (geometry, ids, colors — full fidelity). -- PDFs (.pdf): the extracted text content. Text is extracted from the PDF's content streams — layout order follows the stream, not visual position, so columns or complex layouts may interleave. Scanned or image-only PDFs with no embedded text return an error stating the page count. +- PDFs (.pdf): structured text with document metadata — title, page count, heading hierarchy (from font sizes), fenced code blocks (from monospace fonts), page separators, and a deduplicated links footer. Richer than flat text extraction: headings, code, and hyperlinks that flat extraction loses are preserved. Scanned or image-only PDFs with no embedded text return an error stating the page count. - Text formats (.svg/.json/.txt/.csv/.xml/.log/.base): the file content verbatim as text. .svg is returned as its XML source; .base as its YAML source. When to use: whenever a note references an asset you need to actually see or read — an embedded diagram, a linked canvas, data file, or PDF. Find the assets a note links to (with byte sizes) via vault_get_outgoing_links; browse a folder's assets via vault_list_assets. For .md notes use vault_read_note — this tool rejects them. diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts index 90a58944..78b83bb2 100644 --- a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -8,8 +8,28 @@ vi.mock("../vault-filesystem.js", () => ({ }, })) +const { + mockCleanup, + mockGetDocumentProxy, + mockGetMeta, + mockExtractTextItems, + mockExtractLinks, +} = vi.hoisted(() => { + const mockCleanup = vi.fn() + return { + mockCleanup, + mockGetDocumentProxy: vi.fn(() => ({ cleanup: mockCleanup })), + mockGetMeta: vi.fn(), + mockExtractTextItems: vi.fn(), + mockExtractLinks: vi.fn(), + } +}) + vi.mock("unpdf", () => ({ - extractText: vi.fn(), + getDocumentProxy: mockGetDocumentProxy, + getMeta: mockGetMeta, + extractTextItems: mockExtractTextItems, + extractLinks: mockExtractLinks, })) vi.mock("../../../utils/fit-image-to-byte-budget.js", () => ({ @@ -17,10 +37,8 @@ vi.mock("../../../utils/fit-image-to-byte-budget.js", () => ({ })) import { vaultFs } from "../vault-filesystem.js" -import { extractText } from "unpdf" const mockedReadAsset = vi.mocked(vaultFs.readAsset) -const mockedExtractText = vi.mocked(extractText) const defaultParams = { vaultPath: "/vault", @@ -28,17 +46,49 @@ const defaultParams = { maxImageOutputBytes: 49_152, } +/** Builds a single-page StructuredTextItem array from lines of text. Items + * are positioned vertically (descending y, like a real PDF) with the given + * fontSize and fontFamily. */ +const buildPageItems = ( + lines: string[], + options?: { fontSize?: number; fontFamily?: string }, +) => + lines.map((str, index) => ({ + str, + x: 42, + y: 780 - index * 15, + width: str.length * 7, + height: options?.fontSize ?? 10.5, + fontSize: options?.fontSize ?? 10.5, + fontFamily: options?.fontFamily ?? "sans-serif", + dir: "ltr" as const, + hasEOL: true, + })) + describe("readAssetContent — PDF extraction", () => { - it("extracts text from a PDF and returns it as a text result", async () => { - const pdfBuffer = Buffer.from("fake-pdf-bytes") + it("returns structured markdown with title, headings, and text", async () => { mockedReadAsset.mockResolvedValue({ - buffer: pdfBuffer, + buffer: Buffer.from("fake-pdf-bytes"), bytes: 12_345, extension: ".pdf", }) - mockedExtractText.mockResolvedValue({ - totalPages: 3, - text: "Page one content.\nPage two content.\nPage three content.", + mockGetMeta.mockResolvedValue({ + info: { Title: "Research Paper" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [ + [ + ...buildPageItems(["Introduction"], { fontSize: 18 }), + ...buildPageItems(["This is the body text of the paper."], { + fontSize: 10.5, + }).map((item) => ({ ...item, y: 750 })), + ], + ], + }) + mockExtractLinks.mockResolvedValue({ + links: ["https://example.com"], + totalPages: 1, }) const result = await assetOperations.readAssetContent( @@ -48,10 +98,167 @@ describe("readAssetContent — PDF extraction", () => { expect(result).toEqual({ kind: "text", - text: "Page one content.\nPage two content.\nPage three content.", + text: [ + "Title: Research Paper | Pages: 1 | Links: 1", + "", + "# Introduction", + "This is the body text of the paper.", + "", + "Links:", + "- https://example.com", + ].join("\n"), + }) + }) + + it("detects monospace font as fenced code blocks", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 5_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Code Doc" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [ + [ + ...buildPageItems(["Example:"], { fontSize: 10.5 }), + ...buildPageItems(["const x = 42"], { + fontSize: 10.5, + fontFamily: "monospace", + }).map((item) => ({ ...item, y: 750 })), + ...buildPageItems(["return x"], { + fontSize: 10.5, + fontFamily: "monospace", + }).map((item) => ({ ...item, y: 735 })), + ...buildPageItems(["End of example."], { fontSize: 10.5 }).map( + (item) => ({ ...item, y: 720 }), + ), + ], + ], }) - expect(mockedExtractText).toHaveBeenCalledWith(expect.any(Uint8Array), { - mergePages: true, + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "doc.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: [ + "Title: Code Doc | Pages: 1", + "", + "Example:", + "```", + "const x = 42", + "return x", + "```", + "End of example.", + ].join("\n"), + }) + }) + + it("deduplicates links in the footer", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 3_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Links Doc" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [[...buildPageItems(["Some text"])]], + }) + mockExtractLinks.mockResolvedValue({ + links: [ + "https://example.com", + "https://other.com", + "https://example.com", + ], + totalPages: 1, + }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "doc.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: [ + "Title: Links Doc | Pages: 1 | Links: 2", + "", + "Some text", + "", + "Links:", + "- https://example.com", + "- https://other.com", + ].join("\n"), + }) + }) + + it("shows (untitled) when the PDF has no title metadata", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 2_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ info: {} }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [[...buildPageItems(["Hello"])]], + }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "untitled.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: ["Title: (untitled) | Pages: 1", "", "Hello"].join("\n"), + }) + }) + + it("adds page separators for multi-page documents", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 5_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Multi-page" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 2, + items: [ + [...buildPageItems(["Page one content"])], + [...buildPageItems(["Page two content"])], + ], + }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 2 }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "multi.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: [ + "Title: Multi-page | Pages: 2", + "", + "Page one content", + "", + "--- Page 2 ---", + "", + "Page two content", + ].join("\n"), }) }) @@ -61,9 +268,9 @@ describe("readAssetContent — PDF extraction", () => { bytes: 5_000_000, extension: ".pdf", }) - mockedExtractText.mockResolvedValue({ + mockExtractTextItems.mockResolvedValue({ totalPages: 12, - text: "", + items: Array.from({ length: 12 }, () => []), }) await expect( @@ -84,9 +291,9 @@ describe("readAssetContent — PDF extraction", () => { bytes: 1_000, extension: ".pdf", }) - mockedExtractText.mockResolvedValue({ + mockExtractTextItems.mockResolvedValue({ totalPages: 1, - text: " \n\t \n ", + items: [[...buildPageItems([" ", "\t", " \n "])]], }) await expect( @@ -108,20 +315,21 @@ describe("readAssetContent — PDF extraction", () => { bytes: 500_000, extension: ".pdf", }) - mockedExtractText.mockResolvedValue({ - totalPages: 50, - text: largeText, + mockGetMeta.mockResolvedValue({ + info: { Title: "Huge" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [[...buildPageItems([largeText])]], }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) await expect( assetOperations.readAssetContent( { ...defaultParams, path: "huge.pdf" }, logger, ), - ).rejects.toThrow( - 'text output too large: "huge.pdf" renders to 200000 bytes ' + - "(cap 102400 bytes)", - ) + ).rejects.toThrow("text output too large") }) it("includes .pdf in the unsupported-type error's readable types list", async () => { @@ -144,13 +352,13 @@ describe("readAssetContent — PDF extraction", () => { ) }) - it("propagates extractText errors for corrupt PDFs", async () => { + it("propagates getDocumentProxy errors for corrupt PDFs", async () => { mockedReadAsset.mockResolvedValue({ buffer: Buffer.from("not-a-real-pdf"), bytes: 14, extension: ".pdf", }) - mockedExtractText.mockRejectedValue(new Error("Invalid PDF structure")) + mockGetDocumentProxy.mockRejectedValue(new Error("Invalid PDF structure")) await expect( assetOperations.readAssetContent( @@ -158,5 +366,91 @@ describe("readAssetContent — PDF extraction", () => { logger, ), ).rejects.toThrow("Invalid PDF structure") + + // Restore the default mock for other tests + mockGetDocumentProxy.mockResolvedValue({ cleanup: mockCleanup }) + }) + + it("cleans up the document proxy after successful extraction", async () => { + mockCleanup.mockClear() + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 1_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Cleanup Test" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [[...buildPageItems(["Content"])]], + }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) + + await assetOperations.readAssetContent( + { ...defaultParams, path: "test.pdf" }, + logger, + ) + + expect(mockCleanup).toHaveBeenCalledOnce() + }) + + it("cleans up the document proxy even when extraction throws", async () => { + mockCleanup.mockClear() + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 1_000, + extension: ".pdf", + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [[]], + }) + + await expect( + assetOperations.readAssetContent( + { ...defaultParams, path: "scanned.pdf" }, + logger, + ), + ).rejects.toThrow("PDF has no extractable text") + + expect(mockCleanup).toHaveBeenCalledOnce() + }) + + it("skips heading detection when all items share one font size", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 3_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Flat Doc" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [ + [ + ...buildPageItems(["Title Line", "Body text here"], { + fontSize: 11, + }), + ], + ], + }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "flat.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: [ + "Title: Flat Doc | Pages: 1", + "", + "Title Line", + "Body text here", + ].join("\n"), + }) }) }) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 1c9120a7..c5b0a1c6 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -1,4 +1,10 @@ -import { extractText } from "unpdf" +import { + getDocumentProxy, + getMeta, + extractTextItems, + extractLinks, +} from "unpdf" +import type { StructuredTextItem } from "unpdf" import { vaultFs } from "./vault-filesystem.js" import { linearizeCanvas } from "../obsidian-markdown/canvas.js" import { links } from "../obsidian-markdown/links.js" @@ -82,6 +88,128 @@ const decodeUtf8Strict = (params: { buffer: Buffer; path: string }): string => { } } +// ── PDF reconstruction ───────────────────────────────────────── + +/** Groups text items into lines by y-coordinate proximity — items within + * `threshold` pixels of the previous item's y are on the same line. */ +const groupIntoLines = ( + pageItems: readonly StructuredTextItem[], + threshold = 2, +): StructuredTextItem[][] => { + const nonEmpty = pageItems.filter((item) => item.str.trim().length > 0) + const firstItem = nonEmpty[0] + if (!firstItem) return [] + const lines: StructuredTextItem[][] = [[firstItem]] + for (let i = 1; i < nonEmpty.length; i++) { + const prevItem = nonEmpty[i - 1] + const currItem = nonEmpty[i] + if (!prevItem || !currItem) continue + const currentLine = lines[lines.length - 1] + if (!currentLine) continue + if (Math.abs(currItem.y - prevItem.y) < threshold) { + currentLine.push(currItem) + } else { + lines.push([currItem]) + } + } + return lines +} + +/** Builds a heading-level map from the distinct font sizes in the document. + * Up to three sizes larger than the smallest get H1–H3; the smallest is + * always body text. Returns an empty map when only one font size exists. */ +const buildHeadingLevels = ( + allItems: readonly StructuredTextItem[], +): ReadonlyMap => { + const roundedSizes = [ + ...new Set(allItems.map((item) => Math.round(item.fontSize * 10) / 10)), + ].sort((a, b) => b - a) + if (roundedSizes.length <= 1) return new Map() + const headingSizes = roundedSizes.slice(0, -1) + const levels = new Map() + for (let i = 0; i < Math.min(3, headingSizes.length); i++) { + const size = headingSizes[i] + if (size === undefined) continue + levels.set(size, i + 1) + } + return levels +} + +/** Reconstructs a markdown-formatted text rendition from structured PDF items, + * metadata, and links — heading hierarchy from relative font sizes, fenced + * code blocks from monospace fontFamily detection, page separators, and a + * deduplicated links footer. */ +const reconstructPdfMarkdown = (params: { + title: string | undefined + totalPages: number + items: readonly (readonly StructuredTextItem[])[] + pdfLinks: readonly string[] +}): string => { + const { title, totalPages, items, pdfLinks } = params + const allNonEmpty = items.flat().filter((item) => item.str.trim().length > 0) + const headingLevels = buildHeadingLevels(allNonEmpty) + const uniqueLinks = [...new Set(pdfLinks)] + + const headerParts = [ + `Title: ${title ?? "(untitled)"}`, + `Pages: ${totalPages}`, + ] + if (uniqueLinks.length > 0) { + headerParts.push(`Links: ${uniqueLinks.length}`) + } + + const sections: string[] = [headerParts.join(" | "), ""] + + for (let pageIndex = 0; pageIndex < items.length; pageIndex++) { + const pageItems = items[pageIndex] + if (!pageItems) continue + const lines = groupIntoLines(pageItems) + if (lines.length === 0) continue + if (pageIndex > 0) { + sections.push("", `--- Page ${pageIndex + 1} ---`, "") + } + + let inCodeBlock = false + for (const line of lines) { + const lineText = line + .map((item) => item.str) + .join(" ") + .trim() + if (!lineText) continue + + const isMonospace = line.some((item) => item.fontFamily === "monospace") + const maxFontSize = + Math.round(Math.max(...line.map((item) => item.fontSize)) * 10) / 10 + const headingLevel = headingLevels.get(maxFontSize) ?? 0 + + if (isMonospace && !inCodeBlock) { + sections.push("```") + inCodeBlock = true + } else if (!isMonospace && inCodeBlock) { + sections.push("```") + inCodeBlock = false + } + + if (inCodeBlock) { + sections.push(lineText) + } else if (headingLevel > 0) { + sections.push(`${"#".repeat(headingLevel)} ${lineText}`) + } else { + sections.push(lineText) + } + } + if (inCodeBlock) { + sections.push("```") + } + } + + if (uniqueLinks.length > 0) { + sections.push("", "Links:", ...uniqueLinks.map((link) => `- ${link}`)) + } + + return sections.join("\n") +} + /** * Reads a non-markdown vault file and returns its most useful representation * per type; `raw` skips the canvas rendition for the exact JSON source. @@ -136,18 +264,33 @@ const readAssetContent = async ( asset.buffer.byteOffset, asset.buffer.byteLength, ) - const extracted = await extractText(pdfData, { mergePages: true }) - const text = extracted.text - if (!text.trim()) { - throw new Error( - `PDF has no extractable text: "${path}" exists ` + - `(${asset.bytes} bytes, ${extracted.totalPages} pages) but ` + - `contains no text content — it may be a scanned document or ` + - `image-only PDF`, - ) + const proxy = await getDocumentProxy(pdfData) + try { + const meta = await getMeta(proxy) + const { totalPages, items } = await extractTextItems(proxy) + const linkResult = await extractLinks(proxy) + + const hasContent = items.flat().some((item) => item.str.trim().length > 0) + if (!hasContent) { + throw new Error( + `PDF has no extractable text: "${path}" exists ` + + `(${asset.bytes} bytes, ${totalPages} pages) but ` + + `contains no text content — it may be a scanned document or ` + + `image-only PDF`, + ) + } + + const text = reconstructPdfMarkdown({ + title: meta.info?.Title ?? undefined, + totalPages, + items, + pdfLinks: linkResult.links ?? [], + }) + assertTextWithinCap({ text, path }) + return { kind: "text", text } + } finally { + proxy.cleanup() } - assertTextWithinCap({ text, path }) - return { kind: "text", text } } throw new Error( `unsupported asset type "${asset.extension}": "${path}" exists ` + From e2a0b8c000be5b83c6778fd91d9d72ad92b5d30a Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:52:28 -0400 Subject: [PATCH 09/19] =?UTF-8?q?style:=20readability=20pass=20=E2=80=94?= =?UTF-8?q?=20let-justification=20comment,=20null-title=20comment,=20stale?= =?UTF-8?q?=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/mcp-core/tools/asset-tools.ts | 2 +- src/vault-mcp/vault-operations/asset-operations.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/mcp-core/tools/asset-tools.ts b/src/vault-mcp/mcp-core/tools/asset-tools.ts index 0cd6046e..5a12760d 100644 --- a/src/vault-mcp/mcp-core/tools/asset-tools.ts +++ b/src/vault-mcp/mcp-core/tools/asset-tools.ts @@ -43,7 +43,7 @@ Example: vault_read_asset({ path: "attachments/diagram.png" }) — the image its Example: vault_read_asset({ path: "Boards/Roadmap.canvas" }) — a readable outline of the canvas Example: vault_read_asset({ path: "Boards/Roadmap.canvas", raw: true }) — the canvas's exact JSON source Example: vault_read_asset({ path: "exports/data.json" }) — the file content as text -Example: vault_read_asset({ path: "papers/research.pdf" }) — the extracted text content +Example: vault_read_asset({ path: "papers/research.pdf" }) — structured text with title, headings, and links What each type returns: - Images (.png/.jpg/.jpeg/.gif/.webp): the image as a viewable image block — downscaled and recompressed server-side when it exceeds client response limits, delivered untouched otherwise — plus a text line stating the path, delivered format/dimensions/bytes, and the original dimensions when shrunk. Animated GIFs are reduced to their first frame when recompressed to fit the budget. diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index c5b0a1c6..42b9caeb 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -169,6 +169,8 @@ const reconstructPdfMarkdown = (params: { sections.push("", `--- Page ${pageIndex + 1} ---`, "") } + // Fence state machine — tracks whether we're inside a code block + // so monospace→sans-serif transitions emit closing fences. let inCodeBlock = false for (const line of lines) { const lineText = line @@ -280,6 +282,7 @@ const readAssetContent = async ( ) } + // Title can be null in PDF metadata — normalize to undefined const text = reconstructPdfMarkdown({ title: meta.info?.Title ?? undefined, totalPages, From d00a637e57671ded2f5868d48461b510c5f248db Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:03:23 -0400 Subject: [PATCH 10/19] test: exact text-cap assertion + trailing code fence coverage gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Exact error message for text-too-large (was substring match on deterministic output — 200024 bytes computed from header + body) - New test: code fence at end-of-page with no sans-serif transition covers the trailing `if (inCodeBlock)` branch; mutation-verified Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/asset-operations.test.ts | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts index 78b83bb2..790f7b94 100644 --- a/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts +++ b/src/vault-mcp/vault-operations/__tests__/asset-operations.test.ts @@ -329,7 +329,10 @@ describe("readAssetContent — PDF extraction", () => { { ...defaultParams, path: "huge.pdf" }, logger, ), - ).rejects.toThrow("text output too large") + ).rejects.toThrow( + 'text output too large: "huge.pdf" renders to 200024 bytes ' + + "(cap 102400 bytes)", + ) }) it("includes .pdf in the unsupported-type error's readable types list", async () => { @@ -417,6 +420,52 @@ describe("readAssetContent — PDF extraction", () => { expect(mockCleanup).toHaveBeenCalledOnce() }) + it("closes a code fence at end of page when no sans-serif transition follows", async () => { + mockedReadAsset.mockResolvedValue({ + buffer: Buffer.from("fake-pdf"), + bytes: 4_000, + extension: ".pdf", + }) + mockGetMeta.mockResolvedValue({ + info: { Title: "Trailing Code" }, + }) + mockExtractTextItems.mockResolvedValue({ + totalPages: 1, + items: [ + [ + ...buildPageItems(["Preamble"], { fontSize: 10.5 }), + ...buildPageItems(["func main() {"], { + fontSize: 10.5, + fontFamily: "monospace", + }).map((item) => ({ ...item, y: 750 })), + ...buildPageItems([" fmt.Println()"], { + fontSize: 10.5, + fontFamily: "monospace", + }).map((item) => ({ ...item, y: 735 })), + ], + ], + }) + mockExtractLinks.mockResolvedValue({ links: [], totalPages: 1 }) + + const result = await assetOperations.readAssetContent( + { ...defaultParams, path: "code.pdf" }, + logger, + ) + + expect(result).toEqual({ + kind: "text", + text: [ + "Title: Trailing Code | Pages: 1", + "", + "Preamble", + "```", + "func main() {", + "fmt.Println()", + "```", + ].join("\n"), + }) + }) + it("skips heading detection when all items share one font size", async () => { mockedReadAsset.mockResolvedValue({ buffer: Buffer.from("fake-pdf"), From c6ea2ac8d9070d97bc947ff3bec9a0ee32c6fa5b Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:06:27 -0400 Subject: [PATCH 11/19] test: collapse all decomposed assertions in asset handler tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sweep of all 9 asset tool handler integration tests — every decomposed isError + content[0]?.text pattern replaced with a single toEqual on the full result shape. Locks structure so extra fields, type drift, or isError changes are caught. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/tool-definitions.test.ts | 99 ++++++++++++------- 1 file changed, 63 insertions(+), 36 deletions(-) diff --git a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts index 6f75df75..6ec6de43 100644 --- a/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts +++ b/src/vault-mcp/mcp-core/__tests__/tool-definitions.test.ts @@ -1049,14 +1049,15 @@ describe("asset tool handlers", () => { .toBuffer() await writeFile(join(vault, "tiny.png"), png) const result = await readAsset({ path: "tiny.png" }) - expect(result.isError).toBeUndefined() - expect(result.content).toEqual([ - { type: "image", data: png.toString("base64"), mimeType: "image/png" }, - { - type: "text", - text: `tiny.png — image/png, 4×4, ${png.length} bytes (original file, not recompressed)`, - }, - ]) + expect(result).toEqual({ + content: [ + { type: "image", data: png.toString("base64"), mimeType: "image/png" }, + { + type: "text", + text: `tiny.png — image/png, 4×4, ${png.length} bytes (original file, not recompressed)`, + }, + ], + }) }) it("returns the exact canvas JSON source when raw is set", async () => { @@ -1077,8 +1078,9 @@ describe("asset tool handlers", () => { }) await writeFile(join(vault, "Board.canvas"), canvasSource, "utf8") const result = await readAsset({ path: "Board.canvas", raw: true }) - expect(result.isError).toBeUndefined() - expect(result.content).toEqual([{ type: "text", text: canvasSource }]) + expect(result).toEqual({ + content: [{ type: "text", text: canvasSource }], + }) }) it("rejects raw for an image", async () => { @@ -1095,18 +1097,24 @@ describe("asset tool handlers", () => { .toBuffer() await writeFile(join(vault, "tiny.png"), png) const result = await readAsset({ path: "tiny.png", raw: true }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: raw source is not available for images: "tiny.png" is binary — its image block is the delivered form', - ) + expect(result).toEqual({ + isError: true, + content: [ + { + type: "text", + text: '[Error]: raw source is not available for images: "tiny.png" is binary — its image block is the delivered form', + }, + ], + }) }) it("returns a text format's source unchanged when raw is set", async () => { const { vault, readAsset } = await setupAssetHarness() await writeFile(join(vault, "data.json"), '{"key": "value"}', "utf8") const result = await readAsset({ path: "data.json", raw: true }) - expect(result.isError).toBeUndefined() - expect(result.content).toEqual([{ type: "text", text: '{"key": "value"}' }]) + expect(result).toEqual({ + content: [{ type: "text", text: '{"key": "value"}' }], + }) }) it("returns structured markdown from a valid PDF", async () => { @@ -1118,42 +1126,56 @@ describe("asset tool handlers", () => { content: [ { type: "text", - text: expect.stringContaining("Hello PDF"), + text: expect.stringMatching( + /^Title: \(untitled\) \| Pages: 1\n\n[\s\S]*Hello PDF/, + ), }, ], }) - const text = (result.content[0] as { text: string }).text - expect(text).toContain("Title:") - expect(text).toContain("Pages: 1") }) it("rejects an unsupported extension naming the readable types", async () => { const { vault, readAsset } = await setupAssetHarness() await writeFile(join(vault, "song.mp3"), "xxxx", "utf8") const result = await readAsset({ path: "song.mp3" }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: unsupported asset type ".mp3": "song.mp3" exists (4 bytes). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats (.svg/.json/.txt/.csv/.xml/.log/.base)', - ) + expect(result).toEqual({ + isError: true, + content: [ + { + type: "text", + text: '[Error]: unsupported asset type ".mp3": "song.mp3" exists (4 bytes). Readable types: images (.png/.jpg/.jpeg/.gif/.webp), .canvas, .pdf, and text formats (.svg/.json/.txt/.csv/.xml/.log/.base)', + }, + ], + }) }) it("rejects a .md path without touching the note", async () => { const { vault, readAsset } = await setupAssetHarness() await writeFile(join(vault, "note.md"), "# A note\n", "utf8") const result = await readAsset({ path: "note.md" }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: not an asset: "note.md" is a markdown note', - ) + expect(result).toEqual({ + isError: true, + content: [ + { + type: "text", + text: '[Error]: not an asset: "note.md" is a markdown note', + }, + ], + }) }) it("rejects a missing asset with asset not found", async () => { const { readAsset } = await setupAssetHarness() const result = await readAsset({ path: "ghost.png" }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: asset not found: "ghost.png"', - ) + expect(result).toEqual({ + isError: true, + content: [ + { + type: "text", + text: '[Error]: asset not found: "ghost.png"', + }, + ], + }) }) it("rejects a non-UTF-8 text asset instead of corrupting it", async () => { @@ -1162,10 +1184,15 @@ describe("asset tool handlers", () => { // substitute U+FFFD; the tool must refuse instead. await writeFile(join(vault, "latin1.txt"), Buffer.from([0x68, 0x69, 0xff])) const result = await readAsset({ path: "latin1.txt" }) - expect(result.isError).toBe(true) - expect(result.content[0]?.text).toBe( - '[Error]: not valid UTF-8: "latin1.txt" cannot be returned as text', - ) + expect(result).toEqual({ + isError: true, + content: [ + { + type: "text", + text: '[Error]: not valid UTF-8: "latin1.txt" cannot be returned as text', + }, + ], + }) }) it("rejects an oversized text asset instead of truncating it", async () => { From dcc675abcfc38f24308c6b59511b07acfede9af0 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:15:44 -0400 Subject: [PATCH 12/19] =?UTF-8?q?style:=20extract=20roundFontSize=20helper?= =?UTF-8?q?,=20decompose=20nested=20composition,=20rename=20sections?= =?UTF-8?q?=E2=86=92outputLines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 2 +- .../vault-operations/asset-operations.ts | 35 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b4e9072..b80a8676 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -709,7 +709,7 @@ createTestIndex()` at the top of each test. `beforeEach` is only fragment is genuinely under test (one branch among many, or an excerpt of large output). - Never decompose: `toHaveLength(1)` + index property checks is - weaker than `expect(results.map(r => r.path)).toEqual(["foo"])` — + weaker than `expect(results.map(result => result.path)).toEqual(["foo"])` — the decomposed form misses extra items, ordering, and unexpected properties. - Deterministic error messages get exact assertions — if the message diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 42b9caeb..eb369325 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -90,6 +90,11 @@ const decodeUtf8Strict = (params: { buffer: Buffer; path: string }): string => { // ── PDF reconstruction ───────────────────────────────────────── +/** Rounds a font size to one decimal place — used as the bucketing key for + * heading-level detection. Both the map builder (`buildHeadingLevels`) and + * the per-line lookup (`reconstructPdfMarkdown`) must agree on rounding. */ +const roundFontSize = (size: number): number => Math.round(size * 10) / 10 + /** Groups text items into lines by y-coordinate proximity — items within * `threshold` pixels of the previous item's y are on the same line. */ const groupIntoLines = ( @@ -121,9 +126,9 @@ const groupIntoLines = ( const buildHeadingLevels = ( allItems: readonly StructuredTextItem[], ): ReadonlyMap => { - const roundedSizes = [ - ...new Set(allItems.map((item) => Math.round(item.fontSize * 10) / 10)), - ].sort((a, b) => b - a) + const fontSizesRounded = allItems.map((item) => roundFontSize(item.fontSize)) + const uniqueSizes = [...new Set(fontSizesRounded)] + const roundedSizes = uniqueSizes.sort((a, b) => b - a) if (roundedSizes.length <= 1) return new Map() const headingSizes = roundedSizes.slice(0, -1) const levels = new Map() @@ -158,7 +163,7 @@ const reconstructPdfMarkdown = (params: { headerParts.push(`Links: ${uniqueLinks.length}`) } - const sections: string[] = [headerParts.join(" | "), ""] + const outputLines: string[] = [headerParts.join(" | "), ""] for (let pageIndex = 0; pageIndex < items.length; pageIndex++) { const pageItems = items[pageIndex] @@ -166,7 +171,7 @@ const reconstructPdfMarkdown = (params: { const lines = groupIntoLines(pageItems) if (lines.length === 0) continue if (pageIndex > 0) { - sections.push("", `--- Page ${pageIndex + 1} ---`, "") + outputLines.push("", `--- Page ${pageIndex + 1} ---`, "") } // Fence state machine — tracks whether we're inside a code block @@ -180,36 +185,36 @@ const reconstructPdfMarkdown = (params: { if (!lineText) continue const isMonospace = line.some((item) => item.fontFamily === "monospace") - const maxFontSize = - Math.round(Math.max(...line.map((item) => item.fontSize)) * 10) / 10 + const lineFontSizes = line.map((item) => item.fontSize) + const maxFontSize = roundFontSize(Math.max(...lineFontSizes)) const headingLevel = headingLevels.get(maxFontSize) ?? 0 if (isMonospace && !inCodeBlock) { - sections.push("```") + outputLines.push("```") inCodeBlock = true } else if (!isMonospace && inCodeBlock) { - sections.push("```") + outputLines.push("```") inCodeBlock = false } if (inCodeBlock) { - sections.push(lineText) + outputLines.push(lineText) } else if (headingLevel > 0) { - sections.push(`${"#".repeat(headingLevel)} ${lineText}`) + outputLines.push(`${"#".repeat(headingLevel)} ${lineText}`) } else { - sections.push(lineText) + outputLines.push(lineText) } } if (inCodeBlock) { - sections.push("```") + outputLines.push("```") } } if (uniqueLinks.length > 0) { - sections.push("", "Links:", ...uniqueLinks.map((link) => `- ${link}`)) + outputLines.push("", "Links:", ...uniqueLinks.map((link) => `- ${link}`)) } - return sections.join("\n") + return outputLines.join("\n") } /** From 437f751197c2f7abead90cc087bb208c4f704c32 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:24:09 -0400 Subject: [PATCH 13/19] style: add blank lines between guard-continue blocks in groupIntoLines Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index eb369325..6a9461ca 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -109,8 +109,10 @@ const groupIntoLines = ( const prevItem = nonEmpty[i - 1] const currItem = nonEmpty[i] if (!prevItem || !currItem) continue + const currentLine = lines[lines.length - 1] if (!currentLine) continue + if (Math.abs(currItem.y - prevItem.y) < threshold) { currentLine.push(currItem) } else { From 2e01bb5b81971cf92ee797816d2539fceb8fbef0 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:25:20 -0400 Subject: [PATCH 14/19] style: consistent blank-line spacing between logical steps Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 6a9461ca..7e8b88b6 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -132,11 +132,13 @@ const buildHeadingLevels = ( const uniqueSizes = [...new Set(fontSizesRounded)] const roundedSizes = uniqueSizes.sort((a, b) => b - a) if (roundedSizes.length <= 1) return new Map() + const headingSizes = roundedSizes.slice(0, -1) const levels = new Map() for (let i = 0; i < Math.min(3, headingSizes.length); i++) { const size = headingSizes[i] if (size === undefined) continue + levels.set(size, i + 1) } return levels @@ -170,8 +172,10 @@ const reconstructPdfMarkdown = (params: { for (let pageIndex = 0; pageIndex < items.length; pageIndex++) { const pageItems = items[pageIndex] if (!pageItems) continue + const lines = groupIntoLines(pageItems) if (lines.length === 0) continue + if (pageIndex > 0) { outputLines.push("", `--- Page ${pageIndex + 1} ---`, "") } From bb87a27c9b644d8ce4e48b6f62ca0cb514460048 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:29:26 -0400 Subject: [PATCH 15/19] fix: add missing raw? param to vault_read_asset ARCHITECTURE table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool input table listed only `path` but the tool also accepts `raw?` — every other tool in the table lists all optional params. Co-Authored-By: Claude Opus 4.6 (1M context) --- ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3924a8fb..ead42118 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -288,7 +288,7 @@ Link queries use a `links` table populated during indexing: | Tool | Input | Annotation | | ------------------- | ------------------------------ | ------------ | -| `vault_read_asset` | `path` | readOnlyHint | +| `vault_read_asset` | `path, raw?` | readOnlyHint | | `vault_list_assets` | `folder?, extensions?, limit?` | readOnlyHint | `vault_read_asset` reads non-markdown vault files, dispatching on extension to the most useful representation per type: From d5f3d4ad60916a8ea30e24a32a1b593b4a4c6411 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:45:22 -0400 Subject: [PATCH 16/19] style: simplify groupIntoLines and buildHeadingLevels groupIntoLines: for...of with tracked lastY replaces index-based iteration and eliminates three type-narrowing guards that could never fire at runtime. buildHeadingLevels: Map constructor with .map() replaces mutable for-loop. Double slice (remove smallest, cap at 3) reads sequentially. Pipeline naming fixed (sortedSizes, not roundedSizes). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../vault-operations/asset-operations.ts | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 7e8b88b6..011dc32f 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -102,22 +102,19 @@ const groupIntoLines = ( threshold = 2, ): StructuredTextItem[][] => { const nonEmpty = pageItems.filter((item) => item.str.trim().length > 0) - const firstItem = nonEmpty[0] - if (!firstItem) return [] - const lines: StructuredTextItem[][] = [[firstItem]] - for (let i = 1; i < nonEmpty.length; i++) { - const prevItem = nonEmpty[i - 1] - const currItem = nonEmpty[i] - if (!prevItem || !currItem) continue + if (nonEmpty.length === 0) return [] + // Sequential state: track the previous item's y to detect line breaks. + const lines: StructuredTextItem[][] = [] + let lastY = -Infinity + for (const item of nonEmpty) { const currentLine = lines[lines.length - 1] - if (!currentLine) continue - - if (Math.abs(currItem.y - prevItem.y) < threshold) { - currentLine.push(currItem) + if (currentLine && Math.abs(item.y - lastY) < threshold) { + currentLine.push(item) } else { - lines.push([currItem]) + lines.push([item]) } + lastY = item.y } return lines } @@ -128,20 +125,13 @@ const groupIntoLines = ( const buildHeadingLevels = ( allItems: readonly StructuredTextItem[], ): ReadonlyMap => { - const fontSizesRounded = allItems.map((item) => roundFontSize(item.fontSize)) - const uniqueSizes = [...new Set(fontSizesRounded)] - const roundedSizes = uniqueSizes.sort((a, b) => b - a) - if (roundedSizes.length <= 1) return new Map() - - const headingSizes = roundedSizes.slice(0, -1) - const levels = new Map() - for (let i = 0; i < Math.min(3, headingSizes.length); i++) { - const size = headingSizes[i] - if (size === undefined) continue - - levels.set(size, i + 1) - } - return levels + const sortedSizes = [ + ...new Set(allItems.map((item) => roundFontSize(item.fontSize))), + ].sort((a, b) => b - a) + if (sortedSizes.length <= 1) return new Map() + + const headingSizes = sortedSizes.slice(0, -1).slice(0, 3) + return new Map(headingSizes.map((size, index) => [size, index + 1])) } /** Reconstructs a markdown-formatted text rendition from structured PDF items, From 7d0f2c63a8bfee87f69224ae8e88411d07b1c32c Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:55:27 -0400 Subject: [PATCH 17/19] style: clarify groupIntoLines with inline comments Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 011dc32f..491d5970 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -104,14 +104,18 @@ const groupIntoLines = ( const nonEmpty = pageItems.filter((item) => item.str.trim().length > 0) if (nonEmpty.length === 0) return [] - // Sequential state: track the previous item's y to detect line breaks. + // PDF text items arrive in content-stream order with y-coordinates + // descending down the page. Items on the same visual line share a y + // (within `threshold` px); a jump in y starts a new line. const lines: StructuredTextItem[][] = [] let lastY = -Infinity for (const item of nonEmpty) { const currentLine = lines[lines.length - 1] if (currentLine && Math.abs(item.y - lastY) < threshold) { + // Same visual line — append to the current group currentLine.push(item) } else { + // Y jumped — start a new line group lines.push([item]) } lastY = item.y From 5205955558e998726338ee97b2baffe1a68910ce Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:57:56 -0400 Subject: [PATCH 18/19] style: add inline comments across PDF reconstruction logic Annotate the heading-level builder (visual hierarchy, body-text exclusion), line classifier (monospace/heading detection), fence transitions, end-of-page fence close, sequential proxy calls (structuredClone constraint), and scanned-PDF guard. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 491d5970..4ac24095 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -129,11 +129,13 @@ const groupIntoLines = ( const buildHeadingLevels = ( allItems: readonly StructuredTextItem[], ): ReadonlyMap => { + // Distinct sizes sorted largest-first — the visual hierarchy of the PDF. const sortedSizes = [ ...new Set(allItems.map((item) => roundFontSize(item.fontSize))), ].sort((a, b) => b - a) if (sortedSizes.length <= 1) return new Map() + // Drop the smallest (body text) and cap at 3 heading levels. const headingSizes = sortedSizes.slice(0, -1).slice(0, 3) return new Map(headingSizes.map((size, index) => [size, index + 1])) } @@ -184,11 +186,13 @@ const reconstructPdfMarkdown = (params: { .trim() if (!lineText) continue + // Classify the line: monospace font → code, large font → heading const isMonospace = line.some((item) => item.fontFamily === "monospace") const lineFontSizes = line.map((item) => item.fontSize) const maxFontSize = roundFontSize(Math.max(...lineFontSizes)) const headingLevel = headingLevels.get(maxFontSize) ?? 0 + // Toggle code fences on monospace ↔ sans-serif transitions if (isMonospace && !inCodeBlock) { outputLines.push("```") inCodeBlock = true @@ -205,6 +209,7 @@ const reconstructPdfMarkdown = (params: { outputLines.push(lineText) } } + // Close any code fence left open at end of page if (inCodeBlock) { outputLines.push("```") } @@ -273,10 +278,13 @@ const readAssetContent = async ( ) const proxy = await getDocumentProxy(pdfData) try { + // Sequential — the unpdf worker can't handle concurrent calls + // on the same proxy (structuredClone error on Node 24). const meta = await getMeta(proxy) const { totalPages, items } = await extractTextItems(proxy) const linkResult = await extractLinks(proxy) + // Scanned/image-only PDFs produce items with no text content const hasContent = items.flat().some((item) => item.str.trim().length > 0) if (!hasContent) { throw new Error( From 8a7176e6641eeb5fea261d608e7b6ccb865377ec Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:13:19 -0400 Subject: [PATCH 19/19] style: document the 2px threshold in groupIntoLines Co-Authored-By: Claude Opus 4.6 (1M context) --- src/vault-mcp/vault-operations/asset-operations.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/vault-operations/asset-operations.ts b/src/vault-mcp/vault-operations/asset-operations.ts index 4ac24095..ec2f7f3a 100644 --- a/src/vault-mcp/vault-operations/asset-operations.ts +++ b/src/vault-mcp/vault-operations/asset-operations.ts @@ -96,7 +96,10 @@ const decodeUtf8Strict = (params: { buffer: Buffer; path: string }): string => { const roundFontSize = (size: number): number => Math.round(size * 10) / 10 /** Groups text items into lines by y-coordinate proximity — items within - * `threshold` pixels of the previous item's y are on the same line. */ + * `threshold` pixels of the previous item's y are on the same line. + * Default 2px absorbs sub-pixel jitter from font metrics and inline + * elements while staying well below the smallest real line gap (~12px + * for body text at typical PDF sizes). */ const groupIntoLines = ( pageItems: readonly StructuredTextItem[], threshold = 2,