From 5b64fb72c62bc4c9701ed70b0ae9a150b59dd186 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 19:43:03 -0700 Subject: [PATCH 1/5] Add a failing end-to-end proof that a zero-chunk shortfall is invisible to the audit Forty .mdx pages, eight of them pure-JSX stubs whose prose lives in an excluded snippet. The real chunker strips them to nothing and returns [], the real pipeline writes nothing, and the index ends up holding 32 of 40 files. runReindexAudit() reports []. --- src/__tests__/reindex-audit-shortfall.test.ts | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/__tests__/reindex-audit-shortfall.test.ts diff --git a/src/__tests__/reindex-audit-shortfall.test.ts b/src/__tests__/reindex-audit-shortfall.test.ts new file mode 100644 index 0000000..dcc2153 --- /dev/null +++ b/src/__tests__/reindex-audit-shortfall.test.ts @@ -0,0 +1,164 @@ +/** + * End-to-end proof for the post-reindex shortfall audit. + * + * Models the real failure that ran undetected for months: a `.mdx` page whose + * prose lives entirely in an excluded snippet, so the file is walked, matched, + * read, stripped to nothing, and chunks to ZERO. The chunker returns `[]`, the + * pipeline writes nothing, and the file silently vanishes from the index. + * + * The chunker and the pipeline here are REAL — only the database and the disk + * walk are faked — so the test exercises the actual drop path rather than a + * hand-built "db is smaller" fixture. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const { + mockGetConfig, + mockGetServerConfig, + mockWalkSourceFiles, + indexedChunks, +} = vi.hoisted(() => ({ + mockGetConfig: vi.fn(), + mockGetServerConfig: vi.fn(), + mockWalkSourceFiles: vi.fn(), + // source_name → file_path → chunk count. Stands in for the chunks table. + indexedChunks: new Map>(), +})); + +function chunkTable(source: string): Map { + let t = indexedChunks.get(source); + if (!t) { + t = new Map(); + indexedChunks.set(source, t); + } + return t; +} + +vi.mock("../config.js", () => ({ + getConfig: (...args: unknown[]) => mockGetConfig(...args), + getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args), +})); + +vi.mock("../indexing/utils.js", () => ({ + walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args), +})); + +// A tiny in-memory stand-in for the chunks table, shared by the pipeline +// (writer) and the audit (reader) so the audit sees exactly what the pipeline +// actually persisted. +vi.mock("../db/queries.js", () => ({ + replaceChunksForFile: async ( + source: string, + filePath: string, + chunks: unknown[], + ) => { + const t = chunkTable(source); + // Mirrors the real delete+insert: an empty array deletes the row and + // inserts nothing, so the file disappears from `SELECT DISTINCT file_path`. + if (chunks.length === 0) t.delete(filePath); + else t.set(filePath, chunks.length); + }, + deleteChunksByFile: async (source: string, filePath: string) => { + chunkTable(source).delete(filePath); + }, + getIndexedItemIds: async (source: string) => + new Set(chunkTable(source).keys()), +})); + +import { IndexingPipeline } from "../indexing/pipeline.js"; +import { runReindexAudit, resetAuditCache } from "../indexing/reindex-audit.js"; +import type { SourceConfig } from "../types.js"; +import type { ContentItem } from "../indexing/providers/types.js"; + +const sourceConfig = { + name: "docs", + type: "markdown", + path: "/repo/docs", + file_patterns: ["**/*.mdx"], + chunk: {}, +} satisfies Partial as SourceConfig; + +function appConfig() { + return { + databaseUrl: "postgresql://test", + openaiApiKey: "test-key", + githubToken: "", + cloneDir: "/tmp/test", + slackWebhookUrl: "", + }; +} + +/** A prose page: real markdown, chunks to something. */ +function prosePage(n: number): string { + return `---\ntitle: Page ${n}\n---\n\n# Page ${n}\n\nReal prose for page ${n}.\n`; +} + +/** + * A pure-JSX stub: the visible prose lives in an excluded snippet component, so + * after MDX stripping there is nothing left. This is the 130-file case. + */ +function jsxStubPage(n: number): string { + return `---\ntitle: Stub ${n}\n---\n\n\n\n`; +} + +const embeddingProvider = { + embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]), +}; + +/** Index 40 pages, 8 of which are pure-JSX stubs. Returns the disk file set. */ +async function indexCorpus(): Promise> { + const pipeline = new IndexingPipeline( + embeddingProvider as never, + sourceConfig, + ); + const items: ContentItem[] = []; + const disk = new Set(); + for (let n = 0; n < 40; n++) { + const isStub = n % 5 === 0; // 8 of 40 → a 20% shortfall + const id = `page-${n}.mdx`; + disk.add(id); + items.push({ id, content: isStub ? jsxStubPage(n) : prosePage(n) }); + } + const { failedIds } = await pipeline.indexItems(items, "sha-1"); + expect(failedIds).toEqual([]); + return disk; +} + +describe("post-reindex shortfall audit (zero-chunk files)", () => { + beforeEach(() => { + vi.clearAllMocks(); + indexedChunks.clear(); + resetAuditCache(); + mockGetConfig.mockReturnValue(appConfig()); + mockGetServerConfig.mockReturnValue({ sources: [sourceConfig] }); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("reports a db_has_fewer finding when pure-JSX stubs chunk to zero", async () => { + const disk = await indexCorpus(); + mockWalkSourceFiles.mockResolvedValue(disk); + + // Precondition: the pipeline really did drop the stubs (the bug's mechanism). + const indexed = chunkTable("docs"); + expect(disk.size).toBe(40); + expect(indexed.size).toBe(32); + + const findings = await runReindexAudit(["docs"]); + + const shortfall = findings.find( + (f) => f.check === "count_divergence" && f.direction === "db_has_fewer", + ); + expect(shortfall).toBeDefined(); + expect(shortfall!.source).toBe("docs"); + expect(shortfall!.count).toBe(8); + // The finding names the files, so the operator can open one and see why. + expect(shortfall!.samples).toEqual( + expect.arrayContaining(["page-0.mdx", "page-5.mdx"]), + ); + expect(shortfall!.samples.length).toBeLessThanOrEqual(10); + }); +}); From a3e6ccbb2a311affe830cf368ca633cc08e445b9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 19:45:18 -0700 Subject: [PATCH 2/5] Report the index shortfall the audit was structurally unable to see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check 3 only ever fired on db_has_more. The db_has_fewer direction existed in the finding type and was never implemented, so an index holding fewer files than disk produced no finding at all — the blindness that let a wrong crawl scope, a wedged source, and 130 zero-chunk stubs run undetected. Measured on the set difference (disk minus db), not the raw count delta: the delta is a lossy proxy that misses an equal-count stale/missing swap, and the set difference yields the actual paths so the finding names the files. Gated on a per-source tolerance (5% of walked files by default, with a 3-file floor for small sources, overridable via the new unindexed_tolerance source key) so a healthy source stays silent instead of emitting a finding on every reindex. The dedup cache now keys on direction too; both directions can fire for one source in one run, and the old key let one overwrite the other. --- src/__tests__/reindex-audit-shortfall.test.ts | 53 +++++++-- src/__tests__/reindex-audit.test.ts | 111 +++++++++++++++++- src/indexing/reindex-audit.ts | 71 ++++++++++- src/types.ts | 8 ++ 4 files changed, 226 insertions(+), 17 deletions(-) diff --git a/src/__tests__/reindex-audit-shortfall.test.ts b/src/__tests__/reindex-audit-shortfall.test.ts index dcc2153..a947777 100644 --- a/src/__tests__/reindex-audit-shortfall.test.ts +++ b/src/__tests__/reindex-audit-shortfall.test.ts @@ -105,16 +105,19 @@ const embeddingProvider = { embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]), }; -/** Index 40 pages, 8 of which are pure-JSX stubs. Returns the disk file set. */ -async function indexCorpus(): Promise> { - const pipeline = new IndexingPipeline( - embeddingProvider as never, - sourceConfig, - ); +/** + * Index 40 pages, `stubEvery`-th of which is a pure-JSX stub. Returns the disk + * file set. `config` lets a test exercise a per-source tolerance override. + */ +async function indexCorpus( + stubEvery: number, + config: SourceConfig = sourceConfig, +): Promise> { + const pipeline = new IndexingPipeline(embeddingProvider as never, config); const items: ContentItem[] = []; const disk = new Set(); for (let n = 0; n < 40; n++) { - const isStub = n % 5 === 0; // 8 of 40 → a 20% shortfall + const isStub = n % stubEvery === 0; const id = `page-${n}.mdx`; disk.add(id); items.push({ id, content: isStub ? jsxStubPage(n) : prosePage(n) }); @@ -139,7 +142,7 @@ describe("post-reindex shortfall audit (zero-chunk files)", () => { }); it("reports a db_has_fewer finding when pure-JSX stubs chunk to zero", async () => { - const disk = await indexCorpus(); + const disk = await indexCorpus(5); // 8 of 40 → a 20% shortfall mockWalkSourceFiles.mockResolvedValue(disk); // Precondition: the pipeline really did drop the stubs (the bug's mechanism). @@ -161,4 +164,38 @@ describe("post-reindex shortfall audit (zero-chunk files)", () => { ); expect(shortfall!.samples.length).toBeLessThanOrEqual(10); }); + + // ── Negative assertion: the new check must not become noise ───────────── + // + // Every source drops SOME files legitimately. If a normal reindex of a + // healthy source emits a finding, operators mute the audit and the next real + // shrink goes unseen — which is how the shortfall direction got suppressed in + // the first place. A healthy source must stay silent. + it("does NOT report a shortfall for a source skipping a normal share of files", async () => { + // 1 empty page out of 40 → 2.5%, inside the 5% default tolerance. + const disk = await indexCorpus(40); + mockWalkSourceFiles.mockResolvedValue(disk); + expect(chunkTable("docs").size).toBe(39); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("reports that same small shortfall once the source sets unindexed_tolerance: 0", async () => { + // A source known to index everything it walks opts into a zero baseline, + // and the audit then flags the very first regression. + const strict = { ...sourceConfig, unindexed_tolerance: 0 }; + mockGetServerConfig.mockReturnValue({ sources: [strict] }); + + const disk = await indexCorpus(40, strict); + mockWalkSourceFiles.mockResolvedValue(disk); + + const findings = await runReindexAudit(["docs"]); + + const shortfall = findings.find((f) => f.direction === "db_has_fewer"); + expect(shortfall).toBeDefined(); + expect(shortfall!.count).toBe(1); + expect(shortfall!.samples).toEqual(["page-0.mdx"]); + }); }); diff --git a/src/__tests__/reindex-audit.test.ts b/src/__tests__/reindex-audit.test.ts index 12fbcfc..0c3a29c 100644 --- a/src/__tests__/reindex-audit.test.ts +++ b/src/__tests__/reindex-audit.test.ts @@ -296,16 +296,118 @@ describe("runReindexAudit", () => { expect(divergence!.count).toBe(2); // difference: 3 - 1 }); - it("does not report divergence when disk > DB (db_has_fewer is expected from content filtering)", async () => { + it("returns count_divergence with direction db_has_fewer when disk > DB beyond tolerance", async () => { mockGetIndexedItemIds.mockResolvedValue(new Set(["a.md"])); mockWalkSourceFiles.mockResolvedValue( - new Set(["a.md", "b.md", "c.md", "d.md"]), + new Set(["a.md", "b.md", "c.md", "d.md", "e.md"]), ); const findings = await runReindexAudit(["docs"]); const divergence = findings.find((f) => f.check === "count_divergence"); - expect(divergence).toBeUndefined(); + expect(divergence).toBeDefined(); + expect(divergence!.direction).toBe("db_has_fewer"); + expect(divergence!.count).toBe(4); + // Names the missing files — the whole point of the check. + expect(divergence!.samples).toEqual(["b.md", "c.md", "d.md", "e.md"]); + }); + + it("stays silent when the shortfall is inside the default tolerance", async () => { + // 3 of 200 walked files missing (1.5%), under the 5% default. A healthy + // source must not emit a finding on every reindex. + const disk = new Set( + Array.from({ length: 200 }, (_, i) => `page-${i}.md`), + ); + const db = new Set([...disk].slice(3)); + mockGetIndexedItemIds.mockResolvedValue(db); + mockWalkSourceFiles.mockResolvedValue(disk); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("reports once the shortfall crosses the proportional tolerance", async () => { + // 11 of 200 (5.5%) is over budget (floor(200 * 0.05) = 10). + const disk = new Set( + Array.from({ length: 200 }, (_, i) => `page-${i}.md`), + ); + const db = new Set([...disk].slice(11)); + mockGetIndexedItemIds.mockResolvedValue(db); + mockWalkSourceFiles.mockResolvedValue(disk); + + const findings = await runReindexAudit(["docs"]); + + const divergence = findings.find((f) => f.check === "count_divergence"); + expect(divergence).toBeDefined(); + expect(divergence!.direction).toBe("db_has_fewer"); + expect(divergence!.count).toBe(11); + expect(divergence!.samples).toHaveLength(10); + }); + + it("does not report 1-2 missing files on a small source (absolute floor)", async () => { + // 2 of 10 is 20% but only 2 files: the floor keeps small sources quiet. + const disk = new Set(Array.from({ length: 10 }, (_, i) => `p${i}.md`)); + const db = new Set([...disk].slice(2)); + mockGetIndexedItemIds.mockResolvedValue(db); + mockWalkSourceFiles.mockResolvedValue(disk); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("honours a per-source unindexed_tolerance override of 0", async () => { + mockGetServerConfig.mockReturnValue( + serverConfig([fileSource("docs", { unindexed_tolerance: 0 })]), + ); + mockGetIndexedItemIds.mockResolvedValue(new Set(["a.md"])); + mockWalkSourceFiles.mockResolvedValue(new Set(["a.md", "b.md"])); + + const findings = await runReindexAudit(["docs"]); + + const divergence = findings.find((f) => f.check === "count_divergence"); + expect(divergence).toBeDefined(); + expect(divergence!.direction).toBe("db_has_fewer"); + expect(divergence!.count).toBe(1); + expect(divergence!.samples).toEqual(["b.md"]); + }); + + it("honours a per-source unindexed_tolerance override that widens the budget", async () => { + mockGetServerConfig.mockReturnValue( + serverConfig([fileSource("docs", { unindexed_tolerance: 0.5 })]), + ); + mockGetIndexedItemIds.mockResolvedValue(new Set(["a.md"])); + mockWalkSourceFiles.mockResolvedValue( + new Set(["a.md", "b.md", "c.md", "d.md"]), + ); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("reports BOTH directions when the index is simultaneously stale and short", async () => { + // 6 stale rows in the index and 5 real files missing from it. Keying the + // dedup cache on source:check alone would let one direction overwrite the + // other; both must survive as separate findings. + mockGetIndexedItemIds.mockResolvedValue( + new Set([ + "a.md", + ...Array.from({ length: 6 }, (_, i) => `gone-${i}.md`), + ]), + ); + mockWalkSourceFiles.mockResolvedValue( + new Set(["a.md", "b.md", "c.md", "d.md", "e.md", "f.md"]), + ); + + const findings = await runReindexAudit(["docs"]); + + const directions = findings + .filter((f) => f.check === "count_divergence") + .map((f) => f.direction) + .sort(); + expect(directions).toEqual(["db_has_fewer", "db_has_more"]); }); it("does not report divergence when counts match", async () => { @@ -491,7 +593,8 @@ describe("runReindexAudit", () => { expect(docsStale!.count).toBe(1); expect(docsStale!.samples).toContain("old.md"); - // code: disk > DB (db_has_fewer) is no longer reported — expected from content filtering + // code: 2 of 3 files unindexed. A big fraction, but only 2 files — the + // absolute floor keeps a source this small from alerting on noise. const codeDivergence = findings.find( (f) => f.source === "code" && f.check === "count_divergence", ); diff --git a/src/indexing/reindex-audit.ts b/src/indexing/reindex-audit.ts index e0d6fc6..dc4201d 100644 --- a/src/indexing/reindex-audit.ts +++ b/src/indexing/reindex-audit.ts @@ -5,9 +5,38 @@ import { isFileSourceConfig } from "../types.js"; import type { FileSourceConfig } from "../types.js"; // Dedup: only alert when findings change from the previous audit run. -// Key = "source:check", value = count. If the counts match, skip the alert. +// Key = "source:check:direction", value = count. If the counts match, skip the +// alert. The direction is part of the key because a single source can produce +// BOTH directions of count_divergence in one run (stale rows in the index AND +// unindexed files on disk); keying on "source:check" alone would let one +// overwrite the other and silently suppress its alert. const lastAuditFindings = new Map(); +/** + * Default fraction of a source's walked files that may be absent from the + * index without the audit reporting a shortfall. + * + * Some shortfall is legitimate: the indexer walks and matches a file, then + * drops it because it carries no semantic content (an SVG, a base64 blob, an + * empty file). Reporting every such file would put a finding on every source + * on every reindex, and an alarm that always fires is an alarm that gets + * muted — which is exactly how the shortfall direction came to be suppressed + * outright. So the check reports only shortfall ABOVE a per-source baseline. + * + * 5% is deliberately loose for an unconfigured source. Once a source is known + * to index everything it walks, set `unindexed_tolerance: 0` on it and the + * audit flags the first regression instead. + */ +const DEFAULT_UNINDEXED_TOLERANCE = 0.05; + +/** + * Floor on the tolerance budget, so a small source does not alert on one or + * two legitimately-empty files (on a 20-file source, 5% rounds down to 1). + * Bypassed entirely when a source sets `unindexed_tolerance: 0`, which is an + * explicit request to hear about any shortfall at all. + */ +const MIN_UNINDEXED_FILES = 3; + export function resetAuditCache(): void { lastAuditFindings.clear(); } @@ -73,8 +102,7 @@ export async function runReindexAudit( } } - // Check 3 — Count divergence (db_has_more only; db_has_fewer is expected - // when the indexer filters low-semantic-value files like SVGs, base64, etc.) + // Check 3 — Count divergence, both directions. const dbCount = dbFiles.size; const diskCount = diskFiles.size; if (dbCount > diskCount) { @@ -86,6 +114,34 @@ export async function runReindexAudit( direction: "db_has_more", }); } + + // Check 3b — Shortfall: files on disk that the index does not hold. + // + // Measured on the SET DIFFERENCE rather than on `diskCount - dbCount`. + // The raw count difference is a lossy proxy: a source holding one stale + // row and missing one real file has matching counts and would report + // nothing, which is the same blindness in miniature. The set difference + // fires in every case the count comparison would, plus that one — and it + // yields the actual file paths, so the finding names the files instead of + // saying "the count is off by 130". + const unindexed = [...diskFiles].filter((p) => !dbFiles.has(p)); + if (unindexed.length > 0) { + const tolerance = + sourceConfig.unindexed_tolerance ?? DEFAULT_UNINDEXED_TOLERANCE; + const budget = Math.max( + tolerance > 0 ? MIN_UNINDEXED_FILES : 0, + Math.floor(diskCount * tolerance), + ); + if (unindexed.length > budget) { + findings.push({ + source: sourceConfig.name, + check: "count_divergence", + count: unindexed.length, + samples: unindexed.slice(0, 10), + direction: "db_has_fewer", + }); + } + } } // Always log findings to console @@ -100,7 +156,7 @@ export async function runReindexAudit( // Only Slack-alert on NEW or CHANGED findings (dedup) const newFindings = findings.filter((f) => { - const key = `${f.source}:${f.check}`; + const key = dedupKey(f); const prev = lastAuditFindings.get(key); return prev === undefined || prev !== f.count; }); @@ -113,7 +169,7 @@ export async function runReindexAudit( } } for (const f of findings) { - lastAuditFindings.set(`${f.source}:${f.check}`, f.count); + lastAuditFindings.set(dedupKey(f), f.count); } if (newFindings.length > 0 && cfg.slackWebhookUrl) { @@ -130,6 +186,11 @@ export async function runReindexAudit( } } +/** Dedup identity of a finding: source + check + direction. */ +function dedupKey(f: AuditFinding): string { + return `${f.source}:${f.check}${f.direction ? `:${f.direction}` : ""}`; +} + async function sendSlackAlert( findings: AuditFinding[], webhookUrl: string, diff --git a/src/types.ts b/src/types.ts index b7784c0..feac3b0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -130,6 +130,14 @@ export const FileSourceConfigSchema = z.object({ exclude_patterns: z.array(z.string()).optional(), skip_dirs: z.array(z.string()).optional(), max_file_size: z.number().int().positive().optional(), + // Fraction of this source's walked files that may be absent from the index + // before the post-reindex audit reports a shortfall (see + // src/indexing/reindex-audit.ts, Check 3). Some shortfall is legitimate — + // the indexer drops files with no semantic content — so the audit needs a + // per-source baseline rather than a single global guess. Set it to 0 once a + // source is known to index every file, and the audit will then flag the very + // first regression. Defaults to 0.05. + unindexed_tolerance: z.number().min(0).max(1).optional(), }); // Slack source schema — different required fields From 9fd97405db4610d4c792de08bd0c56ebcd56e730 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 19:45:56 -0700 Subject: [PATCH 3/5] Add failing tests for the two silent zero-chunk drop sites --- src/__tests__/markdown-chunker.test.ts | 67 +++++++++++++++++++++++++- src/__tests__/pipeline.test.ts | 45 +++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/__tests__/markdown-chunker.test.ts b/src/__tests__/markdown-chunker.test.ts index e04d503..188d4c4 100644 --- a/src/__tests__/markdown-chunker.test.ts +++ b/src/__tests__/markdown-chunker.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, afterEach, vi } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -3115,4 +3115,69 @@ describe("chunkMarkdown inlined-snippet byte normalization", () => { expect(/[\u{E000}-\u{E003}]/u.test(chunk.content)).toBe(false); } }); + // ── Zero-chunk files must be audible ──────────────────────────────── + // + // A file that chunks to nothing is dropped from the index. Silently doing + // that is how 130 pure-JSX stub pages — 19% of a source — stayed missing for + // months. Every path that returns [] has to say which file and why. + + describe("zero-chunk warnings", () => { + it("warns, naming the file, when the content is empty", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + expect(chunkMarkdown(" \n\n ", "docs/blank.mdx", mkConfig())).toEqual( + [], + ); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("docs/blank.mdx"), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("no chunks")); + } finally { + warn.mockRestore(); + } + }); + + it("warns, naming the file and the reason, when MDX stripping empties a pure-JSX stub", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const stub = [ + "---", + "title: Quickstart", + "---", + "", + '', + '', + ].join("\n"); + + expect(chunkMarkdown(stub, "docs/quickstart.mdx", mkConfig())).toEqual( + [], + ); + + const messages = warn.mock.calls.map((c) => String(c[0])); + expect(messages.some((m) => m.includes("docs/quickstart.mdx"))).toBe( + true, + ); + // The reason must distinguish "the file was empty" from "we stripped + // it to empty" — they have completely different fixes. + expect(messages.some((m) => m.includes("MDX stripping"))).toBe(true); + } finally { + warn.mockRestore(); + } + }); + + it("does not warn for a file that chunks normally", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const chunks = chunkMarkdown( + "# Title\n\nSome real prose.\n", + "docs/real.mdx", + mkConfig(), + ); + expect(chunks.length).toBeGreaterThan(0); + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } + }); + }); }); diff --git a/src/__tests__/pipeline.test.ts b/src/__tests__/pipeline.test.ts index 5806fc3..49d5728 100644 --- a/src/__tests__/pipeline.test.ts +++ b/src/__tests__/pipeline.test.ts @@ -123,6 +123,51 @@ describe("IndexingPipeline", () => { expect(embeddingClient.embedBatch).not.toHaveBeenCalled(); }); + it("warns, naming the item, when an item produces zero chunks", async () => { + // Dropping an item from the index without a word is what let 130 stub + // pages vanish for months. The pipeline must say which item it dropped and + // that nothing was written, so the drop is greppable in the logs the way + // next_acquire_reason and quarantined_items are. + const { getChunker } = await import("../indexing/chunking/index.js"); + vi.mocked(getChunker).mockReturnValueOnce(() => []); + + const embeddingClient = new EmbeddingClient("key", "model", 1536); + const pipeline = new IndexingPipeline(embeddingClient, testConfig); + + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await pipeline.indexItems( + [{ id: "stub.mdx", content: "" }], + "abc", + ); + + const messages = warn.mock.calls.map((c) => String(c[0])); + expect(messages.some((m) => m.includes("stub.mdx"))).toBe(true); + expect(messages.some((m) => m.includes("[pipeline:test-source]"))).toBe( + true, + ); + expect(messages.some((m) => m.includes("zero chunks"))).toBe(true); + } finally { + warn.mockRestore(); + } + }); + + it("does not warn for an item that produces chunks", async () => { + const embeddingClient = new EmbeddingClient("key", "model", 1536); + const pipeline = new IndexingPipeline(embeddingClient, testConfig); + + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await pipeline.indexItems( + [{ id: "real.md", content: "# Real\n\nprose" }], + "abc", + ); + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } + }); + it("removes items by ID", async () => { const embeddingClient = new EmbeddingClient("key", "model", 1536); const pipeline = new IndexingPipeline(embeddingClient, testConfig); From 23a7c7746568239a000cec1165e075febdf25ff7 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 19:46:20 -0700 Subject: [PATCH 4/5] Make the two silent zero-chunk drop sites audible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chunkMarkdown returned [] from three places and pipeline.indexItem wrote nothing, all without a word. A walked, matched, read file left the pipeline and the only trace was a file count that quietly did not add up. Each chunker exit now names the file and a distinct reason — empty file, MDX stripping left no prose, every split trimmed to empty — because the fixes differ. The pipeline logs that the index actually lost the item, prefixed by source and id. --- src/indexing/chunking/markdown.ts | 28 ++++++++++++++++++++++++++++ src/indexing/pipeline.ts | 12 ++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/indexing/chunking/markdown.ts b/src/indexing/chunking/markdown.ts index 7598ed2..dc761fd 100644 --- a/src/indexing/chunking/markdown.ts +++ b/src/indexing/chunking/markdown.ts @@ -1424,6 +1424,15 @@ export function chunkMarkdown( absoluteFilePath?: string, ): ChunkOutput[] { if (!content || !content.trim()) { + // A file that chunks to nothing is dropped from the index, and a drop with + // no log line is invisible: the walker matched the file, the reader read + // it, and the pipeline writes nothing, so the only trace is a file count + // that quietly does not add up. Name the file and the reason at every + // return-[] site so an operator can grep "[chunker] no chunks" and see + // exactly which pages are missing and why. + console.warn( + `[chunker] no chunks for ${filePath}: file is empty or whitespace-only; it will not be indexed`, + ); return []; } @@ -1483,6 +1492,15 @@ export function chunkMarkdown( const cleanBody = stripMdx(inlinedBody); if (!cleanBody.trim()) { + // The distinct, actionable case: the file HAS content, but every byte of + // it was JSX/import syntax that MDX stripping removed — a pure-component + // stub whose prose lives somewhere the indexer does not read (an excluded + // snippet directory, for instance). Keep this reason separate from the + // empty-file one above: the fixes are entirely different. + console.warn( + `[chunker] no chunks for ${filePath}: MDX stripping left no prose ` + + `(the file is JSX/imports only — its text may live in a snippet or component that is not indexed); it will not be indexed`, + ); return []; } @@ -1560,5 +1578,15 @@ export function chunkMarkdown( }); } + if (chunks.length === 0) { + // The third way out with nothing: the body survived stripping but every + // candidate chunk trimmed away to "" and hit the `continue` above. Rare, + // but it drops the file just as completely as the two returns above, so it + // gets the same treatment rather than a silent empty array. + console.warn( + `[chunker] no chunks for ${filePath}: every split produced empty text after trimming; it will not be indexed`, + ); + } + return chunks; } diff --git a/src/indexing/pipeline.ts b/src/indexing/pipeline.ts index 07083e4..8c6b050 100644 --- a/src/indexing/pipeline.ts +++ b/src/indexing/pipeline.ts @@ -101,6 +101,18 @@ export class IndexingPipeline { ); if (chunkOutputs.length === 0) { + // Say so. An item that chunks to nothing is removed from the index, and + // a removal nobody logs is indistinguishable from a file that was never + // there: the shortfall only shows up later as a file count that does not + // add up. The chunker logs WHY (see the `[chunker] no chunks` warnings); + // this line records that the index actually lost the item, keyed by + // source and id so it is greppable alongside next_acquire_reason and + // quarantined_items. + console.warn( + `${this.logPrefix} ${item.id} produced zero chunks; nothing indexed ` + + `(any previously indexed chunks for it are being cleared)`, + ); + // The item produced zero chunks. If it previously had chunks (and is // routed through `items` rather than `removedIds`), early-returning here // would leave those stale chunks in the index forever. Clear them via the From a13b41a928e6e3886149dfe3140cf7de0da69437 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 19:48:53 -0700 Subject: [PATCH 5/5] Document unindexed_tolerance and type the test embedding double check-test-shapes flagged the `as never` on the pipeline's embedding provider; it is an EmbeddingProvider, so say so. --- docs/config/index.html | 19 +++++++++++++++++++ src/__tests__/reindex-audit-shortfall.test.ts | 10 ++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/config/index.html b/docs/config/index.html index a8ab471..420339b 100644 --- a/docs/config/index.html +++ b/docs/config/index.html @@ -722,6 +722,12 @@

sources

# Max file size in bytes to index (optional) + unindexed_tolerance: + 0.05 + # Share of walked files that may be missing from the index before + the post-reindex audit reports a shortfall (optional) category: faq # Optional. Marks content as FAQ for /faq.txt and knowledge @@ -754,6 +760,19 @@

sources

    +
  • + unindexed_tolerance — After each reindex, an audit + compares the files walked on disk against the files present in the + index and reports a count_divergence finding (direction + db_has_fewer) naming the files the index is missing. + Some shortfall is normal — a file with no extractable prose is + walked and then dropped — so the finding only fires above this share + of the source's walked files (default 0.05, with a + floor of 3 files so small sources stay quiet). Set it to + 0 once a source is known to index everything it walks, + and the audit will flag the very first regression. It does not + affect what gets indexed, so changing it does not trigger a reindex. +
  • type — Determines chunking strategy. markdown splits on headings and uses token-based diff --git a/src/__tests__/reindex-audit-shortfall.test.ts b/src/__tests__/reindex-audit-shortfall.test.ts index a947777..d39de26 100644 --- a/src/__tests__/reindex-audit-shortfall.test.ts +++ b/src/__tests__/reindex-audit-shortfall.test.ts @@ -69,14 +69,15 @@ import { IndexingPipeline } from "../indexing/pipeline.js"; import { runReindexAudit, resetAuditCache } from "../indexing/reindex-audit.js"; import type { SourceConfig } from "../types.js"; import type { ContentItem } from "../indexing/providers/types.js"; +import type { EmbeddingProvider } from "../indexing/embeddings.js"; -const sourceConfig = { +const sourceConfig: SourceConfig = { name: "docs", type: "markdown", path: "/repo/docs", file_patterns: ["**/*.mdx"], chunk: {}, -} satisfies Partial as SourceConfig; +}; function appConfig() { return { @@ -101,7 +102,8 @@ function jsxStubPage(n: number): string { return `---\ntitle: Stub ${n}\n---\n\n\n\n`; } -const embeddingProvider = { +const embeddingProvider: EmbeddingProvider = { + embed: async () => [0.1, 0.2, 0.3], embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]), }; @@ -113,7 +115,7 @@ async function indexCorpus( stubEvery: number, config: SourceConfig = sourceConfig, ): Promise> { - const pipeline = new IndexingPipeline(embeddingProvider as never, config); + const pipeline = new IndexingPipeline(embeddingProvider, config); const items: ContentItem[] = []; const disk = new Set(); for (let n = 0; n < 40; n++) {