Report the index shortfall the reindex audit was structurally unable to see

docs/config/index.html

@@ -722,6 +722,12 @@ <h2 id="sources">sources</h2>
           <span class="comment"
             ># Max file size in bytes to index (optional)</span
           >
+          <span class="key">unindexed_tolerance:</span>
+          <span class="value">0.05</span>
+          <span class="comment"
+            ># Share of walked files that may be missing from the index before
+            the post-reindex audit reports a shortfall (optional)</span
+          >
           <span class="key">category:</span> <span class="value">faq</span>
           <span class="comment"
             ># Optional. Marks content as FAQ for /faq.txt and knowledge
@@ -754,6 +760,19 @@ <h2 id="sources">sources</h2>
         </div>
 
         <ul>
+          <li>
+            <strong>unindexed_tolerance</strong> — After each reindex, an audit
+            compares the files walked on disk against the files present in the
+            index and reports a <code>count_divergence</code> finding (direction
+            <code>db_has_fewer</code>) naming the files the index is missing.
+            Some shortfall is normal — a file with no extractable prose is
+            walked and then dropped — so the finding only fires above this share
+            of the source's walked files (default <code>0.05</code>, with a
+            floor of 3 files so small sources stay quiet). Set it to
+            <code>0</code> once a source is known to index everything it walks,
+            and the audit will flag the very first regression. It does not
+            affect what gets indexed, so changing it does not trigger a reindex.
+          </li>
           <li>
             <strong>type</strong> — Determines chunking strategy.
             <code>markdown</code> splits on headings and uses token-based

src/__tests__/markdown-chunker.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, afterEach } from "vitest";
+import { describe, it, expect, afterEach, vi } from "vitest";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -3115,4 +3115,69 @@ describe("chunkMarkdown inlined-snippet byte normalization", () => {
       expect(/[\u{E000}-\u{E003}]/u.test(chunk.content)).toBe(false);
     }
   });
+  // ── Zero-chunk files must be audible ────────────────────────────────
+  //
+  // A file that chunks to nothing is dropped from the index. Silently doing
+  // that is how 130 pure-JSX stub pages — 19% of a source — stayed missing for
+  // months. Every path that returns [] has to say which file and why.
+
+  describe("zero-chunk warnings", () => {
+    it("warns, naming the file, when the content is empty", () => {
+      const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+      try {
+        expect(chunkMarkdown("   \n\n ", "docs/blank.mdx", mkConfig())).toEqual(
+          [],
+        );
+        expect(warn).toHaveBeenCalledWith(
+          expect.stringContaining("docs/blank.mdx"),
+        );
+        expect(warn).toHaveBeenCalledWith(expect.stringContaining("no chunks"));
+      } finally {
+        warn.mockRestore();
+      }
+    });
+
+    it("warns, naming the file and the reason, when MDX stripping empties a pure-JSX stub", () => {
+      const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+      try {
+        const stub = [
+          "---",
+          "title: Quickstart",
+          "---",
+          "",
+          '<Snippet file="shared/quickstart.mdx" />',
+          '<ComponentDemo name="quickstart" />',
+        ].join("\n");
+
+        expect(chunkMarkdown(stub, "docs/quickstart.mdx", mkConfig())).toEqual(
+          [],
+        );
+
+        const messages = warn.mock.calls.map((c) => String(c[0]));
+        expect(messages.some((m) => m.includes("docs/quickstart.mdx"))).toBe(
+          true,
+        );
+        // The reason must distinguish "the file was empty" from "we stripped
+        // it to empty" — they have completely different fixes.
+        expect(messages.some((m) => m.includes("MDX stripping"))).toBe(true);
+      } finally {
+        warn.mockRestore();
+      }
+    });
+
+    it("does not warn for a file that chunks normally", () => {
+      const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+      try {
+        const chunks = chunkMarkdown(
+          "# Title\n\nSome real prose.\n",
+          "docs/real.mdx",
+          mkConfig(),
+        );
+        expect(chunks.length).toBeGreaterThan(0);
+        expect(warn).not.toHaveBeenCalled();
+      } finally {
+        warn.mockRestore();
+      }
+    });
+  });
 });

src/__tests__/pipeline.test.ts

@@ -123,6 +123,51 @@ describe("IndexingPipeline", () => {
     expect(embeddingClient.embedBatch).not.toHaveBeenCalled();
   });
 
+  it("warns, naming the item, when an item produces zero chunks", async () => {
+    // Dropping an item from the index without a word is what let 130 stub
+    // pages vanish for months. The pipeline must say which item it dropped and
+    // that nothing was written, so the drop is greppable in the logs the way
+    // next_acquire_reason and quarantined_items are.
+    const { getChunker } = await import("../indexing/chunking/index.js");
+    vi.mocked(getChunker).mockReturnValueOnce(() => []);
+
+    const embeddingClient = new EmbeddingClient("key", "model", 1536);
+    const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    try {
+      await pipeline.indexItems(
+        [{ id: "stub.mdx", content: "<Foo />" }],
+        "abc",
+      );
+
+      const messages = warn.mock.calls.map((c) => String(c[0]));
+      expect(messages.some((m) => m.includes("stub.mdx"))).toBe(true);
+      expect(messages.some((m) => m.includes("[pipeline:test-source]"))).toBe(
+        true,
+      );
+      expect(messages.some((m) => m.includes("zero chunks"))).toBe(true);
+    } finally {
+      warn.mockRestore();
+    }
+  });
+
+  it("does not warn for an item that produces chunks", async () => {
+    const embeddingClient = new EmbeddingClient("key", "model", 1536);
+    const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    try {
+      await pipeline.indexItems(
+        [{ id: "real.md", content: "# Real\n\nprose" }],
+        "abc",
+      );
+      expect(warn).not.toHaveBeenCalled();
+    } finally {
+      warn.mockRestore();
+    }
+  });
+
   it("removes items by ID", async () => {
     const embeddingClient = new EmbeddingClient("key", "model", 1536);
     const pipeline = new IndexingPipeline(embeddingClient, testConfig);

src/__tests__/reindex-audit-shortfall.test.ts

@@ -0,0 +1,203 @@
+/**
+ * End-to-end proof for the post-reindex shortfall audit.
+ *
+ * Models the real failure that ran undetected for months: a `.mdx` page whose
+ * prose lives entirely in an excluded snippet, so the file is walked, matched,
+ * read, stripped to nothing, and chunks to ZERO. The chunker returns `[]`, the
+ * pipeline writes nothing, and the file silently vanishes from the index.
+ *
+ * The chunker and the pipeline here are REAL — only the database and the disk
+ * walk are faked — so the test exercises the actual drop path rather than a
+ * hand-built "db is smaller" fixture.
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+const {
+  mockGetConfig,
+  mockGetServerConfig,
+  mockWalkSourceFiles,
+  indexedChunks,
+} = vi.hoisted(() => ({
+  mockGetConfig: vi.fn(),
+  mockGetServerConfig: vi.fn(),
+  mockWalkSourceFiles: vi.fn(),
+  // source_name → file_path → chunk count. Stands in for the chunks table.
+  indexedChunks: new Map<string, Map<string, number>>(),
+}));
+
+function chunkTable(source: string): Map<string, number> {
+  let t = indexedChunks.get(source);
+  if (!t) {
+    t = new Map();
+    indexedChunks.set(source, t);
+  }
+  return t;
+}
+
+vi.mock("../config.js", () => ({
+  getConfig: (...args: unknown[]) => mockGetConfig(...args),
+  getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args),
+}));
+
+vi.mock("../indexing/utils.js", () => ({
+  walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args),
+}));
+
+// A tiny in-memory stand-in for the chunks table, shared by the pipeline
+// (writer) and the audit (reader) so the audit sees exactly what the pipeline
+// actually persisted.
+vi.mock("../db/queries.js", () => ({
+  replaceChunksForFile: async (
+    source: string,
+    filePath: string,
+    chunks: unknown[],
+  ) => {
+    const t = chunkTable(source);
+    // Mirrors the real delete+insert: an empty array deletes the row and
+    // inserts nothing, so the file disappears from `SELECT DISTINCT file_path`.
+    if (chunks.length === 0) t.delete(filePath);
+    else t.set(filePath, chunks.length);
+  },
+  deleteChunksByFile: async (source: string, filePath: string) => {
+    chunkTable(source).delete(filePath);
+  },
+  getIndexedItemIds: async (source: string) =>
+    new Set(chunkTable(source).keys()),
+}));
+
+import { IndexingPipeline } from "../indexing/pipeline.js";
+import { runReindexAudit, resetAuditCache } from "../indexing/reindex-audit.js";
+import type { SourceConfig } from "../types.js";
+import type { ContentItem } from "../indexing/providers/types.js";
+import type { EmbeddingProvider } from "../indexing/embeddings.js";
+
+const sourceConfig: SourceConfig = {
+  name: "docs",
+  type: "markdown",
+  path: "/repo/docs",
+  file_patterns: ["**/*.mdx"],
+  chunk: {},
+};
+
+function appConfig() {
+  return {
+    databaseUrl: "postgresql://test",
+    openaiApiKey: "test-key",
+    githubToken: "",
+    cloneDir: "/tmp/test",
+    slackWebhookUrl: "",
+  };
+}
+
+/** A prose page: real markdown, chunks to something. */
+function prosePage(n: number): string {
+  return `---\ntitle: Page ${n}\n---\n\n# Page ${n}\n\nReal prose for page ${n}.\n`;
+}
+
+/**
+ * A pure-JSX stub: the visible prose lives in an excluded snippet component, so
+ * after MDX stripping there is nothing left. This is the 130-file case.
+ */
+function jsxStubPage(n: number): string {
+  return `---\ntitle: Stub ${n}\n---\n\n<Snippet file="shared/intro.mdx" />\n<ComponentDemo name="demo-${n}" />\n`;
+}
+
+const embeddingProvider: EmbeddingProvider = {
+  embed: async () => [0.1, 0.2, 0.3],
+  embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]),
+};
+
+/**
+ * Index 40 pages, `stubEvery`-th of which is a pure-JSX stub. Returns the disk
+ * file set. `config` lets a test exercise a per-source tolerance override.
+ */
+async function indexCorpus(
+  stubEvery: number,
+  config: SourceConfig = sourceConfig,
+): Promise<Set<string>> {
+  const pipeline = new IndexingPipeline(embeddingProvider, config);
+  const items: ContentItem[] = [];
+  const disk = new Set<string>();
+  for (let n = 0; n < 40; n++) {
+    const isStub = n % stubEvery === 0;
+    const id = `page-${n}.mdx`;
+    disk.add(id);
+    items.push({ id, content: isStub ? jsxStubPage(n) : prosePage(n) });
+  }
+  const { failedIds } = await pipeline.indexItems(items, "sha-1");
+  expect(failedIds).toEqual([]);
+  return disk;
+}
+
+describe("post-reindex shortfall audit (zero-chunk files)", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    indexedChunks.clear();
+    resetAuditCache();
+    mockGetConfig.mockReturnValue(appConfig());
+    mockGetServerConfig.mockReturnValue({ sources: [sourceConfig] });
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true }));
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it("reports a db_has_fewer finding when pure-JSX stubs chunk to zero", async () => {
+    const disk = await indexCorpus(5); // 8 of 40 → a 20% shortfall
+    mockWalkSourceFiles.mockResolvedValue(disk);
+
+    // Precondition: the pipeline really did drop the stubs (the bug's mechanism).
+    const indexed = chunkTable("docs");
+    expect(disk.size).toBe(40);
+    expect(indexed.size).toBe(32);
+
+    const findings = await runReindexAudit(["docs"]);
+
+    const shortfall = findings.find(
+      (f) => f.check === "count_divergence" && f.direction === "db_has_fewer",
+    );
+    expect(shortfall).toBeDefined();
+    expect(shortfall!.source).toBe("docs");
+    expect(shortfall!.count).toBe(8);
+    // The finding names the files, so the operator can open one and see why.
+    expect(shortfall!.samples).toEqual(
+      expect.arrayContaining(["page-0.mdx", "page-5.mdx"]),
+    );
+    expect(shortfall!.samples.length).toBeLessThanOrEqual(10);
+  });
+
+  // ── Negative assertion: the new check must not become noise ─────────────
+  //
+  // Every source drops SOME files legitimately. If a normal reindex of a
+  // healthy source emits a finding, operators mute the audit and the next real
+  // shrink goes unseen — which is how the shortfall direction got suppressed in
+  // the first place. A healthy source must stay silent.
+  it("does NOT report a shortfall for a source skipping a normal share of files", async () => {
+    // 1 empty page out of 40 → 2.5%, inside the 5% default tolerance.
+    const disk = await indexCorpus(40);
+    mockWalkSourceFiles.mockResolvedValue(disk);
+    expect(chunkTable("docs").size).toBe(39);
+
+    const findings = await runReindexAudit(["docs"]);
+
+    expect(findings).toEqual([]);
+  });
+
+  it("reports that same small shortfall once the source sets unindexed_tolerance: 0", async () => {
+    // A source known to index everything it walks opts into a zero baseline,
+    // and the audit then flags the very first regression.
+    const strict = { ...sourceConfig, unindexed_tolerance: 0 };
+    mockGetServerConfig.mockReturnValue({ sources: [strict] });
+
+    const disk = await indexCorpus(40, strict);
+    mockWalkSourceFiles.mockResolvedValue(disk);
+
+    const findings = await runReindexAudit(["docs"]);
+
+    const shortfall = findings.find((f) => f.direction === "db_has_fewer");
+    expect(shortfall).toBeDefined();
+    expect(shortfall!.count).toBe(1);
+    expect(shortfall!.samples).toEqual(["page-0.mdx"]);
+  });
+});