diff --git a/docs/config/index.html b/docs/config/index.html index a8ab471..420339b 100644 --- a/docs/config/index.html +++ b/docs/config/index.html @@ -722,6 +722,12 @@
count_divergence finding (direction
+ db_has_fewer) naming the files the index is missing.
+ Some shortfall is normal — a file with no extractable prose is
+ walked and then dropped — so the finding only fires above this share
+ of the source's walked files (default 0.05, with a
+ floor of 3 files so small sources stay quiet). Set it to
+ 0 once a source is known to index everything it walks,
+ and the audit will flag the very first regression. It does not
+ affect what gets indexed, so changing it does not trigger a reindex.
+ markdown splits on headings and uses token-based
diff --git a/src/__tests__/markdown-chunker.test.ts b/src/__tests__/markdown-chunker.test.ts
index e04d503..188d4c4 100644
--- a/src/__tests__/markdown-chunker.test.ts
+++ b/src/__tests__/markdown-chunker.test.ts
@@ -1,4 +1,4 @@
-import { describe, it, expect, afterEach } from "vitest";
+import { describe, it, expect, afterEach, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -3115,4 +3115,69 @@ describe("chunkMarkdown inlined-snippet byte normalization", () => {
expect(/[\u{E000}-\u{E003}]/u.test(chunk.content)).toBe(false);
}
});
+ // ── Zero-chunk files must be audible ────────────────────────────────
+ //
+ // A file that chunks to nothing is dropped from the index. Silently doing
+ // that is how 130 pure-JSX stub pages — 19% of a source — stayed missing for
+ // months. Every path that returns [] has to say which file and why.
+
+ describe("zero-chunk warnings", () => {
+ it("warns, naming the file, when the content is empty", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ expect(chunkMarkdown(" \n\n ", "docs/blank.mdx", mkConfig())).toEqual(
+ [],
+ );
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining("docs/blank.mdx"),
+ );
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining("no chunks"));
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ it("warns, naming the file and the reason, when MDX stripping empties a pure-JSX stub", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ const stub = [
+ "---",
+ "title: Quickstart",
+ "---",
+ "",
+ '