From 052eec65c3acf3ca7272dd840c3119ce58910769 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 20:42:41 -0700 Subject: [PATCH 1/3] Add failing test for content no configured source claims --- src/__tests__/reindex-audit-unclaimed.test.ts | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 src/__tests__/reindex-audit-unclaimed.test.ts diff --git a/src/__tests__/reindex-audit-unclaimed.test.ts b/src/__tests__/reindex-audit-unclaimed.test.ts new file mode 100644 index 0000000..79d87cf --- /dev/null +++ b/src/__tests__/reindex-audit-unclaimed.test.ts @@ -0,0 +1,284 @@ +/** + * End-to-end proof for the unclaimed-content audit. + * + * Reconstructs the real blind spot: the CopilotKit API reference — 184 `.mdx` + * files under `showcase/shell-docs/src/content/reference/` — was invisible for + * months because the docs source's `path` was the SIBLING directory + * `showcase/shell-docs/src/content/docs/`. Every existing audit check compares + * disk against index, but "disk" is enumerated by walking the path the config + * points at, so both halves were blind identically and agreed perfectly. + * + * The disk here is REAL — a temp-dir repo fixture laid out like the clone the + * indexer walks — and `walkSourceFiles` is the real implementation, so the + * test exercises the actual enumeration path rather than a hand-built + * "these files are missing" fixture. Only the app config and the database are + * faked. + */ +import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const { mockGetConfig, mockGetServerConfig, mockGetIndexedItemIds } = vi.hoisted( + () => ({ + mockGetConfig: vi.fn(), + mockGetServerConfig: vi.fn(), + mockGetIndexedItemIds: vi.fn(), + }), +); + +vi.mock("../config.js", () => ({ + getConfig: (...args: unknown[]) => mockGetConfig(...args), + getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args), +})); + +vi.mock("../db/queries.js", () => ({ + getIndexedItemIds: (...args: unknown[]) => mockGetIndexedItemIds(...args), +})); + +import { runReindexAudit, resetAuditCache } from "../indexing/reindex-audit.js"; +import { walkSourceFiles } from "../indexing/utils.js"; +import type { FileSourceConfig } from "../types.js"; + +// --------------------------------------------------------------------------- +// Repo fixture — the historical layout, shrunk +// --------------------------------------------------------------------------- + +const CONTENT = "showcase/shell-docs/src/content"; +let tmpRoot: string; +let cloneDir: string; +let repoRoot: string; + +function write(rel: string, body = "# page\n\nprose\n"): void { + const full = path.join(repoRoot, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, body); +} + +function writeMany(dir: string, ext: string, n: number, prefix = "page"): void { + for (let i = 0; i < n; i++) write(`${dir}/${prefix}-${i}${ext}`); +} + +beforeAll(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "pf-unclaimed-")); + cloneDir = path.join(tmpRoot, "clones"); + // repoNameFromUrl("https://github.com/acme/shell.git") -> "shell" + repoRoot = path.join(cloneDir, "shell"); + + // Claimed: the prose docs tree the (mis-scoped) source walks. + writeMany(`${CONTENT}/docs`, ".mdx", 30); + writeMany(`${CONTENT}/docs/guides`, ".mdx", 20, "guide"); + + // UNCLAIMED, and the whole point: a large cohesive sibling of the claimed + // tree, same file type, rendered by the site but reachable by no source. + writeMany(`${CONTENT}/reference/hooks`, ".mdx", 20, "use"); + writeMany(`${CONTENT}/reference/components`, ".mdx", 15, "comp"); + + // UNCLAIMED but deliberately so: MDX partials inlined at render time. + // Big enough to cluster, and the case the opt-out exists for. + writeMany(`${CONTENT}/snippets`, ".mdx", 12, "snippet"); + + // Ordinary unclaimed content: a handful of scattered files. + writeMany(`${CONTENT}/misc`, ".mdx", 3, "note"); + + // A tests tree the code source explicitly excludes. + writeMany("packages/core/tests", ".ts", 40, "spec"); + // Claimed code, so .ts is an extension this repo indexes. + writeMany("packages/core/src", ".ts", 25, "mod"); + // Build output: a framework's generated bundle, same extension as claimed + // code. Must never be walked. + writeMany(`${CONTENT}/../.next/static`, ".ts", 50, "chunk"); +}); + +afterAll(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); +}); + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +const REPO = "https://github.com/acme/shell.git"; + +/** The docs source AS IT WAS: walk root pinned to the prose tree. */ +function docsSource(overrides: Partial = {}): FileSourceConfig { + return { + name: "docs", + type: "markdown", + repo: REPO, + path: `${CONTENT}/docs/`, + file_patterns: [`${CONTENT}/docs/**/*.mdx`], + chunk: {}, + ...overrides, + } as FileSourceConfig; +} + +function codeSource(): FileSourceConfig { + return { + name: "code", + type: "code", + repo: REPO, + path: ".", + file_patterns: ["**/*.ts"], + exclude_patterns: ["**/tests/**"], + chunk: {}, + } as FileSourceConfig; +} + +function appConfig() { + return { + databaseUrl: "postgresql://test", + openaiApiKey: "test-key", + githubToken: "", + cloneDir, + slackWebhookUrl: "", + }; +} + +/** + * Point the fake index at exactly what each source walks, so the stale / + * scope-leak / divergence checks all stay silent and any finding the test sees + * is the new one. + */ +function indexInSyncWith(sources: FileSourceConfig[]): void { + mockGetIndexedItemIds.mockImplementation(async (name: string) => { + const src = sources.find((s) => s.name === name)!; + return (await walkSourceFiles(src, cloneDir, "")) ?? new Set(); + }); +} + +function setSources(sources: FileSourceConfig[]): void { + mockGetServerConfig.mockReturnValue({ sources }); + indexInSyncWith(sources); +} + +beforeEach(() => { + vi.clearAllMocks(); + resetAuditCache(); + mockGetConfig.mockReturnValue(appConfig()); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); +}); + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("unclaimed-content audit", () => { + it("reports the reference tree no source's walk root can reach", async () => { + setSources([ + docsSource({ + unclaimed_exempt_paths: [`${CONTENT}/snippets`], + } as Partial), + codeSource(), + ]); + + const findings = await runReindexAudit(["docs", "code"]); + + const unclaimed = findings.filter((f) => f.check === "unclaimed_content"); + expect(unclaimed).toHaveLength(1); + expect(unclaimed[0].path).toBe(`${CONTENT}/reference`); + expect(unclaimed[0].extension).toBe(".mdx"); + expect(unclaimed[0].count).toBe(35); + expect(unclaimed[0].samples.length).toBeGreaterThan(0); + expect(unclaimed[0].samples.every((p) => p.includes("/reference/"))).toBe( + true, + ); + }); + + it("goes silent once the config is fixed to claim the reference tree", async () => { + setSources([ + docsSource({ + path: `${CONTENT}/`, + file_patterns: [ + `${CONTENT}/docs/**/*.mdx`, + `${CONTENT}/reference/**/*.mdx`, + ], + unclaimed_exempt_paths: [`${CONTENT}/snippets`], + } as Partial), + codeSource(), + ]); + + const findings = await runReindexAudit(["docs", "code"]); + + expect(findings.filter((f) => f.check === "unclaimed_content")).toEqual([]); + }); + + // ── Negative assertions ──────────────────────────────────────────────── + // + // Noise control IS the feature. A check that reports every unclaimed file + // gets muted, and a muted check is how the reference tree stayed invisible. + + it("reports a deliberately-excluded directory until it is opted out", async () => { + // Without the opt-out, snippets/ looks exactly like the reference tree. + setSources([docsSource(), codeSource()]); + const before = await runReindexAudit(["docs", "code"]); + expect( + before + .filter((f) => f.check === "unclaimed_content") + .map((f) => f.path) + .sort(), + ).toEqual([`${CONTENT}/reference`, `${CONTENT}/snippets`]); + + // The operator reviews it once, records it, and never sees it again. + resetAuditCache(); + setSources([ + docsSource({ + unclaimed_exempt_paths: [`${CONTENT}/snippets`], + } as Partial), + codeSource(), + ]); + const after = await runReindexAudit(["docs", "code"]); + expect( + after.filter((f) => f.check === "unclaimed_content").map((f) => f.path), + ).toEqual([`${CONTENT}/reference`]); + }); + + it("ignores a scattered handful of unclaimed files", async () => { + setSources([ + docsSource({ + path: `${CONTENT}/`, + file_patterns: [ + `${CONTENT}/docs/**/*.mdx`, + `${CONTENT}/reference/**/*.mdx`, + ], + unclaimed_exempt_paths: [`${CONTENT}/snippets`], + } as Partial), + codeSource(), + ]); + + const findings = await runReindexAudit(["docs", "code"]); + + // content/misc holds 3 unclaimed .mdx — real, unclaimed, and not worth + // an alert. Only a cohesive tree clears the bar. + expect( + findings.some((f) => f.path === `${CONTENT}/misc`), + ).toBe(false); + }); + + it("ignores an explicitly excluded tests tree and generated build output", async () => { + setSources([ + docsSource({ + path: `${CONTENT}/`, + file_patterns: [ + `${CONTENT}/docs/**/*.mdx`, + `${CONTENT}/reference/**/*.mdx`, + ], + unclaimed_exempt_paths: [`${CONTENT}/snippets`], + } as Partial), + codeSource(), + ]); + + const findings = await runReindexAudit(["docs", "code"]); + const paths = findings + .filter((f) => f.check === "unclaimed_content") + .map((f) => f.path); + + // 40 .ts files, an extension the code source indexes, in a big cohesive + // directory — silent ONLY because the source's exclude_patterns already + // say so. An explicit exclusion IS an operator decision on record. + expect(paths).not.toContain("packages/core/tests"); + // 50 .ts files of generated output, never walked at all. + expect(paths.some((p) => p?.includes(".next"))).toBe(false); + expect(findings.filter((f) => f.check === "unclaimed_content")).toEqual([]); + }); +}); From 2037b8a542f2c5aade75231485d44ad97855f1fb Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 20:49:29 -0700 Subject: [PATCH 2/3] Detect repository content that no configured source claims --- src/__tests__/reindex-audit-shortfall.test.ts | 6 +- src/__tests__/reindex-audit-unclaimed.test.ts | 80 +++-- src/__tests__/reindex-audit.test.ts | 6 +- src/indexing/reindex-audit.ts | 82 ++++- src/indexing/unclaimed-audit.ts | 327 ++++++++++++++++++ src/types.ts | 8 + 6 files changed, 480 insertions(+), 29 deletions(-) create mode 100644 src/indexing/unclaimed-audit.ts diff --git a/src/__tests__/reindex-audit-shortfall.test.ts b/src/__tests__/reindex-audit-shortfall.test.ts index d39de26..6bb4c78 100644 --- a/src/__tests__/reindex-audit-shortfall.test.ts +++ b/src/__tests__/reindex-audit-shortfall.test.ts @@ -39,7 +39,11 @@ vi.mock("../config.js", () => ({ getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args), })); -vi.mock("../indexing/utils.js", () => ({ +// Only walkSourceFiles is faked. The rest of the module stays real because +// the unclaimed-content check (src/indexing/unclaimed-audit.ts) matches +// patterns through it, and a module-shaped hole there would fail silently. +vi.mock("../indexing/utils.js", async (importOriginal) => ({ + ...(await importOriginal()), walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args), })); diff --git a/src/__tests__/reindex-audit-unclaimed.test.ts b/src/__tests__/reindex-audit-unclaimed.test.ts index 79d87cf..357e524 100644 --- a/src/__tests__/reindex-audit-unclaimed.test.ts +++ b/src/__tests__/reindex-audit-unclaimed.test.ts @@ -14,18 +14,25 @@ * "these files are missing" fixture. Only the app config and the database are * faked. */ -import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from "vitest"; +import { + describe, + it, + expect, + vi, + beforeAll, + beforeEach, + afterAll, +} from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const { mockGetConfig, mockGetServerConfig, mockGetIndexedItemIds } = vi.hoisted( - () => ({ +const { mockGetConfig, mockGetServerConfig, mockGetIndexedItemIds } = + vi.hoisted(() => ({ mockGetConfig: vi.fn(), mockGetServerConfig: vi.fn(), mockGetIndexedItemIds: vi.fn(), - }), -); + })); vi.mock("../config.js", () => ({ getConfig: (...args: unknown[]) => mockGetConfig(...args), @@ -81,13 +88,19 @@ beforeAll(() => { // Ordinary unclaimed content: a handful of scattered files. writeMany(`${CONTENT}/misc`, ".mdx", 3, "note"); - // A tests tree the code source explicitly excludes. - writeMany("packages/core/tests", ".ts", 40, "spec"); - // Claimed code, so .ts is an extension this repo indexes. + // Claimed code, so `.ts` is an extension this repo indexes. writeMany("packages/core/src", ".ts", 25, "mod"); - // Build output: a framework's generated bundle, same extension as claimed - // code. Must never be walked. - writeMany(`${CONTENT}/../.next/static`, ".ts", 50, "chunk"); + // A tests tree the code source explicitly excludes: its include pattern + // reaches these files and its exclude pattern vetoes them. + writeMany("packages/core/tests", ".ts", 40, "spec"); + + // Claimed Python, so `.py` is also an extension this repo indexes — and + // the two trees below are `.py` that NO include pattern reaches, sitting + // where the parent (the repo root) does publish `.py`. Nothing but the + // generated-directory skip keeps them quiet. + writeMany("sdks/python/agent", ".py", 20, "mod"); + writeMany(".venv/lib/deps/requests", ".py", 60, "dep"); + writeMany("build/lib/agent", ".py", 60, "out"); }); afterAll(() => { @@ -101,7 +114,9 @@ afterAll(() => { const REPO = "https://github.com/acme/shell.git"; /** The docs source AS IT WAS: walk root pinned to the prose tree. */ -function docsSource(overrides: Partial = {}): FileSourceConfig { +function docsSource( + overrides: Partial = {}, +): FileSourceConfig { return { name: "docs", type: "markdown", @@ -119,12 +134,24 @@ function codeSource(): FileSourceConfig { type: "code", repo: REPO, path: ".", - file_patterns: ["**/*.ts"], + file_patterns: ["packages/**/*.ts"], exclude_patterns: ["**/tests/**"], chunk: {}, } as FileSourceConfig; } +/** A second code source, scoped the way the real ag-ui-code source is. */ +function pythonSource(): FileSourceConfig { + return { + name: "python", + type: "code", + repo: REPO, + path: ".", + file_patterns: ["sdks/python/**/*.py"], + chunk: {}, + } as FileSourceConfig; +} + function appConfig() { return { databaseUrl: "postgresql://test", @@ -170,9 +197,10 @@ describe("unclaimed-content audit", () => { unclaimed_exempt_paths: [`${CONTENT}/snippets`], } as Partial), codeSource(), + pythonSource(), ]); - const findings = await runReindexAudit(["docs", "code"]); + const findings = await runReindexAudit(["docs", "code", "python"]); const unclaimed = findings.filter((f) => f.check === "unclaimed_content"); expect(unclaimed).toHaveLength(1); @@ -196,9 +224,10 @@ describe("unclaimed-content audit", () => { unclaimed_exempt_paths: [`${CONTENT}/snippets`], } as Partial), codeSource(), + pythonSource(), ]); - const findings = await runReindexAudit(["docs", "code"]); + const findings = await runReindexAudit(["docs", "code", "python"]); expect(findings.filter((f) => f.check === "unclaimed_content")).toEqual([]); }); @@ -211,7 +240,7 @@ describe("unclaimed-content audit", () => { it("reports a deliberately-excluded directory until it is opted out", async () => { // Without the opt-out, snippets/ looks exactly like the reference tree. setSources([docsSource(), codeSource()]); - const before = await runReindexAudit(["docs", "code"]); + const before = await runReindexAudit(["docs", "code", "python"]); expect( before .filter((f) => f.check === "unclaimed_content") @@ -226,8 +255,9 @@ describe("unclaimed-content audit", () => { unclaimed_exempt_paths: [`${CONTENT}/snippets`], } as Partial), codeSource(), + pythonSource(), ]); - const after = await runReindexAudit(["docs", "code"]); + const after = await runReindexAudit(["docs", "code", "python"]); expect( after.filter((f) => f.check === "unclaimed_content").map((f) => f.path), ).toEqual([`${CONTENT}/reference`]); @@ -244,15 +274,14 @@ describe("unclaimed-content audit", () => { unclaimed_exempt_paths: [`${CONTENT}/snippets`], } as Partial), codeSource(), + pythonSource(), ]); - const findings = await runReindexAudit(["docs", "code"]); + const findings = await runReindexAudit(["docs", "code", "python"]); // content/misc holds 3 unclaimed .mdx — real, unclaimed, and not worth // an alert. Only a cohesive tree clears the bar. - expect( - findings.some((f) => f.path === `${CONTENT}/misc`), - ).toBe(false); + expect(findings.some((f) => f.path === `${CONTENT}/misc`)).toBe(false); }); it("ignores an explicitly excluded tests tree and generated build output", async () => { @@ -266,9 +295,10 @@ describe("unclaimed-content audit", () => { unclaimed_exempt_paths: [`${CONTENT}/snippets`], } as Partial), codeSource(), + pythonSource(), ]); - const findings = await runReindexAudit(["docs", "code"]); + const findings = await runReindexAudit(["docs", "code", "python"]); const paths = findings .filter((f) => f.check === "unclaimed_content") .map((f) => f.path); @@ -277,8 +307,10 @@ describe("unclaimed-content audit", () => { // directory — silent ONLY because the source's exclude_patterns already // say so. An explicit exclusion IS an operator decision on record. expect(paths).not.toContain("packages/core/tests"); - // 50 .ts files of generated output, never walked at all. - expect(paths.some((p) => p?.includes(".next"))).toBe(false); + // 120 `.py` files that no include pattern reaches, in a repo that does + // publish `.py` — silent only because neither directory is ever walked. + expect(paths).not.toContain(".venv"); + expect(paths).not.toContain("build"); expect(findings.filter((f) => f.check === "unclaimed_content")).toEqual([]); }); }); diff --git a/src/__tests__/reindex-audit.test.ts b/src/__tests__/reindex-audit.test.ts index 0c3a29c..0c4a0fc 100644 --- a/src/__tests__/reindex-audit.test.ts +++ b/src/__tests__/reindex-audit.test.ts @@ -29,7 +29,11 @@ vi.mock("../db/queries.js", () => ({ getIndexedItemIds: (...args: unknown[]) => mockGetIndexedItemIds(...args), })); -vi.mock("../indexing/utils.js", () => ({ +// Only walkSourceFiles is faked. The rest of the module stays real because +// the unclaimed-content check (src/indexing/unclaimed-audit.ts) matches +// patterns through it, and a module-shaped hole there would fail silently. +vi.mock("../indexing/utils.js", async (importOriginal) => ({ + ...(await importOriginal()), walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args), })); diff --git a/src/indexing/reindex-audit.ts b/src/indexing/reindex-audit.ts index dc4201d..1b9bb0b 100644 --- a/src/indexing/reindex-audit.ts +++ b/src/indexing/reindex-audit.ts @@ -1,6 +1,11 @@ import { getConfig, getServerConfig } from "../config.js"; import { getIndexedItemIds } from "../db/queries.js"; import { walkSourceFiles } from "./utils.js"; +import { + findUnclaimedClusters, + groupSourcesByRepoRoot, + repoRootFor, +} from "./unclaimed-audit.js"; import { isFileSourceConfig } from "../types.js"; import type { FileSourceConfig } from "../types.js"; @@ -43,10 +48,15 @@ export function resetAuditCache(): void { export interface AuditFinding { source: string; - check: "stale_files" | "scope_leak" | "count_divergence"; + check: + "stale_files" | "scope_leak" | "count_divergence" | "unclaimed_content"; count: number; samples: string[]; direction?: "db_has_more" | "db_has_fewer"; + /** unclaimed_content: repo-root-relative directory holding the files. */ + path?: string; + /** unclaimed_content: the file extension the cluster is made of. */ + extension?: string; } export async function runReindexAudit( @@ -144,11 +154,65 @@ export async function runReindexAudit( } } + // Check 4 — Unclaimed content: files the repository holds that NO + // source's walk root and patterns would ever reach. + // + // Repo-scoped, not source-scoped, and deliberately so: checks 1-3 all + // compare disk against index using the config's own walk root, so a walk + // root pointed at the wrong subtree makes both sides agree perfectly on an + // incomplete corpus. Only a comparison anchored to the repository can see + // that. + // + // Claims are computed from EVERY configured source on the repo, not just + // the audited ones — a file claimed by a source that did not happen to + // reindex is still claimed. + const repoGroups = groupSourcesByRepoRoot( + serverCfg.sources.filter(isFileSourceConfig), + cfg.cloneDir, + ); + const scannedRoots = new Set(); + for (const sourceConfig of fileSources) { + const repoRoot = repoRootFor(sourceConfig, cfg.cloneDir); + if (scannedRoots.has(repoRoot)) continue; + scannedRoots.add(repoRoot); + try { + const clusters = await findUnclaimedClusters( + repoRoot, + repoGroups.get(repoRoot) ?? [sourceConfig], + ); + for (const cluster of clusters) { + findings.push({ + // Attributed to the first AUDITED source on this repo so the + // dedup cache, which is cleared per audited source name, can + // retire the finding once it is resolved. + source: sourceConfig.name, + check: "unclaimed_content", + count: cluster.count, + samples: cluster.samples, + path: cluster.dir, + extension: cluster.extension, + }); + } + } catch (err) { + console.warn( + `[reindex-audit] Unclaimed-content scan of ${repoRoot} failed:`, + err instanceof Error ? err.message : err, + ); + } + } + // Always log findings to console for (const f of findings) { - const detail = f.direction ? ` (${f.direction})` : ""; const samples = f.samples.length > 0 ? `: ${f.samples.slice(0, 5).join(", ")}` : ""; + if (f.check === "unclaimed_content") { + console.warn( + `[reindex-audit] ${f.source} — unclaimed_content: ${f.count} ${f.extension} files under ${f.path}/ that no configured source claims` + + ` (widen a source's path/file_patterns, or record it in unclaimed_exempt_paths if it is correctly unclaimed)${samples}`, + ); + continue; + } + const detail = f.direction ? ` (${f.direction})` : ""; console.warn( `[reindex-audit] ${f.source} — ${f.check}: ${f.count} issues${detail}${samples}`, ); @@ -188,7 +252,14 @@ export async function runReindexAudit( /** Dedup identity of a finding: source + check + direction. */ function dedupKey(f: AuditFinding): string { - return `${f.source}:${f.check}${f.direction ? `:${f.direction}` : ""}`; + // The directory is part of the identity: two unclaimed trees on one source + // are two findings, and keying on source+check alone would let one overwrite + // the other in the cache and silently suppress its alert. + return ( + `${f.source}:${f.check}` + + (f.direction ? `:${f.direction}` : "") + + (f.path ? `:${f.path}:${f.extension}` : "") + ); } async function sendSlackAlert( @@ -196,6 +267,11 @@ async function sendSlackAlert( webhookUrl: string, ): Promise { const lines = findings.map((f) => { + if (f.check === "unclaimed_content") { + let msg = `*${f.source}* — unclaimed_content: ${f.count} \`${f.extension}\` files under \`${f.path}/\` that no configured source claims`; + if (f.samples.length > 0) msg += `\n ${f.samples.join("\n ")}`; + return msg; + } let msg = `*${f.source}* — ${f.check}: ${f.count} issues`; if (f.direction) msg += ` (${f.direction})`; if (f.samples.length > 0) { diff --git a/src/indexing/unclaimed-audit.ts b/src/indexing/unclaimed-audit.ts new file mode 100644 index 0000000..1f2f939 --- /dev/null +++ b/src/indexing/unclaimed-audit.ts @@ -0,0 +1,327 @@ +// Detect content that exists in a repository but that NO configured source +// claims. +// +// Every other reindex-audit check compares what is on disk against what is in +// the index — but "what is on disk" is enumerated by walking the path the +// CONFIG points at, the same path the indexer used. When that walk root is +// wrong or too narrow, both halves of the comparison are blind identically and +// agree perfectly. +// +// That is not hypothetical. The CopilotKit API reference — 184 `.mdx` files +// under `showcase/shell-docs/src/content/reference/` — was invisible for +// months because the docs source's `path` was the SIBLING directory +// `showcase/shell-docs/src/content/docs/`. The indexer walked `docs/` and +// found 682 files, the audit walked `docs/` and found 682, the index held 682: +// a perfect match, with 183 live documentation pages missing from search. +// +// So this check is anchored to the REPOSITORY rather than to the config: it +// walks the whole repo and asks which indexable-looking files no source's walk +// root and patterns would ever reach. + +import fs from "node:fs"; +import path from "node:path"; +import { globToRegex, matchesPatterns } from "./utils.js"; +import type { FileSourceConfig } from "../types.js"; + +/** + * Minimum files of ONE extension in a single unclaimed directory tree before + * it is worth an operator's attention. + * + * Noise control is the whole problem here. Most of a repository is + * legitimately unclaimed, and a check that reports every unclaimed file is + * worthless — it gets muted, which is exactly how the original blindness + * arose. What distinguished the reference tree from ordinary unclaimed content + * was its SHAPE: a large cohesive directory of the same file type the source + * already indexes, sitting as a sibling of a claimed tree. This constant is the + * "large" half of that shape. + */ +export const MIN_UNCLAIMED_CLUSTER_FILES = 10; + +/** + * Directories that never hold authored content. Dot-directories are skipped + * wholesale (`.git`, `.next`, `.venv`, `.turbo`, `.pytest_cache`, …) since a + * dot-prefixed directory is tooling state, not published material. + */ +const GENERATED_DIRS = new Set([ + "node_modules", + "dist", + "build", + "out", + "coverage", + "target", + "vendor", + "venv", + "__pycache__", + "site-packages", + "storybook-static", +]); + +/** + * Ceiling on files examined per repository. A runaway walk must degrade the + * audit, not the server it runs inside. + */ +const MAX_WALKED_FILES = 250_000; + +export interface UnclaimedCluster { + /** Repo-root-relative directory holding the unclaimed files. */ + dir: string; + /** The file extension, including the dot. */ + extension: string; + count: number; + samples: string[]; +} + +/** Derive a repository directory name from a git URL. */ +function repoNameFromUrl(url: string): string { + return ( + url + .split("/") + .pop() + ?.replace(/\.git$/, "") ?? "repo" + ); +} + +/** + * The directory that file paths for this source are relative to: the clone + * root for a git source, the resolved path for a local one. + * + * A local source has no repository around it, so its own directory IS the + * whole visible world and this check can only see inside it. Git sources are + * where the blind spot lives, because there the config picks a subtree of + * something larger. + */ +export function repoRootFor( + source: FileSourceConfig, + cloneDir: string, +): string { + return source.repo + ? path.join(cloneDir, repoNameFromUrl(source.repo)) + : path.resolve(source.path); +} + +/** Group file sources by the repository root they read from. */ +export function groupSourcesByRepoRoot( + sources: FileSourceConfig[], + cloneDir: string, +): Map { + const groups = new Map(); + for (const source of sources) { + const root = repoRootFor(source, cloneDir); + const existing = groups.get(root); + if (existing) existing.push(source); + else groups.set(root, [source]); + } + return groups; +} + +/** Repo-root-relative prefix of a source's walk root ("" for the whole repo). */ +function walkRootPrefix(source: FileSourceConfig): string { + if (!source.repo) return ""; + const normalized = source.path + .replace(/\\/g, "/") + .replace(/^\.\//, "") + .replace(/\/+$/, ""); + return normalized === "" || normalized === "." ? "" : `${normalized}/`; +} + +/** Compiled include/exclude/exempt patterns, built once per source. */ +interface SourceMatcher { + prefix: string; + source: FileSourceConfig; + includes: RegExp[]; + excludes: RegExp[]; +} + +function compile(source: FileSourceConfig): SourceMatcher { + return { + prefix: walkRootPrefix(source), + source, + includes: source.file_patterns.map(globToRegex), + excludes: (source.exclude_patterns ?? []).map(globToRegex), + }; +} + +/** Does an exemption entry cover this repo-root-relative path? */ +function isExempt(rel: string, exemptions: string[]): boolean { + for (const raw of exemptions) { + const entry = raw + .replace(/\\/g, "/") + .replace(/^\.\//, "") + .replace(/\/+$/, ""); + if (entry === "" || entry === ".") return true; + if (rel === entry || rel.startsWith(`${entry}/`)) return true; + if (/[*?]/.test(entry) && globToRegex(entry).test(rel)) return true; + } + return false; +} + +type FileStatus = "claimed" | "accounted" | "unclaimed"; + +/** + * How a repository file relates to the configured sources. + * + * `accounted` is the load-bearing middle state: a file the source's own + * `exclude_patterns` knock out was reviewed by an operator and rejected on + * purpose, so it is not a blind spot. The exclusion only counts when one of + * that source's INCLUDE patterns would otherwise have taken the file — + * otherwise a code source whose globstar exclusion covers all of `showcase/` + * would silently vouch for every `.mdx` under `showcase/` too, and that is + * precisely the tree the reference pages were hiding in. + */ +function classify(rel: string, matchers: SourceMatcher[]): FileStatus { + let accounted = false; + for (const m of matchers) { + if (m.prefix && !rel.startsWith(m.prefix)) continue; + if (matchesPatterns(rel, m.source)) return "claimed"; + if ( + m.excludes.length > 0 && + m.includes.some((re) => re.test(rel)) && + m.excludes.some((re) => re.test(rel)) + ) { + accounted = true; + } + } + return accounted ? "accounted" : "unclaimed"; +} + +function parentDir(dir: string): string { + const i = dir.lastIndexOf("/"); + return i === -1 ? "" : dir.slice(0, i); +} + +interface DirStats { + claimed: number; + claimedByExt: Map; + candidateByExt: Map; +} + +function statsFor(map: Map, dir: string): DirStats { + let s = map.get(dir); + if (!s) { + s = { claimed: 0, claimedByExt: new Map(), candidateByExt: new Map() }; + map.set(dir, s); + } + return s; +} + +/** + * Find directory trees of indexable-looking files that no source claims. + * + * `sources` must be EVERY configured source reading from `repoRoot`, not just + * the ones that happened to reindex — a file claimed by a source that did not + * run is still claimed. + */ +export async function findUnclaimedClusters( + repoRoot: string, + sources: FileSourceConfig[], +): Promise { + if (sources.length === 0 || !fs.existsSync(repoRoot)) return []; + + const matchers = sources.map(compile); + const exemptions = sources.flatMap((s) => s.unclaimed_exempt_paths ?? []); + const skipDirs = new Set([ + ...GENERATED_DIRS, + ...sources.flatMap((s) => s.skip_dirs ?? []), + ]); + + const dirs = new Map(); + let walked = 0; + let truncated = false; + + async function walk(absDir: string, relDir: string): Promise { + if (truncated) return; + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(absDir, { withFileTypes: true }); + } catch (err) { + console.warn( + `[reindex-audit] Failed to read ${absDir}:`, + err instanceof Error ? err.message : err, + ); + return; + } + // Sorted so samples and finding order are stable across runs and hosts. + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + + for (const entry of entries) { + if (truncated) return; + const rel = relDir ? `${relDir}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + if (entry.name.startsWith(".") || skipDirs.has(entry.name)) continue; + await walk(path.join(absDir, entry.name), rel); + continue; + } + if (!entry.isFile()) continue; + if (++walked > MAX_WALKED_FILES) { + truncated = true; + console.warn( + `[reindex-audit] Unclaimed-content scan of ${repoRoot} hit the ${MAX_WALKED_FILES}-file ceiling; skipping`, + ); + return; + } + + const ext = path.extname(entry.name); + if (!ext) continue; + const status = classify(rel, matchers); + // An exempt file is one an operator already reviewed and signed off on; + // it is neither a claim nor a blind spot. + const candidate = + status === "unclaimed" && !isExempt(rel, exemptions) ? ext : null; + if (status !== "claimed" && !candidate) continue; + + for (let dir = relDir; ; dir = parentDir(dir)) { + const s = statsFor(dirs, dir); + if (status === "claimed") { + s.claimed++; + s.claimedByExt.set(ext, (s.claimedByExt.get(ext) ?? 0) + 1); + } else if (candidate) { + let c = s.candidateByExt.get(candidate); + if (!c) { + c = { count: 0, samples: [] }; + s.candidateByExt.set(candidate, c); + } + c.count++; + if (c.samples.length < 10) c.samples.push(rel); + } + if (dir === "") break; + } + } + } + + await walk(repoRoot, ""); + if (truncated) return []; + + // Roll up to the MAXIMAL unclaimed subtree: the highest directory holding no + // claimed file at all whose parent DOES hold one. `content/reference/` rolls + // up as a single 184-file finding rather than fragmenting into + // `reference/hooks/`, `reference/components/`, and a dozen more — each too + // small to clear the bar and far too many to read. + const clusters: UnclaimedCluster[] = []; + for (const [dir, stats] of dirs) { + if (dir === "" || stats.claimed > 0) continue; + const parent = dirs.get(parentDir(dir)); + if (!parent || parent.claimed === 0) continue; + + for (const [ext, c] of stats.candidateByExt) { + if (c.count < MIN_UNCLAIMED_CLUSTER_FILES) continue; + // The extension must be one this repository already publishes THROUGH + // THE SAME PARENT — the reference tree's signature. A repo's stray + // `.ts` scripts sitting beside an indexed docs tree are not a + // documentation gap, and saying so here is what keeps the check quiet. + if ((parent.claimedByExt.get(ext) ?? 0) === 0) continue; + clusters.push({ + dir, + extension: ext, + count: c.count, + samples: c.samples, + }); + } + } + + clusters.sort( + (a, b) => + a.dir.localeCompare(b.dir) || a.extension.localeCompare(b.extension), + ); + return clusters; +} diff --git a/src/types.ts b/src/types.ts index feac3b0..c7e1e2f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -138,6 +138,14 @@ export const FileSourceConfigSchema = z.object({ // source is known to index every file, and the audit will then flag the very // first regression. Defaults to 0.05. unindexed_tolerance: z.number().min(0).max(1).optional(), + // Repo-root-relative directories (prefixes or globs) that this source's + // operator has reviewed and confirmed are CORRECTLY unclaimed, so the + // unclaimed-content audit stays quiet about them (see + // src/indexing/unclaimed-audit.ts, Check 4). Entries are pooled across every + // source reading the same repository, since any one of their operators can + // vouch for a subtree. `snippets/` — MDX partials inlined at render time — + // is the archetype: real content, real file type, deliberately not a page. + unclaimed_exempt_paths: z.array(z.string()).optional(), }); // Slack source schema — different required fields From 0e2514bd8a00b8079186475aa2de16a825a211a9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Thu, 10 Sep 2026 20:53:46 -0700 Subject: [PATCH 3/3] Record the trees the new check fires on that are correctly unclaimed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the check against the real production configs over fresh checkouts of CopilotKit/CopilotKit, ag-ui-protocol/ag-ui, CopilotKit/aimock and CopilotKit/pathfinder. Nine clusters; seven are content nobody ever meant to index (MDX partials inlined at render time, example-app READMEs, per-integration setup notes, the dojo demo app, a cross-language test harness, repo maintenance scripts) and are now recorded as exempt. The two left unexempted are real. showcase/shell-docs/src/content/ag-ui/ is 96 .mdx pages served live at docs.copilotkit.ai/ag-ui/ — all 96 are in the production sitemap and no source indexes one of them, which is the reference-docs failure again, still open. ag-ui middlewares/ is six published @ag-ui/* packages whose source no search tool can reach. --- deploy/aimock-docs.yaml | 5 +++++ deploy/copilotkit-docs.yaml | 28 ++++++++++++++++++++++++++++ docs/config/index.html | 26 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/deploy/aimock-docs.yaml b/deploy/aimock-docs.yaml index 2b345d8..b49cf5c 100644 --- a/deploy/aimock-docs.yaml +++ b/deploy/aimock-docs.yaml @@ -36,6 +36,11 @@ sources: - "**/__tests__/**" - "**/*.test.*" - "**/*.spec.*" + # Correctly unclaimed (unclaimed-content audit, reindex-audit Check 4): + # scripts/ holds repo maintenance tooling (changelog radar, converters), + # not library code anyone searches for. + unclaimed_exempt_paths: + - "scripts" skip_dirs: - node_modules - dist diff --git a/deploy/copilotkit-docs.yaml b/deploy/copilotkit-docs.yaml index ad8f228..ae6d13f 100644 --- a/deploy/copilotkit-docs.yaml +++ b/deploy/copilotkit-docs.yaml @@ -42,6 +42,25 @@ sources: file_patterns: - "showcase/shell-docs/src/content/docs/**/*.mdx" - "showcase/shell-docs/src/content/reference/**/*.mdx" + # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded + # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about + # them. Pooled across every source reading CopilotKit/CopilotKit. + # snippets/ — MDX partials inlined into pages at render time; they are + # never standalone pages and their prose is already indexed + # through the pages that include them. + # examples/ — per-example READMEs for the demo apps, not on the site. + # showcase/integrations/ — per-integration setup notes that live with + # the integration app, not on docs.copilotkit.ai. + # NOT listed, deliberately: showcase/shell-docs/src/content/ag-ui/. Those + # 96 files ARE live pages (docs.copilotkit.ai/ag-ui/, all 96 in the + # sitemap) served by src/app/ag-ui/[[...slug]] — the same shape as the + # reference tree this check was built for. The audit should keep saying so + # until someone decides whether to index them here or leave them to + # docs.ag-ui.com. + unclaimed_exempt_paths: + - "showcase/shell-docs/src/content/snippets" + - "examples" + - "showcase/integrations" chunk: target_tokens: 600 overlap_tokens: 50 @@ -136,6 +155,15 @@ sources: # hand-written signal - "**/generated/**" - "**/*.pb.*" + # Correctly unclaimed in ag-ui-protocol/ag-ui (see Check 4 above): + # apps/ — the dojo demo app and its READMEs; demo code, not SDK. + # sdks/dotnet/ — the only .ts under it is the cross-language Vitest + # harness that drives the .NET server. + # NOT listed: middlewares/. Those are six published @ag-ui/* packages + # whose source no search tool can reach today. + unclaimed_exempt_paths: + - "apps" + - "sdks/dotnet" skip_dirs: - node_modules - dist diff --git a/docs/config/index.html b/docs/config/index.html index 420339b..2a6f4b2 100644 --- a/docs/config/index.html +++ b/docs/config/index.html @@ -728,6 +728,12 @@

sources

># Share of walked files that may be missing from the index before the post-reindex audit reports a shortfall (optional) + unclaimed_exempt_paths: + # Repo-relative directories confirmed to be correctly unindexed + (optional) + - "docs/snippets" category: faq # Optional. Marks content as FAQ for /faq.txt and knowledge @@ -773,6 +779,26 @@

sources

and the audit will flag the very first regression. It does not affect what gets indexed, so changing it does not trigger a reindex. +
  • + unclaimed_exempt_paths — The shortfall check above + can only compare the index against the files the + configured walk root reaches, so a walk root pointed at the + wrong subtree leaves both halves blind in the same way and they + agree perfectly. A separate audit therefore walks the whole + repository and reports an unclaimed_content finding for + any directory tree of at least 10 files sharing one extension that + no source's path and file_patterns would ever reach — + when that extension is one the repository already indexes through + the tree's parent. Files a source's own + exclude_patterns reject are treated as accounted for, + since an exclusion is a decision on record. List here the + repo-root-relative directories (prefixes or globs) an operator has + reviewed and confirmed are correctly unclaimed — partials inlined at + render time, example apps, maintenance scripts — and the audit stops + reporting them. Entries are pooled across every source reading the + same repository. It does not affect what gets indexed, so changing + it does not trigger a reindex. +
  • type — Determines chunking strategy. markdown splits on headings and uses token-based