Detect repository content that no configured source claims

deploy/aimock-docs.yaml

@@ -36,6 +36,11 @@ sources:
       - "**/__tests__/**"
       - "**/*.test.*"
       - "**/*.spec.*"
+    # Correctly unclaimed (unclaimed-content audit, reindex-audit Check 4):
+    # scripts/ holds repo maintenance tooling (changelog radar, converters),
+    # not library code anyone searches for.
+    unclaimed_exempt_paths:
+      - "scripts"
     skip_dirs:
       - node_modules
       - dist

deploy/copilotkit-docs.yaml

@@ -42,6 +42,25 @@ sources:
     file_patterns:
       - "showcase/shell-docs/src/content/docs/**/*.mdx"
       - "showcase/shell-docs/src/content/reference/**/*.mdx"
+    # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded
+    # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about
+    # them. Pooled across every source reading CopilotKit/CopilotKit.
+    #   snippets/  — MDX partials inlined into pages at render time; they are
+    #                never standalone pages and their prose is already indexed
+    #                through the pages that include them.
+    #   examples/  — per-example READMEs for the demo apps, not on the site.
+    #   showcase/integrations/ — per-integration setup notes that live with
+    #                the integration app, not on docs.copilotkit.ai.
+    # NOT listed, deliberately: showcase/shell-docs/src/content/ag-ui/. Those
+    # 96 files ARE live pages (docs.copilotkit.ai/ag-ui/<slug>, all 96 in the
+    # sitemap) served by src/app/ag-ui/[[...slug]] — the same shape as the
+    # reference tree this check was built for. The audit should keep saying so
+    # until someone decides whether to index them here or leave them to
+    # docs.ag-ui.com.
+    unclaimed_exempt_paths:
+      - "showcase/shell-docs/src/content/snippets"
+      - "examples"
+      - "showcase/integrations"
     chunk:
       target_tokens: 600
       overlap_tokens: 50
@@ -136,6 +155,15 @@ sources:
       # hand-written signal
       - "**/generated/**"
       - "**/*.pb.*"
+    # Correctly unclaimed in ag-ui-protocol/ag-ui (see Check 4 above):
+    #   apps/         — the dojo demo app and its READMEs; demo code, not SDK.
+    #   sdks/dotnet/  — the only .ts under it is the cross-language Vitest
+    #                   harness that drives the .NET server.
+    # NOT listed: middlewares/. Those are six published @ag-ui/* packages
+    # whose source no search tool can reach today.
+    unclaimed_exempt_paths:
+      - "apps"
+      - "sdks/dotnet"
     skip_dirs:
       - node_modules
       - dist

docs/config/index.html

@@ -728,6 +728,12 @@ <h2 id="sources">sources</h2>
             ># Share of walked files that may be missing from the index before
             the post-reindex audit reports a shortfall (optional)</span
           >
+          <span class="key">unclaimed_exempt_paths:</span>
+          <span class="comment"
+            ># Repo-relative directories confirmed to be correctly unindexed
+            (optional)</span
+          >
+          - <span class="value">"docs/snippets"</span>
           <span class="key">category:</span> <span class="value">faq</span>
           <span class="comment"
             ># Optional. Marks content as FAQ for /faq.txt and knowledge
@@ -773,6 +779,26 @@ <h2 id="sources">sources</h2>
             and the audit will flag the very first regression. It does not
             affect what gets indexed, so changing it does not trigger a reindex.
           </li>
+          <li>
+            <strong>unclaimed_exempt_paths</strong> — The shortfall check above
+            can only compare the index against the files the
+            <em>configured walk root</em> reaches, so a walk root pointed at the
+            wrong subtree leaves both halves blind in the same way and they
+            agree perfectly. A separate audit therefore walks the whole
+            repository and reports an <code>unclaimed_content</code> finding for
+            any directory tree of at least 10 files sharing one extension that
+            no source's path and <code>file_patterns</code> would ever reach —
+            when that extension is one the repository already indexes through
+            the tree's parent. Files a source's own
+            <code>exclude_patterns</code> reject are treated as accounted for,
+            since an exclusion is a decision on record. List here the
+            repo-root-relative directories (prefixes or globs) an operator has
+            reviewed and confirmed are correctly unclaimed — partials inlined at
+            render time, example apps, maintenance scripts — and the audit stops
+            reporting them. Entries are pooled across every source reading the
+            same repository. It does not affect what gets indexed, so changing
+            it does not trigger a reindex.
+          </li>
           <li>
             <strong>type</strong> — Determines chunking strategy.
             <code>markdown</code> splits on headings and uses token-based

src/__tests__/reindex-audit-shortfall.test.ts

@@ -39,7 +39,11 @@ vi.mock("../config.js", () => ({
   getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args),
 }));
 
-vi.mock("../indexing/utils.js", () => ({
+// Only walkSourceFiles is faked. The rest of the module stays real because
+// the unclaimed-content check (src/indexing/unclaimed-audit.ts) matches
+// patterns through it, and a module-shaped hole there would fail silently.
+vi.mock("../indexing/utils.js", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../indexing/utils.js")>()),
   walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args),
 }));