Index the 96 AG-UI documentation pages hosted on docs.copilotkit.ai

deploy/copilotkit-docs.yaml

@@ -34,14 +34,27 @@ sources:
       strip_suffix: ".mdx"
       strip_route_groups: true
       strip_index: true
-    # Repo-root-relative, and now load-bearing: the wider walk root also
-    # exposes snippets/ (MDX partials), ag-ui/ (owned by the ag-ui-docs
-    # source) and framework-overviews/, none of which are standalone pages.
-    # These patterns are also what scopes the incremental webhook path,
-    # which filters changed files by pattern alone with no walk-root check.
+    # Repo-root-relative, and load-bearing: the wide walk root also exposes
+    # snippets/ (MDX partials) and framework-overviews/, neither of which is
+    # a standalone page. These patterns are also what scopes the incremental
+    # webhook path, which filters changed files by pattern alone with no
+    # walk-root check.
+    #
+    # ag-ui/ IS listed: those 96 files are live pages on THIS site,
+    # docs.copilotkit.ai/ag-ui/<slug>, served by src/app/ag-ui/[[...slug]],
+    # and all 96 are in the production sitemap. CopilotKit includes AG-UI, so
+    # they belong in the CopilotKit docs index. They are near-duplicates of
+    # the upstream ag-ui-protocol/ag-ui docs tree indexed by the `ag-ui-docs`
+    # source below, but that copy answers for docs.ag-ui.com and hands back
+    # docs.ag-ui.com links; this one hands back the copilotkit.ai links a
+    # CopilotKit user is reading. The ordered strip_prefix above needs no
+    # change for them — ag-ui/ is not under content/docs/, so it falls
+    # through to the content/ prefix and keeps its directory:
+    # content/ag-ui/concepts/agents.mdx -> /ag-ui/concepts/agents.
     file_patterns:
       - "showcase/shell-docs/src/content/docs/**/*.mdx"
       - "showcase/shell-docs/src/content/reference/**/*.mdx"
+      - "showcase/shell-docs/src/content/ag-ui/**/*.mdx"
     # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded
     # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about
     # them. Pooled across every source reading CopilotKit/CopilotKit.
@@ -51,12 +64,9 @@ sources:
     #   examples/  — per-example READMEs for the demo apps, not on the site.
     #   showcase/integrations/ — per-integration setup notes that live with
     #                the integration app, not on docs.copilotkit.ai.
-    # NOT listed, deliberately: showcase/shell-docs/src/content/ag-ui/. Those
-    # 96 files ARE live pages (docs.copilotkit.ai/ag-ui/<slug>, all 96 in the
-    # sitemap) served by src/app/ag-ui/[[...slug]] — the same shape as the
-    # reference tree this check was built for. The audit should keep saying so
-    # until someone decides whether to index them here or leave them to
-    # docs.ag-ui.com.
+    # content/ag-ui/ is deliberately absent from this list and always will
+    # be: it is not exempt, it is CLAIMED by the file_patterns above. An
+    # exemption there would re-blind the audit to a tree of live pages.
     unclaimed_exempt_paths:
       - "showcase/shell-docs/src/content/snippets"
       - "examples"
@@ -177,7 +187,7 @@ sources:
 tools:
   - name: search-docs
     type: search
-    description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, and how-tos for building with CopilotKit. Use this for CopilotKit usage and configuration questions. NOT for AG-UI protocol docs (use search-ag-ui-docs) and NOT for source code (use search-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, how-tos for building with CopilotKit, and the AG-UI protocol pages hosted on the CopilotKit docs site (docs.copilotkit.ai/ag-ui/...). Use this for CopilotKit usage and configuration questions. For the upstream AG-UI protocol documentation at docs.ag-ui.com use search-ag-ui-docs; for source code use search-code. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: docs
     default_limit: 5
     max_limit: 20
@@ -197,7 +207,7 @@ tools:
 
   - name: search-ag-ui-docs
     type: search
-    description: "Search the AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the upstream AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI when you want the canonical ag-ui.com source; the CopilotKit docs site hosts its own near-identical copy of these pages, which search-docs returns with docs.copilotkit.ai links. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: ag-ui-docs
     default_limit: 5
     max_limit: 20

scripts/test-path-filter.ts

@@ -63,9 +63,11 @@ const codeConfig = makeFileSourceConfig(
   ],
 );
 
-// Docs source config — mirrors the real `docs` source: only *.mdx, no excludes.
-// (The source path is showcase/shell-docs/src/content/docs/, so derived paths
-// live under that tree.)
+// Docs source config — a deliberately generic *.mdx matcher for the glob
+// logic itself. The real `docs` source walks showcase/shell-docs/src/content/
+// and names its three page subtrees (docs/, reference/, ag-ui/) explicitly;
+// that shipped config is asserted against directly in
+// src/__tests__/copilotkit-docs-config.test.ts.
 const docsConfig = makeFileSourceConfig(["**/*.mdx"]);
 
 console.log("=== Path Filter Tests ===\n");
@@ -187,7 +189,7 @@ assert(
 assert(matchesPatterns("src/index.ts", codeConfig), "root src file included");
 
 // --- docs (*.mdx only) ---
-// Paths reflect the current docs source tree: showcase/shell-docs/src/content/docs/
+// Paths reflect the prose subtree of the docs source: content/docs/.
 console.log("\n--- docs (*.mdx only) ---");
 assert(
   matchesPatterns(

src/__tests__/copilotkit-docs-config.test.ts

@@ -0,0 +1,88 @@
+// Guards the SHIPPED production config, deploy/copilotkit-docs.yaml, on the
+// two properties that decide whether a live documentation page is findable and
+// whether the link a search result hands back actually resolves:
+//
+//   1. which repository files the `docs` source claims, and
+//   2. the URL each claimed file derives.
+//
+// Both were silently wrong before. The API reference — 184 pages under
+// src/content/reference/ — went unindexed for months because the walk root was
+// its sibling, and the ag-ui tree's 96 live pages went unindexed because
+// file_patterns never named it. Neither failure was visible from any unit test
+// over synthetic configs, because the defect lived in the YAML that ships.
+//
+// The derivation half matters just as much in the other direction: the ordered
+// strip_prefix list is first-match-wins, so reordering or widening it mints
+// plausible URLs that 404 — a search result that lies, which is worse than a
+// missing one. These assertions pin one page per subtree.
+
+import { describe, it, expect } from "vitest";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse as parseYaml } from "yaml";
+import { ServerConfigSchema, isFileSourceConfig } from "../types.js";
+import { matchesPatterns } from "../indexing/utils.js";
+import { deriveUrl } from "../indexing/url-derivation.js";
+
+const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+const CONFIG_PATH = resolve(REPO_ROOT, "deploy/copilotkit-docs.yaml");
+
+const config = ServerConfigSchema.parse(
+  parseYaml(readFileSync(CONFIG_PATH, "utf8")),
+);
+const docsSource = config.sources.find((s) => s.name === "docs");
+if (!docsSource || !isFileSourceConfig(docsSource)) {
+  throw new Error("deploy/copilotkit-docs.yaml has no file source named docs");
+}
+
+const CONTENT = "showcase/shell-docs/src/content/";
+
+describe("deploy/copilotkit-docs.yaml — docs source coverage", () => {
+  it.each([
+    ["ag-ui/concepts/agents.mdx"],
+    ["ag-ui/concepts/architecture.mdx"],
+    ["ag-ui/agentic-protocols.mdx"],
+    ["ag-ui/sdk/js/core/events.mdx"],
+    ["docs/quickstart.mdx"],
+    ["reference/hooks/useAgent.mdx"],
+  ])("claims %s", (rel) => {
+    expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true);
+  });
+
+  it("does not claim MDX partials, which are inlined into pages", () => {
+    expect(
+      matchesPatterns(CONTENT + "snippets/installation.mdx", docsSource),
+    ).toBe(false);
+  });
+
+  it("leaves the ag-ui tree out of the unclaimed-audit exemptions now that it is claimed", () => {
+    const exempt = docsSource.unclaimed_exempt_paths ?? [];
+    expect(exempt.some((p) => p.includes("content/ag-ui"))).toBe(false);
+  });
+});
+
+describe("deploy/copilotkit-docs.yaml — derived URLs match the live routes", () => {
+  it.each([
+    // src/app/ag-ui/[[...slug]] serves the ag-ui tree UNDER /ag-ui/.
+    [
+      "ag-ui/concepts/agents.mdx",
+      "https://docs.copilotkit.ai/ag-ui/concepts/agents",
+    ],
+    [
+      "ag-ui/sdk/js/core/events.mdx",
+      "https://docs.copilotkit.ai/ag-ui/sdk/js/core/events",
+    ],
+    ["ag-ui/introduction.mdx", "https://docs.copilotkit.ai/ag-ui/introduction"],
+    // Prose pages live at the site ROOT — the longer content/docs/ prefix
+    // must keep winning over the shorter content/ one.
+    ["docs/quickstart.mdx", "https://docs.copilotkit.ai/quickstart"],
+    // Reference pages keep their directory.
+    [
+      "reference/hooks/useAgent.mdx",
+      "https://docs.copilotkit.ai/reference/hooks/useAgent",
+    ],
+  ])("derives %s -> %s", (rel, expected) => {
+    expect(deriveUrl(CONTENT + rel, docsSource)).toBe(expected);
+  });
+});