Stop indexing the AG-UI documentation mirror that upstream deleted

deploy/copilotkit-docs.yaml

@@ -40,21 +40,15 @@ sources:
     # webhook path, which filters changed files by pattern alone with no
     # walk-root check.
     #
-    # ag-ui/ IS listed: those 96 files are live pages on THIS site,
-    # docs.copilotkit.ai/ag-ui/<slug>, served by src/app/ag-ui/[[...slug]],
-    # and all 96 are in the production sitemap. CopilotKit includes AG-UI, so
-    # they belong in the CopilotKit docs index. They are near-duplicates of
-    # the upstream ag-ui-protocol/ag-ui docs tree indexed by the `ag-ui-docs`
-    # source below, but that copy answers for docs.ag-ui.com and hands back
-    # docs.ag-ui.com links; this one hands back the copilotkit.ai links a
-    # CopilotKit user is reading. The ordered strip_prefix above needs no
-    # change for them — ag-ui/ is not under content/docs/, so it falls
-    # through to the content/ prefix and keeps its directory:
-    # content/ag-ui/concepts/agents.mdx -> /ag-ui/concepts/agents.
+    # content/ag-ui/ is NOT listed. It was, briefly: 96 .mdx mirrored from
+    # ag-ui-protocol/ag-ui and served at docs.copilotkit.ai/ag-ui/<slug>.
+    # CopilotKit#7092 deleted that whole tree on 2026-09-11, so indexing it
+    # now would mint links to pages that disappear as the docs site rebuilds.
+    # The canonical copy is the upstream repo, indexed by the `ag-ui-docs`
+    # source below, which hands back docs.ag-ui.com links that resolve.
     file_patterns:
       - "showcase/shell-docs/src/content/docs/**/*.mdx"
       - "showcase/shell-docs/src/content/reference/**/*.mdx"
-      - "showcase/shell-docs/src/content/ag-ui/**/*.mdx"
     # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded
     # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about
     # them. Pooled across every source reading CopilotKit/CopilotKit.
@@ -64,9 +58,9 @@ sources:
     #   examples/  — per-example READMEs for the demo apps, not on the site.
     #   showcase/integrations/ — per-integration setup notes that live with
     #                the integration app, not on docs.copilotkit.ai.
-    # content/ag-ui/ is deliberately absent from this list and always will
-    # be: it is not exempt, it is CLAIMED by the file_patterns above. An
-    # exemption there would re-blind the audit to a tree of live pages.
+    # content/ag-ui/ needs no entry: CopilotKit#7092 deleted that tree, so the
+    # audit has nothing to walk there and cannot report it either way. An
+    # exemption would only blind the audit to whatever lands at that path next.
     unclaimed_exempt_paths:
       - "showcase/shell-docs/src/content/snippets"
       - "examples"
@@ -187,7 +181,7 @@ sources:
 tools:
   - name: search-docs
     type: search
-    description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, how-tos for building with CopilotKit, and the AG-UI protocol pages hosted on the CopilotKit docs site (docs.copilotkit.ai/ag-ui/...). Use this for CopilotKit usage and configuration questions. For the upstream AG-UI protocol documentation at docs.ag-ui.com use search-ag-ui-docs; for source code use search-code. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, and how-tos for building with CopilotKit. Use this for CopilotKit usage and configuration questions. NOT for AG-UI protocol docs (use search-ag-ui-docs) and NOT for source code (use search-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: docs
     default_limit: 5
     max_limit: 20
@@ -207,7 +201,7 @@ tools:
 
   - name: search-ag-ui-docs
     type: search
-    description: "Search the upstream AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI when you want the canonical ag-ui.com source; the CopilotKit docs site hosts its own near-identical copy of these pages, which search-docs returns with docs.copilotkit.ai links. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
+    description: "Search the AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need."
     source: ag-ui-docs
     default_limit: 5
     max_limit: 20

scripts/test-path-filter.ts

@@ -65,8 +65,8 @@ const codeConfig = makeFileSourceConfig(
 
 // Docs source config — a deliberately generic *.mdx matcher for the glob
 // logic itself. The real `docs` source walks showcase/shell-docs/src/content/
-// and names its three page subtrees (docs/, reference/, ag-ui/) explicitly;
-// that shipped config is asserted against directly in
+// and names its page subtrees (docs/, reference/) explicitly; that shipped
+// config is asserted against directly in
 // src/__tests__/copilotkit-docs-config.test.ts.
 const docsConfig = makeFileSourceConfig(["**/*.mdx"]);
 

src/__tests__/copilotkit-docs-config.test.ts

@@ -1,15 +1,20 @@
 // Guards the SHIPPED production config, deploy/copilotkit-docs.yaml, on the
-// two properties that decide whether a live documentation page is findable and
-// whether the link a search result hands back actually resolves:
+// three properties that decide whether a live documentation page is findable,
+// whether the link a search result hands back actually resolves, and whether
+// the tool description an LLM reads to route its query is true:
 //
-//   1. which repository files the `docs` source claims, and
-//   2. the URL each claimed file derives.
+//   1. which repository files the `docs` source claims,
+//   2. the URL each claimed file derives, and
+//   3. what `search-docs` advertises that it covers.
 //
-// Both were silently wrong before. The API reference — 184 pages under
-// src/content/reference/ — went unindexed for months because the walk root was
-// its sibling, and the ag-ui tree's 96 live pages went unindexed because
-// file_patterns never named it. Neither failure was visible from any unit test
-// over synthetic configs, because the defect lived in the YAML that ships.
+// Both (1) and (2) were silently wrong before. The API reference — 184 pages
+// under src/content/reference/ — went unindexed for months because the walk
+// root was its sibling. Then the opposite failure: src/content/ag-ui/ was
+// added to file_patterns hours before CopilotKit#7092 deleted that tree
+// upstream, so the index held 96 pages whose source no longer exists and
+// `search-docs` advertised links to dying URLs. Neither was visible from any
+// unit test over synthetic configs, because the defect lived in the YAML that
+// ships.
 //
 // The derivation half matters just as much in the other direction: the ordered
 // strip_prefix list is first-match-wins, so reordering or widening it mints
@@ -39,54 +44,102 @@ if (!docsSource || !isFileSourceConfig(docsSource)) {
 const CONTENT = "showcase/shell-docs/src/content/";
 
 describe("deploy/copilotkit-docs.yaml — docs source coverage", () => {
-  it.each([
-    ["ag-ui/concepts/agents.mdx"],
-    ["ag-ui/concepts/architecture.mdx"],
-    ["ag-ui/agentic-protocols.mdx"],
-    ["ag-ui/sdk/js/core/events.mdx"],
-    ["docs/quickstart.mdx"],
-    ["reference/hooks/useAgent.mdx"],
-  ])("claims %s", (rel) => {
-    expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true);
-  });
+  it.each([["docs/quickstart.mdx"], ["reference/hooks/useAgent.mdx"]])(
+    "claims %s",
+    (rel) => {
+      expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true);
+    },
+  );
 
   it("does not claim MDX partials, which are inlined into pages", () => {
     expect(
       matchesPatterns(CONTENT + "snippets/installation.mdx", docsSource),
     ).toBe(false);
   });
 
-  it("leaves the ag-ui tree out of the unclaimed-audit exemptions now that it is claimed", () => {
+  // CopilotKit#7092 deleted showcase/shell-docs/src/content/ag-ui/ upstream on
+  // 2026-09-11. Claiming a tree that no longer exists indexes pages whose
+  // source is gone and whose docs.copilotkit.ai/ag-ui/... URLs die as the site
+  // rebuilds. The canonical copy is the upstream ag-ui-protocol/ag-ui docs
+  // tree, already indexed by the `ag-ui-docs` source, which answers with
+  // docs.ag-ui.com links that resolve.
+  it.each([
+    ["ag-ui/concepts/agents.mdx"],
+    ["ag-ui/concepts/architecture.mdx"],
+    ["ag-ui/agentic-protocols.mdx"],
+    ["ag-ui/sdk/js/core/events.mdx"],
+  ])("does not claim the retired mirror page %s", (rel) => {
+    expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(false);
+  });
+
+  // An exemption records a tree that EXISTS and is correctly unclaimed. The
+  // ag-ui tree no longer exists, so the unclaimed-content audit cannot fire on
+  // it either way; an entry here would be a permanent blind spot for whatever
+  // lands at that path next.
+  it("adds no unclaimed-audit exemption for the deleted ag-ui path", () => {
     const exempt = docsSource.unclaimed_exempt_paths ?? [];
     expect(exempt.some((p) => p.includes("content/ag-ui"))).toBe(false);
   });
 });
 
 describe("deploy/copilotkit-docs.yaml — derived URLs match the live routes", () => {
   it.each([
-    // src/app/ag-ui/[[...slug]] serves the ag-ui tree UNDER /ag-ui/.
-    [
-      "ag-ui/concepts/agents.mdx",
-      "https://docs.copilotkit.ai/ag-ui/concepts/agents",
-    ],
-    [
-      "ag-ui/sdk/js/core/events.mdx",
-      "https://docs.copilotkit.ai/ag-ui/sdk/js/core/events",
-    ],
-    ["ag-ui/introduction.mdx", "https://docs.copilotkit.ai/ag-ui/introduction"],
     // Prose pages live at the site ROOT — the longer content/docs/ prefix
     // must keep winning over the shorter content/ one.
     ["docs/quickstart.mdx", "https://docs.copilotkit.ai/quickstart"],
-    // Reference pages keep their directory.
+    // Reference pages are a SIBLING of the prose tree, so they fall through to
+    // the shorter content/ prefix and keep their directory. This is the pin
+    // that catches a reordered or widened strip_prefix list.
     [
       "reference/hooks/useAgent.mdx",
       "https://docs.copilotkit.ai/reference/hooks/useAgent",
     ],
+    ["docs/index.mdx", "https://docs.copilotkit.ai/"],
   ])("derives %s -> %s", (rel, expected) => {
     expect(deriveUrl(CONTENT + rel, docsSource)).toBe(expected);
   });
 });
 
+// A search tool's description is what an LLM reads to decide which tool to
+// call, so a stale coverage claim there misroutes every query it touches.
+// search-docs advertised the docs.copilotkit.ai/ag-ui/... mirror right up
+// until CopilotKit#7092 deleted it; the canonical AG-UI copy belongs to
+// search-ag-ui-docs.
+describe("deploy/copilotkit-docs.yaml — search-docs advertises only what it indexes", () => {
+  const tools =
+    (
+      config as unknown as {
+        tools?: Array<{ name?: string; description?: string }>;
+      }
+    ).tools ?? [];
+  const byName = Object.fromEntries(
+    tools.map((t) => [String(t.name), String(t.description ?? "")]),
+  );
+
+  // Naming AG-UI to route AWAY from it is correct and stays. What must not
+  // come back is a COVERAGE claim — the docs.copilotkit.ai/ag-ui/... link
+  // shape that told an LLM search-docs could answer AG-UI questions itself.
+  it("does not claim to cover the retired docs.copilotkit.ai/ag-ui pages", () => {
+    const description = byName["search-docs"];
+    expect(description).toBeDefined();
+    expect(description).not.toMatch(/copilotkit\.ai\/ag-ui/i);
+    expect(description).not.toMatch(/hosted on the CopilotKit docs site/i);
+  });
+
+  it("points AG-UI protocol questions at search-ag-ui-docs", () => {
+    expect(byName["search-docs"]).toMatch(
+      /NOT for AG-UI protocol docs \(use search-ag-ui-docs\)/,
+    );
+  });
+
+  it("describes search-ag-ui-docs as the AG-UI documentation, with no second copy", () => {
+    const description = byName["search-ag-ui-docs"];
+    expect(description).toBeDefined();
+    expect(description).toContain("https://docs.ag-ui.com");
+    expect(description).not.toMatch(/docs\.copilotkit\.ai/);
+  });
+});
+
 // The shipped exclusion for the GitHub-issue triage relay. It is pinned here
 // rather than left to the YAML alone because the rule is what keeps relayed
 // issue bodies — SEO spam included — out of the weekly Notion search report