From a5286d66050690783f16122f965934e4c6d04064 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 13 Sep 2026 09:05:03 -0700 Subject: [PATCH 1/2] Pin that the docs source no longer claims the retired AG-UI mirror (RED) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CopilotKit#7092 deleted showcase/shell-docs/src/content/ag-ui/ upstream on 2026-09-11, about 14 hours after #164 added that tree to the docs source's file_patterns. The config still claims 96 files whose source is gone, and search-docs still advertises docs.copilotkit.ai/ag-ui/... links that die as the site rebuilds. These assertions fail against the shipped YAML as it stands: 4 ag-ui coverage cases plus both tool-description cases. A third describe block is new — a search tool's description is what an LLM reads to route a query, so a stale coverage claim there was invisible to every test we had. --- src/__tests__/copilotkit-docs-config.test.ts | 109 +++++++++++++------ 1 file changed, 78 insertions(+), 31 deletions(-) diff --git a/src/__tests__/copilotkit-docs-config.test.ts b/src/__tests__/copilotkit-docs-config.test.ts index af0edab..2b8ab4a 100644 --- a/src/__tests__/copilotkit-docs-config.test.ts +++ b/src/__tests__/copilotkit-docs-config.test.ts @@ -1,15 +1,20 @@ // Guards the SHIPPED production config, deploy/copilotkit-docs.yaml, on the -// two properties that decide whether a live documentation page is findable and -// whether the link a search result hands back actually resolves: +// three properties that decide whether a live documentation page is findable, +// whether the link a search result hands back actually resolves, and whether +// the tool description an LLM reads to route its query is true: // -// 1. which repository files the `docs` source claims, and -// 2. the URL each claimed file derives. +// 1. which repository files the `docs` source claims, +// 2. the URL each claimed file derives, and +// 3. what `search-docs` advertises that it covers. // -// Both were silently wrong before. The API reference — 184 pages under -// src/content/reference/ — went unindexed for months because the walk root was -// its sibling, and the ag-ui tree's 96 live pages went unindexed because -// file_patterns never named it. Neither failure was visible from any unit test -// over synthetic configs, because the defect lived in the YAML that ships. +// Both (1) and (2) were silently wrong before. The API reference — 184 pages +// under src/content/reference/ — went unindexed for months because the walk +// root was its sibling. Then the opposite failure: src/content/ag-ui/ was +// added to file_patterns hours before CopilotKit#7092 deleted that tree +// upstream, so the index held 96 pages whose source no longer exists and +// `search-docs` advertised links to dying URLs. Neither was visible from any +// unit test over synthetic configs, because the defect lived in the YAML that +// ships. // // The derivation half matters just as much in the other direction: the ordered // strip_prefix list is first-match-wins, so reordering or widening it mints @@ -39,16 +44,12 @@ if (!docsSource || !isFileSourceConfig(docsSource)) { const CONTENT = "showcase/shell-docs/src/content/"; describe("deploy/copilotkit-docs.yaml — docs source coverage", () => { - it.each([ - ["ag-ui/concepts/agents.mdx"], - ["ag-ui/concepts/architecture.mdx"], - ["ag-ui/agentic-protocols.mdx"], - ["ag-ui/sdk/js/core/events.mdx"], - ["docs/quickstart.mdx"], - ["reference/hooks/useAgent.mdx"], - ])("claims %s", (rel) => { - expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true); - }); + it.each([["docs/quickstart.mdx"], ["reference/hooks/useAgent.mdx"]])( + "claims %s", + (rel) => { + expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true); + }, + ); it("does not claim MDX partials, which are inlined into pages", () => { expect( @@ -56,7 +57,26 @@ describe("deploy/copilotkit-docs.yaml — docs source coverage", () => { ).toBe(false); }); - it("leaves the ag-ui tree out of the unclaimed-audit exemptions now that it is claimed", () => { + // CopilotKit#7092 deleted showcase/shell-docs/src/content/ag-ui/ upstream on + // 2026-09-11. Claiming a tree that no longer exists indexes pages whose + // source is gone and whose docs.copilotkit.ai/ag-ui/... URLs die as the site + // rebuilds. The canonical copy is the upstream ag-ui-protocol/ag-ui docs + // tree, already indexed by the `ag-ui-docs` source, which answers with + // docs.ag-ui.com links that resolve. + it.each([ + ["ag-ui/concepts/agents.mdx"], + ["ag-ui/concepts/architecture.mdx"], + ["ag-ui/agentic-protocols.mdx"], + ["ag-ui/sdk/js/core/events.mdx"], + ])("does not claim the retired mirror page %s", (rel) => { + expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(false); + }); + + // An exemption records a tree that EXISTS and is correctly unclaimed. The + // ag-ui tree no longer exists, so the unclaimed-content audit cannot fire on + // it either way; an entry here would be a permanent blind spot for whatever + // lands at that path next. + it("adds no unclaimed-audit exemption for the deleted ag-ui path", () => { const exempt = docsSource.unclaimed_exempt_paths ?? []; expect(exempt.some((p) => p.includes("content/ag-ui"))).toBe(false); }); @@ -64,29 +84,56 @@ describe("deploy/copilotkit-docs.yaml — docs source coverage", () => { describe("deploy/copilotkit-docs.yaml — derived URLs match the live routes", () => { it.each([ - // src/app/ag-ui/[[...slug]] serves the ag-ui tree UNDER /ag-ui/. - [ - "ag-ui/concepts/agents.mdx", - "https://docs.copilotkit.ai/ag-ui/concepts/agents", - ], - [ - "ag-ui/sdk/js/core/events.mdx", - "https://docs.copilotkit.ai/ag-ui/sdk/js/core/events", - ], - ["ag-ui/introduction.mdx", "https://docs.copilotkit.ai/ag-ui/introduction"], // Prose pages live at the site ROOT — the longer content/docs/ prefix // must keep winning over the shorter content/ one. ["docs/quickstart.mdx", "https://docs.copilotkit.ai/quickstart"], - // Reference pages keep their directory. + // Reference pages are a SIBLING of the prose tree, so they fall through to + // the shorter content/ prefix and keep their directory. This is the pin + // that catches a reordered or widened strip_prefix list. [ "reference/hooks/useAgent.mdx", "https://docs.copilotkit.ai/reference/hooks/useAgent", ], + ["docs/index.mdx", "https://docs.copilotkit.ai/"], ])("derives %s -> %s", (rel, expected) => { expect(deriveUrl(CONTENT + rel, docsSource)).toBe(expected); }); }); +// A search tool's description is what an LLM reads to decide which tool to +// call, so a stale coverage claim there misroutes every query it touches. +// search-docs advertised the docs.copilotkit.ai/ag-ui/... mirror right up +// until CopilotKit#7092 deleted it; the canonical AG-UI copy belongs to +// search-ag-ui-docs. +describe("deploy/copilotkit-docs.yaml — search-docs advertises only what it indexes", () => { + const tools = + ( + config as unknown as { + tools?: Array<{ name?: string; description?: string }>; + } + ).tools ?? []; + const byName = Object.fromEntries( + tools.map((t) => [String(t.name), String(t.description ?? "")]), + ); + + it("does not claim to cover the retired docs.copilotkit.ai/ag-ui pages", () => { + const description = byName["search-docs"]; + expect(description).toBeDefined(); + expect(description).not.toMatch(/ag-ui/i); + }); + + it("points AG-UI protocol questions at search-ag-ui-docs", () => { + expect(byName["search-docs"]).toContain("search-ag-ui-docs"); + }); + + it("describes search-ag-ui-docs as the AG-UI documentation, with no second copy", () => { + const description = byName["search-ag-ui-docs"]; + expect(description).toBeDefined(); + expect(description).toContain("https://docs.ag-ui.com"); + expect(description).not.toMatch(/docs\.copilotkit\.ai/); + }); +}); + // The shipped exclusion for the GitHub-issue triage relay. It is pinned here // rather than left to the YAML alone because the rule is what keeps relayed // issue bodies — SEO spam included — out of the weekly Notion search report From 22d03d34baa95c295609f94ed552a4125fbba682 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 13 Sep 2026 09:06:47 -0700 Subject: [PATCH 2/2] Stop indexing the AG-UI mirror upstream deleted, and unsay that we cover it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CopilotKit#7092 (merged 2026-09-11T22:44:24Z, 112 files, -25,059 lines) deleted showcase/shell-docs/src/content/ag-ui/ from CopilotKit/CopilotKit. #164 had added that tree to the docs source's file_patterns and its full walk finished 2026-09-11T08:11:58Z — about 14 hours earlier. We indexed a tree that was retired that night. Removed the ag-ui file_patterns entry and the comment block arguing for it. Restored both tool descriptions to their pre-#164 text, verbatim. The search-docs one is the point of this change: it claimed to cover "the AG-UI protocol pages hosted on the CopilotKit docs site (docs.copilotkit.ai/ag-ui/ ...)", which is what an LLM reads to pick a tool, and those pages no longer exist. search-ag-ui-docs already indexes the canonical upstream copy and hands back docs.ag-ui.com links that resolve, so nothing becomes unsearchable — only the link target changes to the one that survives. The unclaimed_exempt_paths comment claiming content/ag-ui/ is "CLAIMED, and always will be" is replaced rather than inverted: the directory is gone, so the audit cannot fire on it either way and an exemption would only blind it to whatever lands at that path next. No exemption added. scripts/test-path-filter.ts kept, comment corrected — #164 only edited comments there, and one now names a third subtree that does not exist. --- deploy/copilotkit-docs.yaml | 28 ++++++++------------ scripts/test-path-filter.ts | 4 +-- src/__tests__/copilotkit-docs-config.test.ts | 10 +++++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/deploy/copilotkit-docs.yaml b/deploy/copilotkit-docs.yaml index 54fbfbf..b853bb0 100644 --- a/deploy/copilotkit-docs.yaml +++ b/deploy/copilotkit-docs.yaml @@ -40,21 +40,15 @@ sources: # webhook path, which filters changed files by pattern alone with no # walk-root check. # - # ag-ui/ IS listed: those 96 files are live pages on THIS site, - # docs.copilotkit.ai/ag-ui/, served by src/app/ag-ui/[[...slug]], - # and all 96 are in the production sitemap. CopilotKit includes AG-UI, so - # they belong in the CopilotKit docs index. They are near-duplicates of - # the upstream ag-ui-protocol/ag-ui docs tree indexed by the `ag-ui-docs` - # source below, but that copy answers for docs.ag-ui.com and hands back - # docs.ag-ui.com links; this one hands back the copilotkit.ai links a - # CopilotKit user is reading. The ordered strip_prefix above needs no - # change for them — ag-ui/ is not under content/docs/, so it falls - # through to the content/ prefix and keeps its directory: - # content/ag-ui/concepts/agents.mdx -> /ag-ui/concepts/agents. + # content/ag-ui/ is NOT listed. It was, briefly: 96 .mdx mirrored from + # ag-ui-protocol/ag-ui and served at docs.copilotkit.ai/ag-ui/. + # CopilotKit#7092 deleted that whole tree on 2026-09-11, so indexing it + # now would mint links to pages that disappear as the docs site rebuilds. + # The canonical copy is the upstream repo, indexed by the `ag-ui-docs` + # source below, which hands back docs.ag-ui.com links that resolve. file_patterns: - "showcase/shell-docs/src/content/docs/**/*.mdx" - "showcase/shell-docs/src/content/reference/**/*.mdx" - - "showcase/shell-docs/src/content/ag-ui/**/*.mdx" # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about # them. Pooled across every source reading CopilotKit/CopilotKit. @@ -64,9 +58,9 @@ sources: # examples/ — per-example READMEs for the demo apps, not on the site. # showcase/integrations/ — per-integration setup notes that live with # the integration app, not on docs.copilotkit.ai. - # content/ag-ui/ is deliberately absent from this list and always will - # be: it is not exempt, it is CLAIMED by the file_patterns above. An - # exemption there would re-blind the audit to a tree of live pages. + # content/ag-ui/ needs no entry: CopilotKit#7092 deleted that tree, so the + # audit has nothing to walk there and cannot report it either way. An + # exemption would only blind the audit to whatever lands at that path next. unclaimed_exempt_paths: - "showcase/shell-docs/src/content/snippets" - "examples" @@ -187,7 +181,7 @@ sources: tools: - name: search-docs type: search - description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, how-tos for building with CopilotKit, and the AG-UI protocol pages hosted on the CopilotKit docs site (docs.copilotkit.ai/ag-ui/...). Use this for CopilotKit usage and configuration questions. For the upstream AG-UI protocol documentation at docs.ag-ui.com use search-ag-ui-docs; for source code use search-code. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." + description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, and how-tos for building with CopilotKit. Use this for CopilotKit usage and configuration questions. NOT for AG-UI protocol docs (use search-ag-ui-docs) and NOT for source code (use search-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." source: docs default_limit: 5 max_limit: 20 @@ -207,7 +201,7 @@ tools: - name: search-ag-ui-docs type: search - description: "Search the upstream AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI when you want the canonical ag-ui.com source; the CopilotKit docs site hosts its own near-identical copy of these pages, which search-docs returns with docs.copilotkit.ai links. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." + description: "Search the AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." source: ag-ui-docs default_limit: 5 max_limit: 20 diff --git a/scripts/test-path-filter.ts b/scripts/test-path-filter.ts index 56d83d7..299e949 100644 --- a/scripts/test-path-filter.ts +++ b/scripts/test-path-filter.ts @@ -65,8 +65,8 @@ const codeConfig = makeFileSourceConfig( // Docs source config — a deliberately generic *.mdx matcher for the glob // logic itself. The real `docs` source walks showcase/shell-docs/src/content/ -// and names its three page subtrees (docs/, reference/, ag-ui/) explicitly; -// that shipped config is asserted against directly in +// and names its page subtrees (docs/, reference/) explicitly; that shipped +// config is asserted against directly in // src/__tests__/copilotkit-docs-config.test.ts. const docsConfig = makeFileSourceConfig(["**/*.mdx"]); diff --git a/src/__tests__/copilotkit-docs-config.test.ts b/src/__tests__/copilotkit-docs-config.test.ts index 2b8ab4a..036898b 100644 --- a/src/__tests__/copilotkit-docs-config.test.ts +++ b/src/__tests__/copilotkit-docs-config.test.ts @@ -116,14 +116,20 @@ describe("deploy/copilotkit-docs.yaml — search-docs advertises only what it in tools.map((t) => [String(t.name), String(t.description ?? "")]), ); + // Naming AG-UI to route AWAY from it is correct and stays. What must not + // come back is a COVERAGE claim — the docs.copilotkit.ai/ag-ui/... link + // shape that told an LLM search-docs could answer AG-UI questions itself. it("does not claim to cover the retired docs.copilotkit.ai/ag-ui pages", () => { const description = byName["search-docs"]; expect(description).toBeDefined(); - expect(description).not.toMatch(/ag-ui/i); + expect(description).not.toMatch(/copilotkit\.ai\/ag-ui/i); + expect(description).not.toMatch(/hosted on the CopilotKit docs site/i); }); it("points AG-UI protocol questions at search-ag-ui-docs", () => { - expect(byName["search-docs"]).toContain("search-ag-ui-docs"); + expect(byName["search-docs"]).toMatch( + /NOT for AG-UI protocol docs \(use search-ag-ui-docs\)/, + ); }); it("describes search-ag-ui-docs as the AG-UI documentation, with no second copy", () => {