diff --git a/deploy/copilotkit-docs.yaml b/deploy/copilotkit-docs.yaml index ae6d13f..960f4dd 100644 --- a/deploy/copilotkit-docs.yaml +++ b/deploy/copilotkit-docs.yaml @@ -34,14 +34,27 @@ sources: strip_suffix: ".mdx" strip_route_groups: true strip_index: true - # Repo-root-relative, and now load-bearing: the wider walk root also - # exposes snippets/ (MDX partials), ag-ui/ (owned by the ag-ui-docs - # source) and framework-overviews/, none of which are standalone pages. - # These patterns are also what scopes the incremental webhook path, - # which filters changed files by pattern alone with no walk-root check. + # Repo-root-relative, and load-bearing: the wide walk root also exposes + # snippets/ (MDX partials) and framework-overviews/, neither of which is + # a standalone page. These patterns are also what scopes the incremental + # webhook path, which filters changed files by pattern alone with no + # walk-root check. + # + # ag-ui/ IS listed: those 96 files are live pages on THIS site, + # docs.copilotkit.ai/ag-ui/, served by src/app/ag-ui/[[...slug]], + # and all 96 are in the production sitemap. CopilotKit includes AG-UI, so + # they belong in the CopilotKit docs index. They are near-duplicates of + # the upstream ag-ui-protocol/ag-ui docs tree indexed by the `ag-ui-docs` + # source below, but that copy answers for docs.ag-ui.com and hands back + # docs.ag-ui.com links; this one hands back the copilotkit.ai links a + # CopilotKit user is reading. The ordered strip_prefix above needs no + # change for them — ag-ui/ is not under content/docs/, so it falls + # through to the content/ prefix and keeps its directory: + # content/ag-ui/concepts/agents.mdx -> /ag-ui/concepts/agents. file_patterns: - "showcase/shell-docs/src/content/docs/**/*.mdx" - "showcase/shell-docs/src/content/reference/**/*.mdx" + - "showcase/shell-docs/src/content/ag-ui/**/*.mdx" # Trees in this repo that hold .mdx but are CORRECTLY unclaimed, recorded # so the unclaimed-content audit (reindex-audit Check 4) stays quiet about # them. Pooled across every source reading CopilotKit/CopilotKit. @@ -51,12 +64,9 @@ sources: # examples/ — per-example READMEs for the demo apps, not on the site. # showcase/integrations/ — per-integration setup notes that live with # the integration app, not on docs.copilotkit.ai. - # NOT listed, deliberately: showcase/shell-docs/src/content/ag-ui/. Those - # 96 files ARE live pages (docs.copilotkit.ai/ag-ui/, all 96 in the - # sitemap) served by src/app/ag-ui/[[...slug]] — the same shape as the - # reference tree this check was built for. The audit should keep saying so - # until someone decides whether to index them here or leave them to - # docs.ag-ui.com. + # content/ag-ui/ is deliberately absent from this list and always will + # be: it is not exempt, it is CLAIMED by the file_patterns above. An + # exemption there would re-blind the audit to a tree of live pages. unclaimed_exempt_paths: - "showcase/shell-docs/src/content/snippets" - "examples" @@ -177,7 +187,7 @@ sources: tools: - name: search-docs type: search - description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, and how-tos for building with CopilotKit. Use this for CopilotKit usage and configuration questions. NOT for AG-UI protocol docs (use search-ag-ui-docs) and NOT for source code (use search-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." + description: "Search the CopilotKit product documentation (https://docs.copilotkit.ai) — guides, concepts, quickstarts, API reference, how-tos for building with CopilotKit, and the AG-UI protocol pages hosted on the CopilotKit docs site (docs.copilotkit.ai/ag-ui/...). Use this for CopilotKit usage and configuration questions. For the upstream AG-UI protocol documentation at docs.ag-ui.com use search-ag-ui-docs; for source code use search-code. This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." source: docs default_limit: 5 max_limit: 20 @@ -197,7 +207,7 @@ tools: - name: search-ag-ui-docs type: search - description: "Search the AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." + description: "Search the upstream AG-UI protocol documentation (https://docs.ag-ui.com) — the Agent-User Interaction protocol spec, event types, message schemas, and framework integration guides. Use this for protocol-level questions about AG-UI when you want the canonical ag-ui.com source; the CopilotKit docs site hosts its own near-identical copy of these pages, which search-docs returns with docs.copilotkit.ai links. NOT for CopilotKit product docs (use search-docs) and NOT for source code (use search-ag-ui-code). This is a semantic search, so prefer performing multiple queries with different phrases instead of a single long query, until you find all the context you need." source: ag-ui-docs default_limit: 5 max_limit: 20 diff --git a/scripts/test-path-filter.ts b/scripts/test-path-filter.ts index 9af260b..56d83d7 100644 --- a/scripts/test-path-filter.ts +++ b/scripts/test-path-filter.ts @@ -63,9 +63,11 @@ const codeConfig = makeFileSourceConfig( ], ); -// Docs source config — mirrors the real `docs` source: only *.mdx, no excludes. -// (The source path is showcase/shell-docs/src/content/docs/, so derived paths -// live under that tree.) +// Docs source config — a deliberately generic *.mdx matcher for the glob +// logic itself. The real `docs` source walks showcase/shell-docs/src/content/ +// and names its three page subtrees (docs/, reference/, ag-ui/) explicitly; +// that shipped config is asserted against directly in +// src/__tests__/copilotkit-docs-config.test.ts. const docsConfig = makeFileSourceConfig(["**/*.mdx"]); console.log("=== Path Filter Tests ===\n"); @@ -187,7 +189,7 @@ assert( assert(matchesPatterns("src/index.ts", codeConfig), "root src file included"); // --- docs (*.mdx only) --- -// Paths reflect the current docs source tree: showcase/shell-docs/src/content/docs/ +// Paths reflect the prose subtree of the docs source: content/docs/. console.log("\n--- docs (*.mdx only) ---"); assert( matchesPatterns( diff --git a/src/__tests__/copilotkit-docs-config.test.ts b/src/__tests__/copilotkit-docs-config.test.ts new file mode 100644 index 0000000..413fc0f --- /dev/null +++ b/src/__tests__/copilotkit-docs-config.test.ts @@ -0,0 +1,88 @@ +// Guards the SHIPPED production config, deploy/copilotkit-docs.yaml, on the +// two properties that decide whether a live documentation page is findable and +// whether the link a search result hands back actually resolves: +// +// 1. which repository files the `docs` source claims, and +// 2. the URL each claimed file derives. +// +// Both were silently wrong before. The API reference — 184 pages under +// src/content/reference/ — went unindexed for months because the walk root was +// its sibling, and the ag-ui tree's 96 live pages went unindexed because +// file_patterns never named it. Neither failure was visible from any unit test +// over synthetic configs, because the defect lived in the YAML that ships. +// +// The derivation half matters just as much in the other direction: the ordered +// strip_prefix list is first-match-wins, so reordering or widening it mints +// plausible URLs that 404 — a search result that lies, which is worse than a +// missing one. These assertions pin one page per subtree. + +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { ServerConfigSchema, isFileSourceConfig } from "../types.js"; +import { matchesPatterns } from "../indexing/utils.js"; +import { deriveUrl } from "../indexing/url-derivation.js"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const CONFIG_PATH = resolve(REPO_ROOT, "deploy/copilotkit-docs.yaml"); + +const config = ServerConfigSchema.parse( + parseYaml(readFileSync(CONFIG_PATH, "utf8")), +); +const docsSource = config.sources.find((s) => s.name === "docs"); +if (!docsSource || !isFileSourceConfig(docsSource)) { + throw new Error("deploy/copilotkit-docs.yaml has no file source named docs"); +} + +const CONTENT = "showcase/shell-docs/src/content/"; + +describe("deploy/copilotkit-docs.yaml — docs source coverage", () => { + it.each([ + ["ag-ui/concepts/agents.mdx"], + ["ag-ui/concepts/architecture.mdx"], + ["ag-ui/agentic-protocols.mdx"], + ["ag-ui/sdk/js/core/events.mdx"], + ["docs/quickstart.mdx"], + ["reference/hooks/useAgent.mdx"], + ])("claims %s", (rel) => { + expect(matchesPatterns(CONTENT + rel, docsSource)).toBe(true); + }); + + it("does not claim MDX partials, which are inlined into pages", () => { + expect( + matchesPatterns(CONTENT + "snippets/installation.mdx", docsSource), + ).toBe(false); + }); + + it("leaves the ag-ui tree out of the unclaimed-audit exemptions now that it is claimed", () => { + const exempt = docsSource.unclaimed_exempt_paths ?? []; + expect(exempt.some((p) => p.includes("content/ag-ui"))).toBe(false); + }); +}); + +describe("deploy/copilotkit-docs.yaml — derived URLs match the live routes", () => { + it.each([ + // src/app/ag-ui/[[...slug]] serves the ag-ui tree UNDER /ag-ui/. + [ + "ag-ui/concepts/agents.mdx", + "https://docs.copilotkit.ai/ag-ui/concepts/agents", + ], + [ + "ag-ui/sdk/js/core/events.mdx", + "https://docs.copilotkit.ai/ag-ui/sdk/js/core/events", + ], + ["ag-ui/introduction.mdx", "https://docs.copilotkit.ai/ag-ui/introduction"], + // Prose pages live at the site ROOT — the longer content/docs/ prefix + // must keep winning over the shorter content/ one. + ["docs/quickstart.mdx", "https://docs.copilotkit.ai/quickstart"], + // Reference pages keep their directory. + [ + "reference/hooks/useAgent.mdx", + "https://docs.copilotkit.ai/reference/hooks/useAgent", + ], + ])("derives %s -> %s", (rel, expected) => { + expect(deriveUrl(CONTENT + rel, docsSource)).toBe(expected); + }); +});