Harden toolkit docs generation and indexing

.github/workflows/generate-toolkit-docs.md

@@ -4,8 +4,8 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b
 
 ## What it does
 
-1. Builds the toolkit docs generator.
-2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata and summary endpoints.
+1. Type-checks and tests the toolkit docs generator.
+2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata endpoint.
 3. Syncs integrations sidebar navigation from the generated JSON.
 4. Creates or updates a PR on the stable `automation/toolkit-docs` branch if any files changed. Later runs overwrite that open PR with the latest generated docs.
 

.github/workflows/generate-toolkit-docs.yml

@@ -1,7 +1,7 @@
 name: Generate toolkit docs
 # Description: Generate toolkit JSON from Engine API, then sync sidebar navigation.
 # Flow:
-# 1) Build the toolkit docs generator
+# 1) Type-check and test the toolkit docs generator
 # 2) Generate toolkit JSON into toolkit-docs-generator/data/toolkits
 # 3) Sync integrations sidebar _meta.tsx from toolkit-docs-generator/data/toolkits
 # 4) Create or update a PR if changes were produced
@@ -10,6 +10,11 @@ on:
   repository_dispatch:
     types: [porter_deploy_succeeded]
   workflow_dispatch:
+    inputs:
+      pr_branch:
+        description: "Optional isolated branch for a verification PR"
+        required: false
+        type: string
   # 11:00 UTC = 3 AM PST / 4 AM PDT — late enough that DST drift doesn't matter.
   schedule:
     - cron: "0 11 * * *"
@@ -48,9 +53,8 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Build toolkit docs generator
-        run: pnpm build
-        working-directory: toolkit-docs-generator
+      - name: Validate toolkit docs generator
+        run: pnpm run toolkit-docs:check
 
       - name: Generate toolkit docs
         run: |
@@ -96,22 +100,24 @@ jobs:
         with:
           token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
           commit-message: "[AUTO] Adding MCP Servers docs update"
-          title: "[AUTO] Adding MCP Servers docs update"
+          title: ${{ inputs.pr_branch && '[TEST] Generated toolkit docs verification' || '[AUTO] Adding MCP Servers docs update' }}
           body: |
-            This PR was generated after a Porter deploy succeeded.
+            This PR was generated by the toolkit docs workflow.
 
             - Trigger: ${{ github.event_name }}
+            - Source ref: ${{ github.ref_name }}
             - Deploy env: ${{ github.event.client_payload.env || 'unknown' }}
             - Deploy SHA: ${{ github.event.client_payload.deploy_sha || 'unknown' }}
             - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           # Stable branch so later runs update the open auto-PR instead of
           # opening a new one each time. create-pull-request force-pushes the
           # latest generated docs onto this branch.
-          branch: automation/toolkit-docs
+          base: ${{ inputs.pr_branch && github.ref_name || github.event.repository.default_branch || 'main' }}
+          branch: ${{ inputs.pr_branch || 'automation/toolkit-docs' }}
           delete-branch: true
 
       - name: Request team review
-        if: steps.cpr.outputs.pull-request-number != ''
+        if: steps.cpr.outputs.pull-request-number != '' && (github.event_name != 'workflow_dispatch' || inputs.pr_branch == '')
         continue-on-error: true
         run: gh pr edit ${{ steps.cpr.outputs.pull-request-number }} --add-reviewer ArcadeAI/engineering-tools-and-dx
         env:

.github/workflows/llmstxt.yml

@@ -9,12 +9,14 @@ on:
       - "app/en/**/*.mdx"
       - "app/en/**/_meta.tsx"
       - "scripts/generate-llmstxt.ts"
+      - "toolkit-docs-generator/data/toolkits/**"
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
       - "app/en/**/*.mdx"
       - "app/en/**/_meta.tsx"
       - "scripts/generate-llmstxt.ts"
+      - "toolkit-docs-generator/data/toolkits/**"
 
 permissions:
   contents: write
@@ -24,6 +26,9 @@ jobs:
   llmstxt:
     name: Generate LLMSTXT
     runs-on: ubuntu-latest
+    if: >
+      (github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository)
 
     permissions:
       contents: write
@@ -40,6 +45,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.ref || github.ref }}
           token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
+          fetch-depth: 0
 
       - name: Install dependencies
         run: npm install -g pnpm

CLAUDE.md

@@ -6,7 +6,7 @@ Arcade documentation site built with Next.js + Nextra (App Router), using pnpm a
 
 ```bash
 pnpm dev              # Local dev server (port 3000)
-pnpm build            # Full production build (toolkit-markdown → next build → pagefind)
+pnpm build            # Full Next.js production build
 pnpm lint             # Lint with Ultracite (Biome-based)
 pnpm format           # Auto-format with Ultracite
 pnpm test             # Run all Vitest tests
@@ -27,7 +27,7 @@ pnpm vitest run tests/broken-link-check.test.ts
 - **`app/_lib/`** — Data-fetching utilities (toolkit catalog, slug generation, static params).
 - **`app/api/`** — API routes (markdown export, toolkit-data, glossary).
 - **`toolkit-docs-generator/`** — Generates MCP toolkit documentation from server metadata JSON files in `toolkit-docs-generator/data/toolkits/`.
-- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, pagefind indexing, i18n sync).
+- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, llms.txt generation, i18n sync).
 - **`tests/`** — Vitest tests (broken links, internal link validation, sitemap, smoke tests).
 - **`lib/`** — Next.js utilities (glossary remark plugin, llmstxt plugin).
 - **`next.config.ts`** — Contains ~138 redirect rules.

README.md

@@ -54,11 +54,11 @@ The `vale:fix` command requires an API key in `.env.local`. Without one, you can
 - [STYLEGUIDE.md](./STYLEGUIDE.md) - Writing standards for voice, tone, and structure
 - [AGENTS.md](./AGENTS.md) - Instructions for AI assistants working on docs
 
-## llms.txt Generation
+## llms.txt generation
 
-The project includes a Next.js plugin that automatically generates an `llms.txt` file following the [llms.txt specification](https://llmstxt.org/). This file helps LLMs understand and navigate the documentation.
+The project includes a script that generates an `llms.txt` file following the [llms.txt specification](https://llmstxt.org/). This file helps LLMs understand and navigate the documentation.
 
-**Automatic generation**: Runs during production builds (`pnpm build`)
+**Automatic generation**: Runs in the `Generate LLMs.txt` GitHub workflow
 
 **Manual generation**: Run `pnpm llmstxt` to regenerate the file
 

app/_components/algolia-search.tsx

@@ -2,7 +2,7 @@
 
 import { liteClient as algoliasearch } from "algoliasearch/lite";
 import { Search } from "lucide-react";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import {
   Configure,
   Highlight,
@@ -41,6 +41,64 @@ const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME;
 const searchClient =
   appId && searchKey ? algoliasearch(appId, searchKey) : null;
 
+export const ALGOLIA_SEARCH_CONFIG = {
+  attributesToSnippet: ["content:20"],
+  distinct: true,
+  highlightPreTag: "__ais-highlight__",
+  highlightPostTag: "__/ais-highlight__",
+  hitsPerPage: 15,
+  snippetEllipsisText: "…",
+};
+export const ALGOLIA_SEARCH_DEBOUNCE_MS = 150;
+
+type SearchStatus = "idle" | "loading" | "stalled" | "error";
+type SearchTimer = ReturnType<typeof setTimeout>;
+
+export function getSearchErrorMessage(status: SearchStatus): string | null {
+  return status === "error"
+    ? "Search failed. Check your connection and try again."
+    : null;
+}
+
+type ScheduleSearchOptions = {
+  query: string;
+  search: (nextQuery: string) => void;
+  setTypedQuery: (nextQuery: string) => void;
+  currentTimer: SearchTimer | null;
+  delayMs?: number;
+};
+
+export function scheduleSearch({
+  query,
+  search,
+  setTypedQuery,
+  currentTimer,
+  delayMs = ALGOLIA_SEARCH_DEBOUNCE_MS,
+}: ScheduleSearchOptions): SearchTimer | null {
+  setTypedQuery(query);
+  if (currentTimer) {
+    clearTimeout(currentTimer);
+  }
+  if (!query.trim()) {
+    search(query);
+    return null;
+  }
+  return setTimeout(() => search(query), delayMs);
+}
+
+export function searchResultsAreCurrent(
+  query: string,
+  resultsQuery: string,
+  status: SearchStatus
+): boolean {
+  const normalizedQuery = query.trim();
+  return (
+    normalizedQuery.length > 0 &&
+    status === "idle" &&
+    resultsQuery.trim() === normalizedQuery
+  );
+}
+
 function safeHref(url: string | undefined): string {
   if (!url) {
     return "/";
@@ -138,28 +196,107 @@ function SearchHit({ hit }: { hit: DocSearchRecord }) {
   );
 }
 
-function EmptyQuery() {
-  const { indexUiState } = useInstantSearch();
-  if (indexUiState.query) {
-    return null;
+function SearchResults({ query }: { query: string }) {
+  const { results, status } = useInstantSearch({ catchError: true });
+  if (!query.trim()) {
+    return (
+      <p className="px-4 py-8 text-center text-sm text-muted-foreground">
+        Start typing to search the docs…
+      </p>
+    );
+  }
+
+  const errorMessage = getSearchErrorMessage(status);
+  if (errorMessage) {
+    return (
+      <p
+        className="px-4 py-8 text-center text-sm text-muted-foreground"
+        role="alert"
+      >
+        {errorMessage}
+      </p>
+    );
+  }
+
+  if (
+    !(results && searchResultsAreCurrent(query, results.query ?? "", status))
+  ) {
+    return (
+      <p className="px-4 py-8 text-center text-sm text-muted-foreground">
+        Searching…
+      </p>
+    );
+  }
+
+  if (results.nbHits === 0) {
+    return (
+      <p className="px-4 py-8 text-center text-sm text-muted-foreground">
+        No results for{" "}
+        <strong className="text-foreground">"{results.query}"</strong>
+      </p>
+    );
   }
+
   return (
-    <p className="px-4 py-8 text-center text-sm text-muted-foreground">
-      Start typing to search the docs…
-    </p>
+    <Hits
+      classNames={{ item: "", list: "space-y-0.5", root: "" }}
+      hitComponent={({ hit }) => (
+        <SearchHit hit={hit as unknown as DocSearchRecord} />
+      )}
+    />
   );
 }
 
-function NoResults() {
-  const { results } = useInstantSearch();
-  if (!results?.query || results.nbHits > 0) {
-    return null;
-  }
+type SearchQueryHook = (
+  query: string,
+  search: (nextQuery: string) => void
+) => void;
+
+function SearchContent() {
+  const [typedQuery, setTypedQuery] = useState("");
+  const searchTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const queryHook = useCallback<SearchQueryHook>((query, search) => {
+    searchTimerRef.current = scheduleSearch({
+      query,
+      search,
+      setTypedQuery,
+      currentTimer: searchTimerRef.current,
+    });
+  }, []);
+
+  useEffect(
+    () => () => {
+      if (searchTimerRef.current) {
+        clearTimeout(searchTimerRef.current);
+      }
+    },
+    []
+  );
+
   return (
-    <p className="px-4 py-8 text-center text-sm text-muted-foreground">
-      No results for{" "}
-      <strong className="text-foreground">"{results.query}"</strong>
-    </p>
+    <>
+      <Configure {...ALGOLIA_SEARCH_CONFIG} />
+      <div className="flex items-center border-b border-border px-4">
+        <Search className="size-4 shrink-0 text-muted-foreground" />
+        <SearchBox
+          autoFocus
+          classNames={{
+            form: "flex flex-1",
+            input:
+              "w-full bg-transparent px-3 py-4 text-sm text-foreground placeholder:text-muted-foreground outline-none",
+            loadingIndicator: "hidden",
+            reset: "hidden",
+            root: "flex-1",
+            submit: "hidden",
+          }}
+          placeholder="Search docs…"
+          queryHook={queryHook}
+        />
+      </div>
+      <div className="max-h-[60vh] min-h-24 overflow-y-auto p-2">
+        <SearchResults query={typedQuery} />
+      </div>
+    </>
   );
 }
 
@@ -222,38 +359,7 @@ export function AlgoliaSearch() {
           <div className="relative z-10 w-full max-w-2xl overflow-hidden rounded-xl border border-border bg-popover shadow-2xl">
             {searchClient && indexName ? (
               <InstantSearch indexName={indexName} searchClient={searchClient}>
-                <Configure
-                  attributesToSnippet={["content:20"]}
-                  distinct={true}
-                  hitsPerPage={15}
-                  snippetEllipsisText="…"
-                />
-                <div className="flex items-center border-b border-border px-4">
-                  <Search className="size-4 shrink-0 text-muted-foreground" />
-                  <SearchBox
-                    autoFocus
-                    classNames={{
-                      form: "flex flex-1",
-                      input:
-                        "w-full bg-transparent px-3 py-4 text-sm text-foreground placeholder:text-muted-foreground outline-none",
-                      loadingIndicator: "hidden",
-                      reset: "hidden",
-                      root: "flex-1",
-                      submit: "hidden",
-                    }}
-                    placeholder="Search docs…"
-                  />
-                </div>
-                <div className="max-h-[60vh] overflow-y-auto p-2">
-                  <EmptyQuery />
-                  <NoResults />
-                  <Hits
-                    classNames={{ item: "", list: "space-y-0.5", root: "" }}
-                    hitComponent={({ hit }) => (
-                      <SearchHit hit={hit as unknown as DocSearchRecord} />
-                    )}
-                  />
-                </div>
+                <SearchContent />
               </InstantSearch>
             ) : (
               <SearchUnavailable />