From 533a01793db01f86fd49586e96d1eb097b0e6aa0 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 31 Aug 2026 17:21:54 +1000 Subject: [PATCH] feat(docs): triage script for 404 exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The raw Vercel 404 export is unreadable: 4,771 of 5,000 rows in the 2026-08-26 → 08-31 window were Next.js client segment-cache prefetches, burying the ~230 failures worth acting on under 95% noise. `bun run triage-404s ` dedupes by requestId (the export repeats rows), buckets by shape, drops the prefetch noise, and prints what is left with requester attribution — AI agent, search crawler, other bot, browser. Run against that export it reports 229 actionable requests across 175 distinct paths, which is what the redirect map was built from. The segment prefetches 404 because segment data is only generated when `cacheComponents` is on. Reproduced directly: curl -H 'Next-Router-Segment-Prefetch: /_tree' \ '.../docs/reference/agent-skills.segments/_tree.segment.rsc' -> 404 Without that header the same URL returns 200. The router falls back to a normal prefetch, so nothing is broken for the reader — it is ~1,500 wasted requests a day and a ruined 404 report. Next 16.2.6 has no flag to turn the client segment cache off; enabling `cacheComponents` is the real fix, tracked in CIP-3910. Not wired into prebuild: it reads a file that only exists once someone has pulled an export. Claude-Session: https://claude.ai/code/session_01BsA6C8Y48GDjh6ELL8yVjf --- package.json | 3 +- scripts/triage-404s.ts | 208 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 scripts/triage-404s.ts diff --git a/package.json b/package.json index f851fd1..0cf0db1 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "validate-links": "tsx scripts/validate-links.ts", "validate-redirects": "tsx scripts/validate-v2-redirects.ts", "validate-content": "tsx scripts/validate-content-api.ts", - "validate-mermaid": "tsx scripts/validate-mermaid.ts" + "validate-mermaid": "tsx scripts/validate-mermaid.ts", + "triage-404s": "tsx scripts/triage-404s.ts" }, "dependencies": { "fumadocs-core": "16.6.0", diff --git a/scripts/triage-404s.ts b/scripts/triage-404s.ts new file mode 100644 index 0000000..76d324a --- /dev/null +++ b/scripts/triage-404s.ts @@ -0,0 +1,208 @@ +#!/usr/bin/env tsx +/** + * 404 triage for a Vercel log export. + * + * The raw export is unreadable: in the 2026-08-26 → 08-31 window, 4,771 of + * 5,000 rows were Next.js client segment-cache prefetches, so the ~230 + * failures worth acting on sat under 95% noise. This classifies the rows, + * drops the noise, and prints what is left with requester attribution. + * + * Usage: bun run triage-404s + * Export from the Vercel dashboard: Logs → filter status 404 → Export CSV. + * + * Not wired into prebuild — it reads a file that only exists when someone + * has pulled an export. + */ +import fs from "node:fs"; + +interface Row { + requestId: string; + path: string; + status: string; + userAgent: string; + time: string; +} + +/** + * Requests the Next.js client segment cache makes on hover/viewport. They + * 404 because segment data is only generated when `cacheComponents` is on; + * the router falls back to a normal prefetch, so nothing is broken for the + * reader. Benign, high volume, and the reason raw 404 reports are useless. + */ +const SEGMENT_PREFETCH = /\.segments\/|\.rsc$/; +const ASSET = /\.(?:js|css|png|jpe?g|gif|webp|svg|ico|woff2?|map)$/; +/** `file.ts:44` in docs prose, resolved as a link by an agent. */ +const SOURCE_REF = /\.[cm]?[jt]sx?(?::\d+)?$/; +const MACHINE_ENDPOINT = + /\/(?:mcp|robots\.txt|openapi\.json|api\.json|\.well-known\/[^/]+)$/; + +type Bucket = + | "segment-prefetch" + | "source-ref" + | "markdown-suffix" + | "machine-endpoint" + | "asset" + | "page"; + +function classify(path: string): Bucket { + if (SEGMENT_PREFETCH.test(path)) return "segment-prefetch"; + if (/\.mdx?$/.test(path)) return "markdown-suffix"; + if (SOURCE_REF.test(path)) return "source-ref"; + if (MACHINE_ENDPOINT.test(path)) return "machine-endpoint"; + if (ASSET.test(path)) return "asset"; + return "page"; +} + +// Self-identified AI agents and scripted clients. A browser user-agent is +// not proof of a human: agents that do not identify themselves land in +// "browser", and bursts of sequential machine-shaped requests show up there. +const AI_AGENTS = + /GPTBot|OAI-SearchBot|ChatGPT-User|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|PerplexityBot|Perplexity-User|DuckAssist|Amazonbot|Applebot|Bytespider|Google-Extended|meta-external|cohere|curl\/|python-requests|node-fetch|Go-http-client|axios|undici/i; +const SEARCH_CRAWLERS = /Googlebot|bingbot|DuckDuckBot|YandexBot/i; + +type Requester = "ai" | "search" | "bot" | "browser"; + +function requester(userAgent: string): Requester { + if (AI_AGENTS.test(userAgent)) return "ai"; + if (SEARCH_CRAWLERS.test(userAgent)) return "search"; + if (/bot|spider|crawl/i.test(userAgent)) return "bot"; + return "browser"; +} + +/** Minimal RFC 4180 reader: the export quotes user-agent strings, which contain commas. */ +function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + for (let i = 0; i < text.length; i++) { + const char = text[i]; + if (quoted) { + if (char === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else quoted = false; + } else field += char; + continue; + } + if (char === '"') quoted = true; + else if (char === ",") { + row.push(field); + field = ""; + } else if (char === "\n") { + row.push(field); + rows.push(row); + row = []; + field = ""; + } else if (char !== "\r") field += char; + } + if (field || row.length > 0) { + row.push(field); + rows.push(row); + } + return rows; +} + +const file = process.argv[2]; +if (!file) { + console.error("usage: bun run triage-404s "); + process.exit(1); +} + +const table = parseCsv(fs.readFileSync(file, "utf8")); +const header = table[0]; +const index = (name: string) => header.indexOf(name); +const columns = { + requestId: index("requestId"), + path: index("requestPath"), + status: index("responseStatusCode"), + userAgent: index("requestUserAgent"), + time: index("TimeUTC"), +}; +for (const [name, position] of Object.entries(columns)) { + if (position === -1) { + console.error(`✗ export is missing the ${name} column`); + process.exit(1); + } +} + +// The export repeats rows; requestId is the only stable key. +const seen = new Set(); +const rows: Row[] = []; +for (const cells of table.slice(1)) { + if (cells.length < header.length) continue; + const requestId = cells[columns.requestId]; + if (seen.has(requestId)) continue; + seen.add(requestId); + // requestPath carries the host: "public-docs.vercel.app/docs/x?y=z". + const raw = cells[columns.path]; + const slash = raw.indexOf("/"); + rows.push({ + requestId, + path: (slash > 0 ? raw.slice(slash) : raw).split("?")[0], + status: cells[columns.status], + userAgent: cells[columns.userAgent], + time: cells[columns.time], + }); +} + +const duplicates = table.length - 1 - rows.length; +const buckets = new Map(); +for (const row of rows) { + const bucket = classify(row.path); + const list = buckets.get(bucket); + if (list) list.push(row); + else buckets.set(bucket, [row]); +} + +const times = rows.map((r) => r.time).sort(); +console.log( + `\n${rows.length} unique requests (${duplicates} duplicate rows dropped)`, +); +console.log(`window: ${times[0]} → ${times[times.length - 1]}\n`); + +console.log("bucket requests"); +for (const [bucket, list] of [...buckets].sort( + (a, b) => b[1].length - a[1].length, +)) { + const note = + bucket === "segment-prefetch" ? " (benign Next.js prefetch — ignore)" : ""; + console.log( + ` ${bucket.padEnd(17)} ${String(list.length).padStart(5)}${note}`, + ); +} + +const actionable = rows.filter((r) => classify(r.path) !== "segment-prefetch"); +const byPath = new Map(); +for (const row of actionable) { + const list = byPath.get(row.path); + if (list) list.push(row); + else byPath.set(row.path, [row]); +} + +console.log( + `\n${byPath.size} distinct paths worth triaging, ${actionable.length} requests:\n`, +); +console.log("hits ai search bot browser path"); +for (const [path, list] of [...byPath].sort( + (a, b) => b[1].length - a[1].length, +)) { + const counts: Record = { + ai: 0, + search: 0, + bot: 0, + browser: 0, + }; + for (const row of list) counts[requester(row.userAgent)]++; + console.log( + `${String(list.length).padStart(4)} ${String(counts.ai).padStart(2)} ${String(counts.search).padStart(6)} ${String(counts.bot).padStart(3)} ${String(counts.browser).padStart(7)} ${path}`, + ); +} + +const fromAgents = actionable.filter( + (r) => requester(r.userAgent) === "ai", +).length; +console.log( + `\n${fromAgents} of ${actionable.length} came from self-identified AI agents.`, +);