diff --git a/.changeset/http-transport-cli-redirects.md b/.changeset/http-transport-cli-redirects.md index 61d53a7..8315ecd 100644 --- a/.changeset/http-transport-cli-redirects.md +++ b/.changeset/http-transport-cli-redirects.md @@ -19,7 +19,7 @@ Prerender anything over HTTP, from the command line, with redirects the host can **Query-string pages.** `keepQuery: true` renders `/posts?page=2` apart from `/posts` (parameters sorted for dedupe), following its links and capturing its data, but writes it only when its seed entry names a `filename` — a static host serves a path the same for every query. Off by default; CLI `--keep-query`. -**Router seeding, from the server.** New `prerender-crawler/routers` (no Node imports — for application server code): `tanstackRouterPages(router)` and `solidRouterPages(router | routes, { base? })` list a router's static pages from the instance the app built for the request (leaves and indexes, no params or splats), and `announcePages(request, headers, paths)` puts them on the response's hint header when the request is the crawler's. `@solidjs/prerender` gains `announceRoutes(router)` — one line in the app root (Solid Router) or the request setup (TanStack Router); it takes either router by shape, reads the ambient request event, and is a no-op in the browser. The Vite plugin, the CLI against a module, and the CLI against a running server all seed from the same header. +**Announcing pages, from the server.** New `prerender-crawler/announce` (no Node imports — for application server code): `announcePages(request, headers, paths)` puts a server's static page paths on the response's hint header when the request is the crawler's, so every crawl — the Vite plugin, the CLI against a module, the CLI against a running server — seeds from the same header. The crawler defines only the wire; which routes a router has is the framework integration's to enumerate. `@solidjs/prerender` ships that for the routers Solid apps use: `solidRouterPages(router | routes, { base? })` (a Solid Router 2 `createRouter` instance or definition tree), `tanstackRouterPages(router)` (a TanStack instance's `routesByPath`), and `announceRoutes(router)` — one line in the app root (Solid Router) or the request setup (TanStack Router); it takes either router by shape, reads the ambient request event, and is a no-op in the browser. **Removed:** `fileRoutePages`, `staticRoutePaths`, and the Vite plugin's `fileRoutes` option — build-time seeding that walked a `filesystem-routing` directory and re-derived its path rules. The server's own router is the source of truth for which pages exist, and the header works for crawls that never see the project's disk (the CLI against a running server). `filesystem-routing` is no longer a peer dependency. diff --git a/packages/crawler/README.md b/packages/crawler/README.md index 45443ab..0c2ac08 100644 --- a/packages/crawler/README.md +++ b/packages/crawler/README.md @@ -106,23 +106,25 @@ result.skipped; // SkippedPage[] — failures left out (failOnError: false) `httpTransport` sends the crawl's requests to the target's origin (path and query kept) and hands redirects back as the 3xx responses the server sent. Pass `{ headers }` for an auth token or `{ fetch }` for a custom implementation. `moduleTransport` imports a module exporting `handleRequest`, `fetch`, or `default.fetch` and calls it directly. -### Seeding from the router +### Announcing pages -The crawl finds pages by following links and by reading the hint header (`x-prerender`, comma-separated paths) off responses. A page nothing links to is invisible to the first; the second is how a server that knows its routes declares them — and the thing that knows the routes is the router the app built for the request. `prerender-crawler/routers` has the helpers, free of Node imports so application server code can use them: +The crawl finds pages by following links and by reading the hint header (`x-prerender`, comma-separated paths) off responses. A page nothing links to is invisible to the first; the second is how a server that knows its routes declares them — and the thing that knows the routes is the router the app built for the request. The crawler knows no router; it only defines the wire. `prerender-crawler/announce` is that wire, free of Node imports so application server code can use it: ```ts -import { announcePages, tanstackRouterPages, solidRouterPages } from "prerender-crawler/routers"; - -// TanStack Router (any flavor — the helper reads the instance's `routesByPath`) -announcePages(request, response.headers, tanstackRouterPages(router)); - -// Solid Router (a `createRouter` instance, or a route-definition tree with `{ base }`) -announcePages(request, response.headers, solidRouterPages(Router)); +import { announcePages } from "prerender-crawler/announce"; + +// in the request handler, with whatever the router exposes — e.g. TanStack Router: +const pages = Object.entries(router.routesByPath) + .filter( + ([path, route]) => !path.includes("$") && (route.fullPath.endsWith("/") || !route.children) + ) + .map(([path]) => path); +announcePages(request, response.headers, pages); ``` -`announcePages` writes the header only when the request is the crawler's (it carries the hint header) — a visitor's response is untouched. Each `*Pages` helper returns the paths that address a static page: no parameters or splats, a leaf or an index (a layout with children but no index has no page of its own). Dynamic routes are still found by their links; only a render knows their values. The Vite plugin, the CLI against a module, and the CLI against a running server all send the hint header, so one line in the app seeds all three. +`announcePages` writes the header only when the request is the crawler's (it carries the hint header) — a visitor's response is untouched. What to announce: the paths that address a static page — no parameters or splats (only a render knows their values; the crawl finds those pages by their links), leaves and indexes (a layout with children but no index has no page of its own). The Vite plugin, the CLI against a module, and the CLI against a running server all send the hint header, so one line in the app seeds all three. -[`@solidjs/prerender`](../solid) wraps this as `announceRoutes(Router)` for Solid apps, reading the request from the ambient request event. +Enumerating a specific router's static pages is the framework integration's job, not this package's: [`@solidjs/prerender`](../solid) ships `solidRouterPages`, `tanstackRouterPages`, and `announceRoutes(router)` (which also reads the request from the ambient request event) for the routers Solid apps use. ### Redirects @@ -225,7 +227,7 @@ interface PrerenderContext { - `redirects(options?)`, `formatRedirectsFile(records, force?)` — the redirects integration and its `_redirects` formatter. - `sitemap(options)`, `indexable(page)`, `formatSitemap(entries)` — the sitemap integration and its parts. - `report(options?)` — the crawl report integration. -- `prerender-crawler/routers`: `announcePages(request, headers, paths)`, `tanstackRouterPages(router)`, `solidRouterPages(router | routes, { base? })`, `HINT_HEADER`. +- `prerender-crawler/announce`: `announcePages(request, headers, paths, { header? })`, `HINT_HEADER` — the wire, for application server code. - `extractLinks(html, pageUrl, { keepQuery? })`, `normalizeLink(href, base, origin)`, `normalizeRoute(url)`, `normalizePath(pathname)`, `splitRoute(route)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives. ## Requirements diff --git a/packages/crawler/package.json b/packages/crawler/package.json index 58f9388..a10aeb6 100644 --- a/packages/crawler/package.json +++ b/packages/crawler/package.json @@ -31,9 +31,9 @@ "types": "./dist/vite.d.ts", "default": "./dist/vite.js" }, - "./routers": { - "types": "./dist/routers.d.ts", - "default": "./dist/routers.js" + "./announce": { + "types": "./dist/announce.d.ts", + "default": "./dist/announce.js" } }, "bin": { @@ -61,11 +61,7 @@ } }, "devDependencies": { - "@solidjs/router": "2.0.0-next.21", - "@solidjs/web": "2.0.0-rc.6", - "@tanstack/router-core": "^1.171.27", "@types/node": "^22.0.0", - "solid-js": "2.0.0-rc.6", "typescript": "^5.8.0", "vite": "^8.0.0", "vitest": "^4.0.0" diff --git a/packages/crawler/src/announce.ts b/packages/crawler/src/announce.ts new file mode 100644 index 0000000..8b508bb --- /dev/null +++ b/packages/crawler/src/announce.ts @@ -0,0 +1,52 @@ +/** + * The announce protocol: how a server tells a crawl which pages it has. + * + * The crawl finds pages by following links and by reading the hint header + * (`x-prerender`, comma-separated paths) off responses. A route nothing + * links to is invisible to the first; the second is how a server that + * KNOWS its routes declares them — and the thing that knows the routes is + * the router the app built for the request. Which routes count is the + * router's business, so the enumeration lives with the framework + * integration (`@solidjs/prerender` ships Solid Router's and TanStack + * Router's); this module is only the wire: put the paths on the header + * when the request is the crawler's. Every crawl — the Vite plugin, the + * CLI against a built module, the CLI against a running server — sends + * the hint request header and seeds from the answer. + * + * What to announce: paths that address a static page. No parameters or + * splats (only a render knows their values; the crawl finds them by their + * links), leaves and indexes (a layout with children but no index has no + * page of its own). One spelling per page — the engine normalizes trailing + * slashes, so either is fine. + * + * Imported by application SERVER code: no Node imports here. + */ + +/** The hint header the engine reads, and the request header it sends. */ +export const HINT_HEADER = "x-prerender"; + +export interface AnnounceOptions { + /** The header name, if the crawl was configured with a custom `hintHeader`. @default "x-prerender" */ + header?: string; +} + +/** + * Puts `paths` on the response's hint header — when the request is the + * crawler's (it carries the hint header). Returns whether it did. A + * regular visitor's response is left untouched. + * + * ```ts + * announcePages(event.request, event.response.headers, staticPaths); + * ``` + */ +export function announcePages( + request: Request, + headers: Headers, + paths: readonly string[], + options: AnnounceOptions = {} +): boolean { + const { header = HINT_HEADER } = options; + if (!request.headers.has(header) || paths.length === 0) return false; + headers.set(header, paths.join(",")); + return true; +} diff --git a/packages/crawler/src/index.ts b/packages/crawler/src/index.ts index d1ed0fc..461aab8 100644 --- a/packages/crawler/src/index.ts +++ b/packages/crawler/src/index.ts @@ -6,15 +6,8 @@ export { outputFilename } from "./output.ts"; export { formatRedirectsFile, redirects } from "./redirects.ts"; export type { RedirectsIntegrationOptions } from "./redirects.ts"; export { report } from "./report.ts"; -export { HINT_HEADER, announcePages, solidRouterPages, tanstackRouterPages } from "./routers.ts"; -export type { - AnnounceOptions, - SolidRouteLike, - SolidRouterLike, - SolidRouterPagesOptions, - TanStackRouteLike, - TanStackRouterLike -} from "./routers.ts"; +export { HINT_HEADER, announcePages } from "./announce.ts"; +export type { AnnounceOptions } from "./announce.ts"; export type { PrerenderReport, ReportIntegrationOptions, diff --git a/packages/crawler/src/vite.ts b/packages/crawler/src/vite.ts index c74affa..fe0b762 100644 --- a/packages/crawler/src/vite.ts +++ b/packages/crawler/src/vite.ts @@ -7,7 +7,7 @@ // framework-specific rides along as an integration (see // `PrerenderIntegration`), the same seam the engine exposes to non-Vite // drivers. Which pages exist is the server's to say — through links and -// the hint header (see ./routers.ts) — not something read off the disk. +// the hint header (see ./announce.ts) — not something read off the disk. // // Responsibilities, all build-only: // diff --git a/packages/crawler/test/announce.test.ts b/packages/crawler/test/announce.test.ts new file mode 100644 index 0000000..d9ea867 --- /dev/null +++ b/packages/crawler/test/announce.test.ts @@ -0,0 +1,49 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { announcePages } from "../src/announce.ts"; +import { runPrerender } from "../src/crawl.ts"; +import type { Transport } from "../src/types.ts"; + +describe("announcePages", () => { + it("answers only the crawler's request, on the configured header", () => { + const crawler = new Request("http://localhost/", { headers: { "x-prerender": "1" } }); + const visitor = new Request("http://localhost/"); + const headers = new Headers(); + + expect(announcePages(visitor, headers, ["/a"])).toBe(false); + expect(headers.has("x-prerender")).toBe(false); + + expect(announcePages(crawler, headers, ["/a", "/b"])).toBe(true); + expect(headers.get("x-prerender")).toBe("/a,/b"); + + expect(announcePages(crawler, new Headers(), [])).toBe(false); + + const custom = new Request("http://localhost/", { headers: { "x-pages": "1" } }); + const customHeaders = new Headers(); + expect(announcePages(custom, customHeaders, ["/a"], { header: "x-pages" })).toBe(true); + expect(customHeaders.get("x-pages")).toBe("/a"); + }); + + it("seeds a crawl end to end: the server announces its pages on the first response", async () => { + const fetched: string[] = []; + const transport: Transport = { + async fetch(request) { + const path = new URL(request.url).pathname; + fetched.push(path); + const headers = new Headers({ "content-type": "text/html" }); + announcePages(request, headers, ["/", "/unlinked"]); + return new Response(`