crawler: sitemap(), report(), keepQuery, seed normalization

.changeset/http-transport-cli-redirects.md

@@ -12,7 +12,17 @@ Prerender anything over HTTP, from the command line, with redirects the host can
 
 **`redirects()` integration.** Emits the crawl's redirects as a `_redirects` rules file (Netlify / Cloudflare Pages format; `filename`, `force`, and `format()` options for others) and declares the new `PrerenderIntegration.handlesRedirects`, which makes the engine skip its stubs — on Netlify a stub file would shadow the rule. The new `redirectStubs` option controls stubs directly.
 
-**Integration context.** `PrerenderContext` gains live, read-only `pages` and `redirects` views, complete by `teardown`.
+**`sitemap()` integration.** `sitemap({ hostname })` writes `sitemap.xml` from the pages the crawl actually rendered — dynamic routes expanded — leaving out redirect stubs, non-HTML responses, query spellings, and pages marked `noindex` via `<meta name="robots">` or `X-Robots-Tag`. Options: `filename`, `trailingSlash`, `filter(page)`, `entry(page)` for `lastmod` / `changefreq` / `priority`. `indexable()` and `formatSitemap()` are exported. CLI: `--sitemap <origin>`, `--sitemap-file`.
+
+**`report()` integration.** Writes a JSON account of the run — each page's status, content type, duration, output file, written flag and referrers; redirects; skipped pages with errors; files other integrations emitted; totals. `filename` resolves against the output directory, so `"../prerender-report.json"` keeps it out of the deploy. CLI: `--report`, `--report-file`.
+
+**Query-string pages.** `keepQuery: true` renders `/posts?page=2` apart from `/posts` (parameters sorted for dedupe), following its links and capturing its data, but writes it only when its seed entry names a `filename` — a static host serves a path the same for every query. Off by default; CLI `--keep-query`.
+
+**Seeds are normalized like links.** A seed spelled `about/`, `/a#top`, or `/posts?page=2` now meets the crawled link to the same page in one queue entry. Previously a seed with a query was fetched verbatim and written to a literal `posts?page=2/` directory.
+
+**`RenderedPage.duration`.** Milliseconds from request start to body read, pacing excluded.
+
+**Integration context.** `PrerenderContext` gains live, read-only `pages`, `redirects`, `skipped` and `files` views, complete by `teardown`. `emitFile` filenames now resolve against the output directory (`path.resolve`), so `../` and absolute paths land outside it.
 
 **Fixed:** `interval` now bounds the gap between _actual_ request starts. Previously it spaced claimed time slots, so a start delayed by a busy event loop could be followed by an on-time one less than `interval` later.
 

packages/crawler/README.md

@@ -28,6 +28,11 @@ prerender-crawler <target> --out <dir> [options]
       --hint-header <name> Response header naming extra paths. Default: x-prerender
       --redirects          Write redirects as _redirects rules instead of stubs
       --redirects-file <f> Rules file name (implies --redirects). Default: _redirects
+      --sitemap <origin>   Write sitemap.xml with entries under this public origin
+      --sitemap-file <f>   Sitemap file name. Default: sitemap.xml
+      --report             Write a JSON report of the crawl (pages, timings, referrers, ...)
+      --report-file <f>    Report file name (implies --report). Default: prerender-report.json
+      --keep-query         Render /posts?page=2 apart from /posts (see Query strings)
       --no-links           Do not follow links in rendered pages
       --no-redirect-stubs  Write no meta-refresh stubs at redirected paths
       --continue           Skip pages that fail instead of failing the run
@@ -117,6 +122,39 @@ runPrerender({ integrations: [redirects()] }); // or prerender({ integrations: [
 
 `redirects()` emits a `_redirects` file (`/from /to 301`, the format Netlify and Cloudflare Pages share) and declares `handlesRedirects`, which stops the engine writing stubs — necessary on Netlify, where an existing file shadows the rule. Options: `filename`, `force` (Netlify's `301!`), and `format(records)` for another host's syntax.
 
+### Sitemap
+
+```ts
+import { sitemap } from "prerender-crawler";
+
+runPrerender({ integrations: [sitemap({ hostname: "https://example.com" })] });
+```
+
+Every rendered page becomes a `<url>` entry — the crawl knows the one thing a route manifest cannot, which pages actually exist with dynamic segments expanded. Redirect stubs, non-HTML responses, query spellings, and pages marked `noindex` (`<meta name="robots">` in the head or an `X-Robots-Tag` header) are left out, the same signals a search engine honors on the live site. Options: `filename`, `trailingSlash`, `filter(page)` for further exclusions, and `entry(page)` returning `lastmod` / `changefreq` / `priority` per page. `indexable(page)` and `formatSitemap(entries)` are exported for tooling that formats its own.
+
+### Report
+
+```ts
+import { report } from "prerender-crawler";
+
+runPrerender({ integrations: [report({ filename: "../prerender-report.json" })] });
+```
+
+Writes what the crawl did as JSON: every page with its status, content type, duration, output file, whether it was written and which pages linked to it; every redirect; every skipped page with its error and referrers; every file other integrations emitted; and totals. It answers "why was this page crawled", "which pages are slow" and "what did the build produce" after the process is gone. The default filename lands in the output directory and deploys with the site — `../` keeps it a build artifact.
+
+### Query strings
+
+By default the query is stripped from every URL the crawl sees: `/posts`, `/posts?page=2` and `/posts?utm=x` are one page, rendered once. That is what a static host can serve — a file at a path, the same for every query.
+
+`keepQuery: true` makes each query spelling a page of its own (parameters sorted, so `?a=1&b=2` and `?b=2&a=1` meet). Each renders separately — its links are followed, its data captured — but is **written only when its seed entry names a `filename`**, because `posts/index.html` is already `/posts`. It exists for hybrid builds baking per-query data, and for sites that map queries onto files themselves:
+
+```ts
+runPrerender({
+  keepQuery: true,
+  pages: [{ path: "/posts?page=2", filename: "posts/page/2/index.html" }]
+});
+```
+
 ### Engine options
 
 | Option                   | Default                                         |                                                                                                                                                                                         |
@@ -126,6 +164,7 @@ runPrerender({ integrations: [redirects()] }); // or prerender({ integrations: [
 | `crawlLinks`             | `true`                                          | Follow same-origin links in rendered HTML. The only way dynamic routes are discovered without explicit seeding.                                                                         |
 | `hintHeader`             | `"x-prerender"`                                 | Response header naming additional paths (comma-separated) — the route the data lives on announces the routes built from it.                                                             |
 | `filter`                 |                                                 | `(path) => boolean`; drops a discovered path before it's fetched.                                                                                                                       |
+| `keepQuery`              | `false`                                         | Render query spellings as distinct pages. See [Query strings](#query-strings).                                                                                                          |
 | `concurrency`            | `8`                                             | Pages in flight at once.                                                                                                                                                                |
 | `interval`               | `0`                                             | Minimum ms between the starts of consecutive requests across all workers — a throttle for renders hitting rate-limited APIs.                                                            |
 | `retries` / `retryDelay` | `2` / `500`                                     | Re-fetch attempts for a failed page, and the wait between them.                                                                                                                         |
@@ -156,18 +195,22 @@ interface PrerenderContext {
   outDir: string;
   pages: readonly RenderedPage[]; // complete by teardown
   redirects: readonly RedirectRecord[]; // complete by teardown
+  skipped: readonly SkippedPage[]; // complete by teardown
+  files: readonly EmittedFile[]; // what earlier integrations emitted
   emitFile(file: { filename: string; contents: string | Uint8Array }): void;
 }
 ```
 
-`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. `redirects()` above is the smallest example; [`@solidjs/prerender`](../solid) is the reference integration.
+`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Filenames resolve against the output directory; `../` or an absolute path lands outside it. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. `redirects()`, `sitemap()` and `report()` above are the shipped examples, each a formatter over the context; [`@solidjs/prerender`](../solid) is the reference integration with a runtime half.
 
 ### Utilities
 
 - `httpTransport(target, { headers?, fetch? })`, `moduleTransport(entry)`, `loadHandler(entry)` — the shipped transports.
 - `redirects(options?)`, `formatRedirectsFile(records, force?)` — the redirects integration and its `_redirects` formatter.
+- `sitemap(options)`, `indexable(page)`, `formatSitemap(entries)` — the sitemap integration and its parts.
+- `report(options?)` — the crawl report integration.
 - `fileRoutePages({ root, dir, extensions })` / `staticRoutePaths(entries)` — the static page paths of a `filesystem-routing` manifest, as a `pages` source.
-- `extractLinks(html)`, `normalizeLink(href, from)`, `normalizePath(path)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives.
+- `extractLinks(html, pageUrl, { keepQuery? })`, `normalizeLink(href, base, origin)`, `normalizeRoute(url)`, `normalizePath(pathname)`, `splitRoute(route)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives.
 
 ## Requirements
 

packages/crawler/src/cli-main.ts

@@ -4,6 +4,8 @@ import path from "node:path";
 import { parseArgs } from "node:util";
 import { runPrerender } from "./crawl.ts";
 import { redirects } from "./redirects.ts";
+import { report } from "./report.ts";
+import { sitemap } from "./sitemap.ts";
 import { httpTransport, moduleTransport } from "./transports.ts";
 import type { PrerenderMode, PrerenderIntegration, Transport } from "./types.ts";
 
@@ -28,6 +30,16 @@ Options:
       --redirects          Write the redirects as host rules (_redirects format,
                            Netlify / Cloudflare Pages) instead of meta-refresh stubs
       --redirects-file <f> Rules file name (implies --redirects). Default: _redirects
+      --sitemap <origin>   Write sitemap.xml with entries under this public origin
+                           (https://example.com)
+      --sitemap-file <f>   Sitemap file name. Default: sitemap.xml
+      --report             Write a JSON report of the crawl (pages, timings, referrers,
+                           redirects, skips)
+      --report-file <f>    Report file name (implies --report). Resolved against the
+                           output dir; ../ keeps it out of the deploy.
+                           Default: prerender-report.json
+      --keep-query         Render /posts?page=2 apart from /posts (not written unless
+                           the app maps queries to files — see docs)
       --no-links           Do not follow links in rendered pages
       --no-redirect-stubs  Write no meta-refresh stubs at redirected paths
       --continue           Skip pages that fail instead of failing the run
@@ -88,6 +100,16 @@ export async function main(argv: string[], io: CliIO): Promise<number> {
   if (values.redirects || values["redirects-file"] !== undefined) {
     integrations.push(redirects({ filename: values["redirects-file"] }));
   }
+  if (values.sitemap !== undefined || values["sitemap-file"] !== undefined) {
+    if (!values.sitemap || !/^https?:\/\//.test(values.sitemap)) {
+      io.stderr(`--sitemap needs the site's public origin (https://example.com).`);
+      return 2;
+    }
+    integrations.push(sitemap({ hostname: values.sitemap, filename: values["sitemap-file"] }));
+  }
+  if (values.report || values["report-file"] !== undefined) {
+    integrations.push(report({ filename: values["report-file"] }));
+  }
 
   const outDir = path.resolve(values.out);
   try {
@@ -102,6 +124,7 @@ export async function main(argv: string[], io: CliIO): Promise<number> {
       retries: values.retries !== undefined ? integer(values.retries) : undefined,
       hintHeader: values["hint-header"],
       crawlLinks: !values["no-links"],
+      keepQuery: values["keep-query"],
       redirectStubs: values["no-redirect-stubs"] ? false : undefined,
       failOnError: !values.continue,
       autoSubfolderIndex: !values.flat,
@@ -137,6 +160,11 @@ const spec = {
     "hint-header": { type: "string" },
     redirects: { type: "boolean" },
     "redirects-file": { type: "string" },
+    sitemap: { type: "string" },
+    "sitemap-file": { type: "string" },
+    report: { type: "boolean" },
+    "report-file": { type: "string" },
+    "keep-query": { type: "boolean" },
     "no-links": { type: "boolean" },
     "no-redirect-stubs": { type: "boolean" },
     continue: { type: "boolean" },