diff --git a/.changeset/http-transport-cli-redirects.md b/.changeset/http-transport-cli-redirects.md
index ac1bc01..72e4315 100644
--- a/.changeset/http-transport-cli-redirects.md
+++ b/.changeset/http-transport-cli-redirects.md
@@ -12,7 +12,17 @@ Prerender anything over HTTP, from the command line, with redirects the host can
**`redirects()` integration.** Emits the crawl's redirects as a `_redirects` rules file (Netlify / Cloudflare Pages format; `filename`, `force`, and `format()` options for others) and declares the new `PrerenderIntegration.handlesRedirects`, which makes the engine skip its stubs — on Netlify a stub file would shadow the rule. The new `redirectStubs` option controls stubs directly.
-**Integration context.** `PrerenderContext` gains live, read-only `pages` and `redirects` views, complete by `teardown`.
+**`sitemap()` integration.** `sitemap({ hostname })` writes `sitemap.xml` from the pages the crawl actually rendered — dynamic routes expanded — leaving out redirect stubs, non-HTML responses, query spellings, and pages marked `noindex` via `` or `X-Robots-Tag`. Options: `filename`, `trailingSlash`, `filter(page)`, `entry(page)` for `lastmod` / `changefreq` / `priority`. `indexable()` and `formatSitemap()` are exported. CLI: `--sitemap `, `--sitemap-file`.
+
+**`report()` integration.** Writes a JSON account of the run — each page's status, content type, duration, output file, written flag and referrers; redirects; skipped pages with errors; files other integrations emitted; totals. `filename` resolves against the output directory, so `"../prerender-report.json"` keeps it out of the deploy. CLI: `--report`, `--report-file`.
+
+**Query-string pages.** `keepQuery: true` renders `/posts?page=2` apart from `/posts` (parameters sorted for dedupe), following its links and capturing its data, but writes it only when its seed entry names a `filename` — a static host serves a path the same for every query. Off by default; CLI `--keep-query`.
+
+**Seeds are normalized like links.** A seed spelled `about/`, `/a#top`, or `/posts?page=2` now meets the crawled link to the same page in one queue entry. Previously a seed with a query was fetched verbatim and written to a literal `posts?page=2/` directory.
+
+**`RenderedPage.duration`.** Milliseconds from request start to body read, pacing excluded.
+
+**Integration context.** `PrerenderContext` gains live, read-only `pages`, `redirects`, `skipped` and `files` views, complete by `teardown`. `emitFile` filenames now resolve against the output directory (`path.resolve`), so `../` and absolute paths land outside it.
**Fixed:** `interval` now bounds the gap between _actual_ request starts. Previously it spaced claimed time slots, so a start delayed by a busy event loop could be followed by an on-time one less than `interval` later.
diff --git a/packages/crawler/README.md b/packages/crawler/README.md
index b06dfd7..531a04a 100644
--- a/packages/crawler/README.md
+++ b/packages/crawler/README.md
@@ -28,6 +28,11 @@ prerender-crawler --out [options]
--hint-header Response header naming extra paths. Default: x-prerender
--redirects Write redirects as _redirects rules instead of stubs
--redirects-file Rules file name (implies --redirects). Default: _redirects
+ --sitemap Write sitemap.xml with entries under this public origin
+ --sitemap-file Sitemap file name. Default: sitemap.xml
+ --report Write a JSON report of the crawl (pages, timings, referrers, ...)
+ --report-file Report file name (implies --report). Default: prerender-report.json
+ --keep-query Render /posts?page=2 apart from /posts (see Query strings)
--no-links Do not follow links in rendered pages
--no-redirect-stubs Write no meta-refresh stubs at redirected paths
--continue Skip pages that fail instead of failing the run
@@ -117,6 +122,39 @@ runPrerender({ integrations: [redirects()] }); // or prerender({ integrations: [
`redirects()` emits a `_redirects` file (`/from /to 301`, the format Netlify and Cloudflare Pages share) and declares `handlesRedirects`, which stops the engine writing stubs — necessary on Netlify, where an existing file shadows the rule. Options: `filename`, `force` (Netlify's `301!`), and `format(records)` for another host's syntax.
+### Sitemap
+
+```ts
+import { sitemap } from "prerender-crawler";
+
+runPrerender({ integrations: [sitemap({ hostname: "https://example.com" })] });
+```
+
+Every rendered page becomes a `` entry — the crawl knows the one thing a route manifest cannot, which pages actually exist with dynamic segments expanded. Redirect stubs, non-HTML responses, query spellings, and pages marked `noindex` (`` in the head or an `X-Robots-Tag` header) are left out, the same signals a search engine honors on the live site. Options: `filename`, `trailingSlash`, `filter(page)` for further exclusions, and `entry(page)` returning `lastmod` / `changefreq` / `priority` per page. `indexable(page)` and `formatSitemap(entries)` are exported for tooling that formats its own.
+
+### Report
+
+```ts
+import { report } from "prerender-crawler";
+
+runPrerender({ integrations: [report({ filename: "../prerender-report.json" })] });
+```
+
+Writes what the crawl did as JSON: every page with its status, content type, duration, output file, whether it was written and which pages linked to it; every redirect; every skipped page with its error and referrers; every file other integrations emitted; and totals. It answers "why was this page crawled", "which pages are slow" and "what did the build produce" after the process is gone. The default filename lands in the output directory and deploys with the site — `../` keeps it a build artifact.
+
+### Query strings
+
+By default the query is stripped from every URL the crawl sees: `/posts`, `/posts?page=2` and `/posts?utm=x` are one page, rendered once. That is what a static host can serve — a file at a path, the same for every query.
+
+`keepQuery: true` makes each query spelling a page of its own (parameters sorted, so `?a=1&b=2` and `?b=2&a=1` meet). Each renders separately — its links are followed, its data captured — but is **written only when its seed entry names a `filename`**, because `posts/index.html` is already `/posts`. It exists for hybrid builds baking per-query data, and for sites that map queries onto files themselves:
+
+```ts
+runPrerender({
+ keepQuery: true,
+ pages: [{ path: "/posts?page=2", filename: "posts/page/2/index.html" }]
+});
+```
+
### Engine options
| Option | Default | |
@@ -126,6 +164,7 @@ runPrerender({ integrations: [redirects()] }); // or prerender({ integrations: [
| `crawlLinks` | `true` | Follow same-origin links in rendered HTML. The only way dynamic routes are discovered without explicit seeding. |
| `hintHeader` | `"x-prerender"` | Response header naming additional paths (comma-separated) — the route the data lives on announces the routes built from it. |
| `filter` | | `(path) => boolean`; drops a discovered path before it's fetched. |
+| `keepQuery` | `false` | Render query spellings as distinct pages. See [Query strings](#query-strings). |
| `concurrency` | `8` | Pages in flight at once. |
| `interval` | `0` | Minimum ms between the starts of consecutive requests across all workers — a throttle for renders hitting rate-limited APIs. |
| `retries` / `retryDelay` | `2` / `500` | Re-fetch attempts for a failed page, and the wait between them. |
@@ -156,18 +195,22 @@ interface PrerenderContext {
outDir: string;
pages: readonly RenderedPage[]; // complete by teardown
redirects: readonly RedirectRecord[]; // complete by teardown
+ skipped: readonly SkippedPage[]; // complete by teardown
+ files: readonly EmittedFile[]; // what earlier integrations emitted
emitFile(file: { filename: string; contents: string | Uint8Array }): void;
}
```
-`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. `redirects()` above is the smallest example; [`@solidjs/prerender`](../solid) is the reference integration.
+`emitFile` is the channel for artifacts produced during the crawl — captured server-function results, extracted payloads, sitemaps. Filenames resolve against the output directory; `../` or an absolute path lands outside it. Throwing from `teardown` fails the run: the place to verify the crawl produced everything the runtime half will need. `redirects()`, `sitemap()` and `report()` above are the shipped examples, each a formatter over the context; [`@solidjs/prerender`](../solid) is the reference integration with a runtime half.
### Utilities
- `httpTransport(target, { headers?, fetch? })`, `moduleTransport(entry)`, `loadHandler(entry)` — the shipped transports.
- `redirects(options?)`, `formatRedirectsFile(records, force?)` — the redirects integration and its `_redirects` formatter.
+- `sitemap(options)`, `indexable(page)`, `formatSitemap(entries)` — the sitemap integration and its parts.
+- `report(options?)` — the crawl report integration.
- `fileRoutePages({ root, dir, extensions })` / `staticRoutePaths(entries)` — the static page paths of a `filesystem-routing` manifest, as a `pages` source.
-- `extractLinks(html)`, `normalizeLink(href, from)`, `normalizePath(path)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives.
+- `extractLinks(html, pageUrl, { keepQuery? })`, `normalizeLink(href, base, origin)`, `normalizeRoute(url)`, `normalizePath(pathname)`, `splitRoute(route)`, `outputFilename(path, autoSubfolderIndex)` — the crawl's own primitives.
## Requirements
diff --git a/packages/crawler/src/cli-main.ts b/packages/crawler/src/cli-main.ts
index 4fef73c..693bf26 100644
--- a/packages/crawler/src/cli-main.ts
+++ b/packages/crawler/src/cli-main.ts
@@ -4,6 +4,8 @@ import path from "node:path";
import { parseArgs } from "node:util";
import { runPrerender } from "./crawl.ts";
import { redirects } from "./redirects.ts";
+import { report } from "./report.ts";
+import { sitemap } from "./sitemap.ts";
import { httpTransport, moduleTransport } from "./transports.ts";
import type { PrerenderMode, PrerenderIntegration, Transport } from "./types.ts";
@@ -28,6 +30,16 @@ Options:
--redirects Write the redirects as host rules (_redirects format,
Netlify / Cloudflare Pages) instead of meta-refresh stubs
--redirects-file Rules file name (implies --redirects). Default: _redirects
+ --sitemap Write sitemap.xml with entries under this public origin
+ (https://example.com)
+ --sitemap-file Sitemap file name. Default: sitemap.xml
+ --report Write a JSON report of the crawl (pages, timings, referrers,
+ redirects, skips)
+ --report-file Report file name (implies --report). Resolved against the
+ output dir; ../ keeps it out of the deploy.
+ Default: prerender-report.json
+ --keep-query Render /posts?page=2 apart from /posts (not written unless
+ the app maps queries to files — see docs)
--no-links Do not follow links in rendered pages
--no-redirect-stubs Write no meta-refresh stubs at redirected paths
--continue Skip pages that fail instead of failing the run
@@ -88,6 +100,16 @@ export async function main(argv: string[], io: CliIO): Promise {
if (values.redirects || values["redirects-file"] !== undefined) {
integrations.push(redirects({ filename: values["redirects-file"] }));
}
+ if (values.sitemap !== undefined || values["sitemap-file"] !== undefined) {
+ if (!values.sitemap || !/^https?:\/\//.test(values.sitemap)) {
+ io.stderr(`--sitemap needs the site's public origin (https://example.com).`);
+ return 2;
+ }
+ integrations.push(sitemap({ hostname: values.sitemap, filename: values["sitemap-file"] }));
+ }
+ if (values.report || values["report-file"] !== undefined) {
+ integrations.push(report({ filename: values["report-file"] }));
+ }
const outDir = path.resolve(values.out);
try {
@@ -102,6 +124,7 @@ export async function main(argv: string[], io: CliIO): Promise {
retries: values.retries !== undefined ? integer(values.retries) : undefined,
hintHeader: values["hint-header"],
crawlLinks: !values["no-links"],
+ keepQuery: values["keep-query"],
redirectStubs: values["no-redirect-stubs"] ? false : undefined,
failOnError: !values.continue,
autoSubfolderIndex: !values.flat,
@@ -137,6 +160,11 @@ const spec = {
"hint-header": { type: "string" },
redirects: { type: "boolean" },
"redirects-file": { type: "string" },
+ sitemap: { type: "string" },
+ "sitemap-file": { type: "string" },
+ report: { type: "boolean" },
+ "report-file": { type: "string" },
+ "keep-query": { type: "boolean" },
"no-links": { type: "boolean" },
"no-redirect-stubs": { type: "boolean" },
continue: { type: "boolean" },
diff --git a/packages/crawler/src/crawl.ts b/packages/crawler/src/crawl.ts
index 3546ccc..492e23a 100644
--- a/packages/crawler/src/crawl.ts
+++ b/packages/crawler/src/crawl.ts
@@ -1,6 +1,7 @@
import { mkdir, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { extractLinks, normalizeLink, normalizePath } from "./links.ts";
+import { dirname, resolve } from "node:path";
+import { extractLinks, normalizeLink, normalizeRoute, splitRoute } from "./links.ts";
+import type { LinkOptions } from "./links.ts";
import { outputFilename } from "./output.ts";
import type {
EmittedFile,
@@ -20,16 +21,22 @@ export interface RunOptions extends PrerenderOptions {
const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-async function resolveSeeds(pages: PrerenderOptions["pages"]): Promise {
+async function resolveSeeds(
+ pages: PrerenderOptions["pages"],
+ origin: URL,
+ links: LinkOptions
+): Promise {
const source = typeof pages === "function" ? await pages() : (pages ?? ["/"]);
- // Seed sources overlap routinely (an explicit list plus a route-manifest
- // scan both naming `/`): one render per path, the first spelling wins.
+ // Seeds are spelled by people and route manifests — `about/`, `/a#top`,
+ // `/posts?page=2` — and get the same normalization a crawled link does,
+ // so a seed and a link to the same page meet in one queue entry. Seed
+ // sources overlap routinely (an explicit list plus a route-manifest scan
+ // both naming `/`): one render per path, the first spelling wins.
const byPath = new Map();
for (const entry of source) {
- const page =
- typeof entry === "string"
- ? { path: normalizePath(entry) }
- : { ...entry, path: normalizePath(entry.path) };
+ const spelled = typeof entry === "string" ? entry : entry.path;
+ const path = normalizeRoute(new URL(spelled, origin), links);
+ const page = typeof entry === "string" ? { path } : { ...entry, path };
if (!byPath.has(page.path)) byPath.set(page.path, page);
}
return [...byPath.values()];
@@ -49,6 +56,7 @@ export async function runPrerender(options: RunOptions): Promise void emitted.push(file)
};
- const shouldEmit = (entry: PageEntry) =>
- entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages);
+ // A query spelling without a filename of its own has nowhere correct to
+ // go: `posts/index.html` is `/posts`, and a static host serves it for
+ // every query. It renders (data, links) and leaves no file.
+ const shouldEmit = (entry: PageEntry) => {
+ if (!entry.filename && entry.path.includes("?")) return false;
+ return entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages);
+ };
- const seeds = await resolveSeeds(options.pages);
+ const seeds = await resolveSeeds(options.pages, originUrl, links);
const seen = new Set(seeds.map(page => page.path));
const queue: PageEntry[] = [...seeds];
// Provenance: which pages named each discovered path. Recorded for every
@@ -131,43 +147,50 @@ export async function runPrerender(options: RunOptions): Promise {
+ // `started` is taken after pacing: a page's duration is its own, not the
+ // throttle's.
+ async function fetchUrl(url: URL): Promise {
await pace();
- return transport.fetch(
+ const started = performance.now();
+ const response = await transport.fetch(
new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } })
);
+ return { response, started };
}
// A redirect to a spelling of the SAME page (`/posts` -> `/posts/`, the
// trailing-slash canonicalization static servers do) is not a redirect
// between pages: it is followed here, once, and the page renders as
// itself. Anything else is the caller's to record.
- async function fetchPage(path: string): Promise {
+ async function fetchPage(path: string): Promise {
const url = new URL(path, originUrl);
- const response = await fetchUrl(url);
+ const fetched = await fetchUrl(url);
+ const { response } = fetched;
const location = response.headers.get("location");
- if (!location || response.status < 300 || response.status >= 400) return response;
+ if (!location || response.status < 300 || response.status >= 400) return fetched;
const target = new URL(location, url);
- if (target.origin !== originUrl.origin || normalizePath(target.pathname) !== path) {
- return response;
+ if (target.origin !== originUrl.origin || normalizeRoute(target, links) !== path) {
+ return fetched;
}
- return fetchUrl(target);
+ return { ...(await fetchUrl(target)), started: fetched.started };
}
const pendingRedirects: Array<{
entry: PageEntry;
filename: string;
response: Response;
+ duration: number;
redirect: RedirectRecord;
}> = [];
async function renderPage(entry: PageEntry): Promise {
let response: Response | undefined;
+ let started = 0;
let error: unknown;
for (let attempt = 0; attempt <= retries; attempt++) {
if (attempt > 0) await wait(retryDelay);
try {
- response = await fetchPage(entry.path);
+ ({ response, started } = await fetchPage(entry.path));
error = undefined;
if (response.status < 500) break; // retry only what might heal
} catch (thrown) {
@@ -192,11 +215,12 @@ export async function runPrerender(options: RunOptions): Promise (error instanceof Error ? error.message : String(error));
function redirectStub(location: string): string {
@@ -328,7 +362,7 @@ function redirectStub(location: string): string {
}
async function writeOutput(outDir: string, filename: string, contents: string | Uint8Array) {
- const target = join(outDir, filename);
+ const target = resolve(outDir, filename);
await mkdir(dirname(target), { recursive: true });
await writeFile(target, contents);
}
diff --git a/packages/crawler/src/index.ts b/packages/crawler/src/index.ts
index c34ab06..264be26 100644
--- a/packages/crawler/src/index.ts
+++ b/packages/crawler/src/index.ts
@@ -1,9 +1,19 @@
export { runPrerender } from "./crawl.ts";
export type { RunOptions } from "./crawl.ts";
-export { extractLinks, normalizeLink, normalizePath } from "./links.ts";
+export { extractLinks, normalizeLink, normalizePath, normalizeRoute, splitRoute } from "./links.ts";
+export type { LinkOptions } from "./links.ts";
export { outputFilename } from "./output.ts";
export { formatRedirectsFile, redirects } from "./redirects.ts";
export type { RedirectsIntegrationOptions } from "./redirects.ts";
+export { report } from "./report.ts";
+export type {
+ PrerenderReport,
+ ReportIntegrationOptions,
+ ReportPage,
+ ReportSkip
+} from "./report.ts";
+export { formatSitemap, indexable, sitemap } from "./sitemap.ts";
+export type { SitemapChangeFrequency, SitemapEntry, SitemapIntegrationOptions } from "./sitemap.ts";
export { httpTransport, loadHandler, moduleTransport } from "./transports.ts";
export type { HttpTransportOptions, RequestHandler } from "./transports.ts";
export type {
diff --git a/packages/crawler/src/links.ts b/packages/crawler/src/links.ts
index c57656f..cc74701 100644
--- a/packages/crawler/src/links.ts
+++ b/packages/crawler/src/links.ts
@@ -2,8 +2,8 @@
* Link discovery: which hrefs in a rendered page name more pages of this
* site. The rules here are correctness fixes other prerenderers earned one
* bug report at a time — resolve relative hrefs against the PAGE's URL
- * (not the origin), honor , strip queries and fragments before
- * dedupe, and never leave the origin.
+ * (not the origin), honor , strip queries (unless asked to keep
+ * them) and fragments before dedupe, and never leave the origin.
*/
const LINK_PATTERN = /]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/gis;
@@ -12,12 +12,17 @@ const BASE_PATTERN = /]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/is;
/** Schemes that are never pages. */
const NON_PAGE_SCHEME = /^(?:mailto|tel|javascript|data|blob|about):/i;
+export interface LinkOptions {
+ /** Keep the query string as part of the page's identity. @default false */
+ keepQuery?: boolean;
+}
+
/**
* Extracts the crawlable same-origin route paths from a page's HTML.
- * Returned paths are normalized (`/about`, no query, no fragment, no
- * trailing slash except the root) and deduped.
+ * Returned paths are normalized (`/about`, no fragment, no trailing slash
+ * except the root, query only with `keepQuery`) and deduped.
*/
-export function extractLinks(html: string, pageUrl: URL): string[] {
+export function extractLinks(html: string, pageUrl: URL, options: LinkOptions = {}): string[] {
// shifts what relative hrefs resolve against, exactly as the
// browser would resolve them.
const baseMatch = BASE_PATTERN.exec(html);
@@ -35,7 +40,7 @@ export function extractLinks(html: string, pageUrl: URL): string[] {
for (const match of html.matchAll(LINK_PATTERN)) {
const href = (match[1] ?? match[2] ?? "").trim();
if (!href || NON_PAGE_SCHEME.test(href)) continue;
- const path = normalizeLink(href, base, pageUrl.origin);
+ const path = normalizeLink(href, base, pageUrl.origin, options);
if (path !== undefined) found.add(path);
}
return [...found];
@@ -45,7 +50,12 @@ export function extractLinks(html: string, pageUrl: URL): string[] {
* Resolves one href to a normalized same-origin path, or undefined when it
* is not a page of this site (foreign origin, unparseable).
*/
-export function normalizeLink(href: string, base: URL, origin: string): string | undefined {
+export function normalizeLink(
+ href: string,
+ base: URL,
+ origin: string,
+ options: LinkOptions = {}
+): string | undefined {
let url: URL;
try {
url = new URL(href, base);
@@ -53,15 +63,38 @@ export function normalizeLink(href: string, base: URL, origin: string): string |
return undefined;
}
if (url.origin !== origin) return undefined;
- return normalizePath(url.pathname);
+ return normalizeRoute(url, options);
+}
+
+/**
+ * One spelling per page: the fragment never reaches here (URL parsing split
+ * it off), the trailing slash is dropped (except the root), percent-encoding
+ * is left exactly as the URL parser produced it, and the query is kept only
+ * on request — with its parameters sorted, so `?a=1&b=2` and `?b=2&a=1`
+ * are the one page they are.
+ */
+export function normalizeRoute(url: URL, options: LinkOptions = {}): string {
+ const path = normalizePath(url.pathname);
+ if (!options.keepQuery || !url.search) return path;
+ const params = new URLSearchParams(url.search);
+ params.sort();
+ const query = params.toString();
+ return query ? `${path}?${query}` : path;
}
/**
- * One spelling per page: query and fragment never reach here (URL parsing
- * split them off), the trailing slash is dropped (except the root), and
- * percent-encoding is left exactly as the URL parser produced it.
+ * The pathname half of normalization: the trailing slash is dropped (except
+ * the root) and nothing else is touched.
*/
export function normalizePath(pathname: string): string {
if (pathname === "" || pathname === "/") return "/";
return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
}
+
+/** Splits a normalized route into its pathname and its (possibly empty) query. */
+export function splitRoute(route: string): { pathname: string; search: string } {
+ const at = route.indexOf("?");
+ return at === -1
+ ? { pathname: route, search: "" }
+ : { pathname: route.slice(0, at), search: route.slice(at) };
+}
diff --git a/packages/crawler/src/report.ts b/packages/crawler/src/report.ts
new file mode 100644
index 0000000..cefc063
--- /dev/null
+++ b/packages/crawler/src/report.ts
@@ -0,0 +1,108 @@
+import type { PrerenderIntegration, RedirectRecord } from "./types.ts";
+
+/**
+ * The report integration: what the crawl did, as JSON — every page with
+ * its status, timing, output file and the pages that linked to it; every
+ * redirect; every skipped page and why; every file other integrations
+ * emitted. The answers to "why was this page crawled", "which page is
+ * slow" and "what did the build actually produce" are all in here, and
+ * they are otherwise gone the moment the process exits.
+ */
+
+export interface ReportIntegrationOptions {
+ /**
+ * Where to write, resolved against the output directory. The default
+ * lands inside it and so deploys with the site; point it outside
+ * (`"../prerender-report.json"`) to keep it a build artifact.
+ * @default "prerender-report.json"
+ */
+ filename?: string;
+}
+
+export interface PrerenderReport {
+ generatedAt: string;
+ mode: string;
+ origin: string;
+ totals: {
+ pages: number;
+ written: number;
+ redirects: number;
+ skipped: number;
+ files: number;
+ /** Sum of page durations, milliseconds — render cost, not wall time. */
+ duration: number;
+ };
+ pages: ReportPage[];
+ redirects: RedirectRecord[];
+ skipped: ReportSkip[];
+ /** Files emitted by integrations that ran before this one, this run's pages excluded. */
+ files: string[];
+}
+
+export interface ReportPage {
+ path: string;
+ status: number;
+ contentType: string | null;
+ /** Milliseconds, the successful attempt. */
+ duration: number;
+ filename: string;
+ /** Whether the HTML was written (`false` in hybrid mode, for query spellings, ...). */
+ written: boolean;
+ referrers: string[];
+ redirect?: RedirectRecord;
+}
+
+export interface ReportSkip {
+ path: string;
+ error: string;
+ referrers: string[];
+}
+
+export function report(options: ReportIntegrationOptions = {}): PrerenderIntegration {
+ const { filename = "prerender-report.json" } = options;
+ return {
+ name: "report",
+ teardown(context) {
+ const pages = context.pages
+ .map(page => ({
+ path: page.path,
+ status: page.response.status,
+ contentType: page.response.headers.get("content-type"),
+ duration: Math.round(page.duration * 100) / 100,
+ filename: page.filename,
+ written: page.emitted,
+ referrers: [...page.referrers].sort(),
+ ...(page.redirect ? { redirect: page.redirect } : {})
+ }))
+ .sort(byPath);
+ const skipped = context.skipped
+ .map(miss => ({
+ path: miss.path,
+ error: miss.error instanceof Error ? miss.error.message : String(miss.error),
+ referrers: [...miss.referrers].sort()
+ }))
+ .sort(byPath);
+ const summary: PrerenderReport = {
+ generatedAt: new Date().toISOString(),
+ mode: context.mode,
+ origin: context.origin,
+ totals: {
+ pages: pages.length,
+ written: pages.filter(page => page.written).length,
+ redirects: context.redirects.length,
+ skipped: skipped.length,
+ files: context.files.length,
+ duration: Math.round(pages.reduce((sum, page) => sum + page.duration, 0))
+ },
+ pages,
+ redirects: [...context.redirects],
+ skipped,
+ files: context.files.map(file => file.filename)
+ };
+ context.emitFile({ filename, contents: JSON.stringify(summary, null, 2) + "\n" });
+ }
+ };
+}
+
+const byPath = (a: { path: string }, b: { path: string }) =>
+ a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
diff --git a/packages/crawler/src/sitemap.ts b/packages/crawler/src/sitemap.ts
new file mode 100644
index 0000000..8af5379
--- /dev/null
+++ b/packages/crawler/src/sitemap.ts
@@ -0,0 +1,144 @@
+import type { PrerenderIntegration, RenderedPage } from "./types.ts";
+
+/**
+ * The sitemap integration: every rendered page becomes a `` entry.
+ *
+ * The crawl already knows the one thing a sitemap needs and a route
+ * manifest cannot supply — which pages actually exist, dynamic segments
+ * expanded — so the integration is a formatter over `context.pages`.
+ * Pages are dropped when they are redirects, are not HTML, carry a query
+ * (a static host cannot serve them, see `keepQuery`), or ask not to be
+ * indexed (`` or an `X-Robots-Tag`
+ * header) — the same signals a search engine would honor on the live site.
+ */
+
+export type SitemapChangeFrequency =
+ "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
+
+export interface SitemapEntry {
+ /** Absolute URL of the page. */
+ loc: string;
+ /** Last-modification date, as a `Date` or an already-formatted W3C datetime. */
+ lastmod?: Date | string;
+ changefreq?: SitemapChangeFrequency;
+ /** 0.0 – 1.0 */
+ priority?: number;
+}
+
+export interface SitemapIntegrationOptions {
+ /**
+ * The site's public origin — `https://example.com`. Sitemap entries are
+ * absolute URLs, and the crawl only knows the loopback origin it
+ * rendered against.
+ */
+ hostname: string;
+ /** @default "sitemap.xml" */
+ filename?: string;
+ /**
+ * Spell entries with a trailing slash (`/about/`) — for hosts that
+ * canonicalize that way. The root is `/` either way.
+ * @default false
+ */
+ trailingSlash?: boolean;
+ /**
+ * Which pages are listed. Runs after the built-in exclusions (redirects,
+ * non-HTML, query spellings, `noindex`); return false to drop more.
+ */
+ filter?(page: RenderedPage): boolean;
+ /**
+ * Per-page metadata — `lastmod`, `changefreq`, `priority` — or a
+ * replacement `loc`. Return nothing to list the page with its URL alone.
+ */
+ entry?(page: RenderedPage): Partial | void;
+}
+
+export function sitemap(options: SitemapIntegrationOptions): PrerenderIntegration {
+ const { filename = "sitemap.xml", trailingSlash = false, filter, entry } = options;
+ if (!options.hostname) {
+ throw new Error("sitemap(): `hostname` is required — entries must be absolute URLs");
+ }
+ const hostname = new URL(options.hostname);
+ return {
+ name: "sitemap",
+ teardown(context) {
+ const entries: SitemapEntry[] = [];
+ for (const page of context.pages) {
+ if (!indexable(page) || (filter && !filter(page))) continue;
+ const path = trailingSlash && page.path !== "/" ? `${page.path}/` : page.path;
+ entries.push({ loc: new URL(path, hostname).href, ...entry?.(page) });
+ }
+ entries.sort((a, b) => (a.loc < b.loc ? -1 : a.loc > b.loc ? 1 : 0));
+ context.emitFile({ filename, contents: formatSitemap(entries) });
+ }
+ };
+}
+
+/**
+ * Whether a rendered page belongs in a sitemap by the signals the page
+ * itself gives: an HTML document, not a redirect, not a query spelling,
+ * not marked `noindex`.
+ */
+export function indexable(page: RenderedPage): boolean {
+ if (page.redirect || page.path.includes("?")) return false;
+ if (!(page.response.headers.get("content-type") ?? "").includes("text/html")) return false;
+ if (/\bnoindex\b/i.test(page.response.headers.get("x-robots-tag") ?? "")) return false;
+ return !robotsMeta(page.html).some(content => /\bnoindex\b/i.test(content));
+}
+
+const META_PATTERN = /]*>/gi;
+const ATTRIBUTE = (name: string) =>
+ new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i");
+const NAME = ATTRIBUTE("name");
+const CONTENT = ATTRIBUTE("content");
+
+/** The `content` of every `` in the document's head, any attribute order. */
+function robotsMeta(html: string): string[] {
+ const head = html.slice(0, headEnd(html));
+ const found: string[] = [];
+ for (const [tag] of head.matchAll(META_PATTERN)) {
+ const name = NAME.exec(tag);
+ if (!name || (name[1] ?? name[2] ?? name[3]).trim().toLowerCase() !== "robots") continue;
+ const content = CONTENT.exec(tag);
+ if (content) found.push(content[1] ?? content[2] ?? content[3]);
+ }
+ return found;
+}
+
+function headEnd(html: string): number {
+ const at = html.search(/<\/head\s*>|'];
+ lines.push('');
+ for (const entry of entries) {
+ lines.push(" ");
+ lines.push(` ${escapeXml(entry.loc)}`);
+ if (entry.lastmod !== undefined) {
+ const lastmod =
+ entry.lastmod instanceof Date ? entry.lastmod.toISOString() : String(entry.lastmod);
+ lines.push(` ${escapeXml(lastmod)}`);
+ }
+ if (entry.changefreq) lines.push(` ${entry.changefreq}`);
+ if (entry.priority !== undefined) {
+ lines.push(` ${clampPriority(entry.priority)}`);
+ }
+ lines.push(" ");
+ }
+ lines.push("");
+ return lines.join("\n") + "\n";
+}
+
+function clampPriority(priority: number): string {
+ return Math.min(1, Math.max(0, priority)).toFixed(1);
+}
+
+function escapeXml(value: string): string {
+ return value
+ .replace(/&/g, "&")
+ .replace(//g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+}
diff --git a/packages/crawler/src/types.ts b/packages/crawler/src/types.ts
index cb532bf..3f32e3a 100644
--- a/packages/crawler/src/types.ts
+++ b/packages/crawler/src/types.ts
@@ -70,6 +70,8 @@ export interface RenderedPage {
emitted: boolean;
/** The response the transport answered with (body consumed). */
response: Response;
+ /** Milliseconds from the request's start to its body fully read (the successful attempt). */
+ duration: number;
/**
* The rendered HTML — or, for a redirected path, the meta-refresh stub
* that stands in for it (see `redirect`).
@@ -116,6 +118,15 @@ export interface PrerenderContext {
pages: readonly RenderedPage[];
/** Every redirect observed so far — complete by `teardown`. Live view; do not mutate. */
redirects: readonly RedirectRecord[];
+ /** Pages that failed and were skipped (`failOnError: false`) — complete by `teardown`. */
+ skipped: readonly SkippedPage[];
+ /** Files emitted so far by integrations — those before this one, at `teardown`. */
+ files: readonly EmittedFile[];
+ /**
+ * Queues a file to be written with the pages. `filename` is resolved
+ * against the output directory; a `../` or absolute path lands outside
+ * it (a build report that should not deploy, say).
+ */
emitFile(file: EmittedFile): void;
}
@@ -167,6 +178,18 @@ export interface PrerenderOptions {
hintHeader?: string;
/** Drops a discovered path before it is fetched. */
filter?(path: string): boolean;
+ /**
+ * Treat `/posts?page=2` as a page distinct from `/posts`. Off, the query
+ * is stripped everywhere and one render stands for every spelling. On,
+ * each query spelling renders separately — its links are followed and
+ * its data captured — but is written only when its entry names a
+ * `filename`: a static host serves a path the same regardless of query,
+ * so there is nothing correct to write by default. Meant for hybrid
+ * builds baking per-query data, and for sites that map queries to
+ * files themselves.
+ * @default false
+ */
+ keepQuery?: boolean;
/** Pages in flight at once. @default 8 */
concurrency?: number;
/**
diff --git a/packages/crawler/src/vite.ts b/packages/crawler/src/vite.ts
index 5845f46..cc70b68 100644
--- a/packages/crawler/src/vite.ts
+++ b/packages/crawler/src/vite.ts
@@ -35,6 +35,15 @@ export { fileRoutePages, staticRoutePaths } from "./file-routes.ts";
export type { FileRoutePagesOptions, RouteEntryLike } from "./file-routes.ts";
export { redirects } from "./redirects.ts";
export type { RedirectsIntegrationOptions } from "./redirects.ts";
+export { report } from "./report.ts";
+export type {
+ PrerenderReport,
+ ReportIntegrationOptions,
+ ReportPage,
+ ReportSkip
+} from "./report.ts";
+export { sitemap } from "./sitemap.ts";
+export type { SitemapChangeFrequency, SitemapEntry, SitemapIntegrationOptions } from "./sitemap.ts";
export type * from "./types.ts";
/** The `import.meta.env` key the plugin defines with the build's `PrerenderMode`. */
diff --git a/packages/crawler/test/cli.test.ts b/packages/crawler/test/cli.test.ts
index 7aa8928..849797e 100644
--- a/packages/crawler/test/cli.test.ts
+++ b/packages/crawler/test/cli.test.ts
@@ -121,6 +121,47 @@ describe("cli", () => {
expect(lenient.err[0]).toMatch(/skipped \/missing/);
});
+ it("writes a sitemap and a report when asked, and validates the sitemap origin", async () => {
+ const server = join(dir, "server-seo.mjs");
+ await writeFile(
+ server,
+ `export default { fetch(request) {
+ const path = new URL(request.url).pathname;
+ if (path === "/") return new Response('a', { headers: { "content-type": "text/html" } });
+ if (path === "/about") return new Response("about
", { headers: { "content-type": "text/html" } });
+ return new Response("nope", { status: 404 });
+ } }`
+ );
+ outDir = join(dir, "out-seo");
+ const run = io();
+ const code = await main(
+ [
+ server,
+ "--out",
+ outDir,
+ "--sitemap",
+ "https://example.com",
+ "--report-file",
+ "../seo-report.json"
+ ],
+ run.io
+ );
+ expect(run.err).toEqual([]);
+ expect(code).toBe(0);
+ expect(run.out[0]).toMatch(/rendered 2 page\(s\) \(2 written\), 2 file\(s\) emitted/);
+ expect((await readdir(outDir)).sort()).toEqual(["about", "index.html", "sitemap.xml"]);
+ expect(await readFile(join(outDir, "sitemap.xml"), "utf8")).toContain(
+ "https://example.com/about"
+ );
+ const summary = JSON.parse(await readFile(join(dir, "seo-report.json"), "utf8"));
+ expect(summary.totals).toMatchObject({ pages: 2, written: 2, files: 1 });
+ expect(summary.files).toEqual(["sitemap.xml"]);
+
+ const bad = io();
+ expect(await main([server, "--out", outDir, "--sitemap", "example.com"], bad.io)).toBe(2);
+ expect(bad.err[0]).toMatch(/--sitemap needs/);
+ });
+
it("rejects a non-integer numeric option", async () => {
const run = io();
const server = join(dir, "ok.mjs");
diff --git a/packages/crawler/test/crawl.test.ts b/packages/crawler/test/crawl.test.ts
index 57c6520..575d286 100644
--- a/packages/crawler/test/crawl.test.ts
+++ b/packages/crawler/test/crawl.test.ts
@@ -424,6 +424,157 @@ describe("crawl", () => {
expect(result.pages.find(p => p.path === "/about")?.filename).toBe("custom.html");
});
+ it("normalizes seeds the way it normalizes links", async () => {
+ const { transport, requests } = site({ "/about": html("about"), "/posts": html("posts") });
+ const result = await runPrerender({
+ transport,
+ outDir: await makeOutDir(),
+ pages: ["/about/", "about#top", "/posts?page=2", { path: "/posts", filename: "p.html" }]
+ });
+ expect(result.pages.map(p => p.path).sort()).toEqual(["/about", "/posts"]);
+ expect(requests.sort()).toEqual(["/about", "/posts"]);
+ // the first spelling of /posts (the string) won; the query left no directory behind
+ expect((await readdir(outDir)).sort()).toEqual(["about", "posts"]);
+ });
+
+ describe("keepQuery", () => {
+ /** Like `site`, but keyed on path + query. */
+ function querySite(routes: Record) {
+ const requests: string[] = [];
+ const transport: Transport = {
+ async fetch(request) {
+ const url = new URL(request.url);
+ requests.push(url.pathname + url.search);
+ const answer = routes[url.pathname + url.search];
+ return answer ? answer.clone() : new Response("not found", { status: 404 });
+ }
+ };
+ return { transport, requests };
+ }
+
+ it("renders each query spelling apart, parameters sorted, but writes only the bare path", async () => {
+ const { transport, requests } = querySite({
+ "/posts": html(`2 2s`),
+ "/posts?page=2": html(
+ `same 3`
+ ),
+ "/posts?page=2&sort=asc": html("sorted"),
+ "/posts?page=3": html("3")
+ });
+ const result = await runPrerender({
+ transport,
+ outDir: await makeOutDir(),
+ pages: ["/posts"],
+ keepQuery: true
+ });
+ expect(requests.sort()).toEqual([
+ "/posts",
+ "/posts?page=2",
+ "/posts?page=2&sort=asc",
+ "/posts?page=3"
+ ]);
+ const byPath = Object.fromEntries(result.pages.map(p => [p.path, p]));
+ expect(byPath["/posts"].emitted).toBe(true);
+ expect(byPath["/posts?page=2"].emitted).toBe(false);
+ // the filename is where the page WOULD go — the same file as /posts, which is why it is not written
+ expect(byPath["/posts?page=2"].filename).toBe("posts/index.html");
+ expect(await readFile(join(outDir, "posts/index.html"), "utf8")).toContain(
+ 'href="/posts?page=2"'
+ );
+ });
+
+ it("writes a query spelling when its entry names a filename, and follows query redirects", async () => {
+ const { transport } = querySite({
+ "/posts?page=2": html("page two"),
+ "/latest": new Response(null, { status: 302, headers: { location: "/posts?page=2" } })
+ });
+ const result = await runPrerender({
+ transport,
+ outDir: await makeOutDir(),
+ pages: [{ path: "/posts?page=2", filename: "posts/page/2/index.html" }, "/latest"],
+ keepQuery: true
+ });
+ expect(await readFile(join(outDir, "posts/page/2/index.html"), "utf8")).toBe("page two");
+ expect(result.redirects).toEqual([{ from: "/latest", to: "/posts?page=2", status: 302 }]);
+ expect(await readFile(join(outDir, "latest/index.html"), "utf8")).toContain(
+ 'url=/posts?page=2"'
+ );
+ });
+
+ it("strips queries everywhere when off (the default)", async () => {
+ const { transport, requests } = querySite({
+ "/posts": html(`2`)
+ });
+ await runPrerender({ transport, outDir: await makeOutDir(), pages: ["/posts?page=9"] });
+ expect(requests).toEqual(["/posts"]);
+ });
+ });
+
+ it("times each page, pacing excluded, and exposes the run's state to integrations", async () => {
+ const { transport } = site({
+ "/": html(`s o m`),
+ "/slow": () => {
+ const until = performance.now() + 15;
+ while (performance.now() < until);
+ return html("slow");
+ },
+ "/old": new Response(null, { status: 301, headers: { location: "/" } })
+ });
+ let seen: { pages: number; redirects: number; skipped: string[]; files: string[] } | undefined;
+ const result = await runPrerender({
+ transport,
+ outDir: await makeOutDir(),
+ failOnError: false,
+ interval: 200,
+ integrations: [
+ {
+ name: "first",
+ setup(context) {
+ context.emitFile({ filename: "a.txt", contents: "a" });
+ }
+ },
+ {
+ name: "observer",
+ teardown(context) {
+ seen = {
+ pages: context.pages.length,
+ redirects: context.redirects.length,
+ skipped: context.skipped.map(miss => miss.path),
+ files: context.files.map(file => file.filename)
+ };
+ }
+ }
+ ]
+ });
+ const byPath = Object.fromEntries(result.pages.map(p => [p.path, p]));
+ // /slow started 200ms after / (paced) and spent ~15ms rendering: were the
+ // throttle charged to the page, this would read 200+
+ expect(byPath["/slow"].duration).toBeGreaterThanOrEqual(14);
+ expect(byPath["/slow"].duration).toBeLessThan(150);
+ expect(typeof byPath["/old"].duration).toBe("number");
+ expect(seen).toEqual({ pages: 3, redirects: 1, skipped: ["/missing"], files: ["a.txt"] });
+ });
+
+ it("resolves emitted filenames against the output directory, so ../ escapes it", async () => {
+ const { transport } = site({ "/": html("index") });
+ const root = await makeOutDir();
+ const out = join(root, "dist");
+ await runPrerender({
+ transport,
+ outDir: out,
+ integrations: [
+ {
+ name: "outside",
+ teardown(context) {
+ context.emitFile({ filename: "../report.txt", contents: "outside" });
+ }
+ }
+ ]
+ });
+ expect(await readFile(join(root, "report.txt"), "utf8")).toBe("outside");
+ expect((await readdir(out)).sort()).toEqual(["index.html"]);
+ });
+
it("records referrers for discovered pages and names them on failures", async () => {
const { transport } = site({
"/": html(`a m`),
diff --git a/packages/crawler/test/links.test.ts b/packages/crawler/test/links.test.ts
index 85bcd15..be85497 100644
--- a/packages/crawler/test/links.test.ts
+++ b/packages/crawler/test/links.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { extractLinks, normalizePath } from "../src/links.ts";
+import { extractLinks, normalizePath, normalizeRoute, splitRoute } from "../src/links.ts";
const page = new URL("http://localhost/blog/post-1");
@@ -66,3 +66,28 @@ describe("normalizePath", () => {
expect(normalizePath("/a/b")).toBe("/a/b");
});
});
+
+describe("normalizeRoute", () => {
+ it("keeps the query on request, parameters sorted, fragment dropped", () => {
+ const url = new URL("http://localhost/posts/?b=2&a=1#top");
+ expect(normalizeRoute(url)).toBe("/posts");
+ expect(normalizeRoute(url, { keepQuery: true })).toBe("/posts?a=1&b=2");
+ expect(normalizeRoute(new URL("http://localhost/posts?"), { keepQuery: true })).toBe("/posts");
+ });
+
+ it("extractLinks threads the option through", () => {
+ const html = `a b c`;
+ expect(extractLinks(html, page, { keepQuery: true })).toEqual([
+ "/about?utm=x",
+ "/about?utm=y",
+ "/about"
+ ]);
+ });
+});
+
+describe("splitRoute", () => {
+ it("separates pathname and search", () => {
+ expect(splitRoute("/posts?page=2")).toEqual({ pathname: "/posts", search: "?page=2" });
+ expect(splitRoute("/posts")).toEqual({ pathname: "/posts", search: "" });
+ });
+});
diff --git a/packages/crawler/test/report.test.ts b/packages/crawler/test/report.test.ts
new file mode 100644
index 0000000..52a51d7
--- /dev/null
+++ b/packages/crawler/test/report.test.ts
@@ -0,0 +1,102 @@
+import { mkdtemp, readFile, readdir, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { runPrerender } from "../src/crawl.ts";
+import { redirects } from "../src/redirects.ts";
+import { report } from "../src/report.ts";
+import type { PrerenderReport } from "../src/report.ts";
+import type { Transport } from "../src/types.ts";
+
+const html = (body: string) =>
+ new Response(body, { headers: { "content-type": "text/html; charset=utf-8" } });
+
+function site(routes: Record) {
+ const transport: Transport = {
+ async fetch(request) {
+ const answer = routes[new URL(request.url).pathname];
+ return answer ? answer.clone() : new Response("not found", { status: 404 });
+ }
+ };
+ return transport;
+}
+
+let root: string;
+afterEach(async () => {
+ if (root) await rm(root, { recursive: true, force: true });
+});
+
+describe("report()", () => {
+ it("describes the run: pages, timings, referrers, redirects, skips, files", async () => {
+ root = await mkdtemp(join(tmpdir(), "report-"));
+ const outDir = join(root, "dist");
+ const transport = site({
+ "/": html(`a o m`),
+ "/about": html(`m`),
+ "/old": new Response(null, { status: 301, headers: { location: "/about" } })
+ });
+ await runPrerender({
+ transport,
+ outDir,
+ mode: "hybrid",
+ failOnError: false,
+ integrations: [redirects(), report({ filename: "../prerender-report.json" })]
+ });
+
+ // written outside the deploy, as asked
+ expect(await readdir(outDir)).toEqual(["_redirects"]);
+ const summary: PrerenderReport = JSON.parse(
+ await readFile(join(root, "prerender-report.json"), "utf8")
+ );
+
+ expect(summary.mode).toBe("hybrid");
+ expect(summary.origin).toBe("http://localhost");
+ expect(Date.parse(summary.generatedAt)).not.toBeNaN();
+ expect(summary.totals).toEqual({
+ pages: 3,
+ written: 0,
+ redirects: 1,
+ skipped: 1,
+ files: 1,
+ duration: expect.any(Number)
+ });
+ expect(summary.pages.map(page => page.path)).toEqual(["/", "/about", "/old"]);
+ const about = summary.pages[1];
+ expect(about).toMatchObject({
+ status: 200,
+ contentType: "text/html; charset=utf-8",
+ filename: "about/index.html",
+ written: false,
+ // the redirect from /old counts as a mention of /about
+ referrers: ["/", "/old"]
+ });
+ expect(about.duration).toBeGreaterThanOrEqual(0);
+ expect(about.redirect).toBeUndefined();
+ expect(summary.pages[2]).toMatchObject({
+ path: "/old",
+ status: 301,
+ redirect: { from: "/old", to: "/about", status: 301 }
+ });
+ expect(summary.redirects).toEqual([{ from: "/old", to: "/about", status: 301 }]);
+ // the message names the referrers known at failure time; `referrers`
+ // is completed once the crawl settles, so it is the authoritative list
+ expect(summary.skipped).toEqual([
+ {
+ path: "/missing",
+ error: expect.stringMatching(/^Prerendering \/missing answered 404 \(linked from \//),
+ referrers: ["/", "/about"]
+ }
+ ]);
+ expect(summary.files).toEqual(["_redirects"]);
+ });
+
+ it("lands in the output directory by default", async () => {
+ root = await mkdtemp(join(tmpdir(), "report-"));
+ await runPrerender({
+ transport: site({ "/": html("index") }),
+ outDir: root,
+ integrations: [report()]
+ });
+ expect((await readdir(root)).sort()).toEqual(["index.html", "prerender-report.json"]);
+ });
+});
diff --git a/packages/crawler/test/sitemap.test.ts b/packages/crawler/test/sitemap.test.ts
new file mode 100644
index 0000000..d5c6f9e
--- /dev/null
+++ b/packages/crawler/test/sitemap.test.ts
@@ -0,0 +1,148 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { runPrerender } from "../src/crawl.ts";
+import { formatSitemap, indexable, sitemap } from "../src/sitemap.ts";
+import type { RenderedPage, Transport } from "../src/types.ts";
+
+const html = (body: string, init?: ResponseInit) =>
+ new Response(body, {
+ ...init,
+ headers: { "content-type": "text/html", ...Object.fromEntries(new Headers(init?.headers)) }
+ });
+
+function site(routes: Record) {
+ const transport: Transport = {
+ async fetch(request) {
+ const answer = routes[new URL(request.url).pathname];
+ return answer ? answer.clone() : new Response("not found", { status: 404 });
+ }
+ };
+ return transport;
+}
+
+let outDir: string;
+afterEach(async () => {
+ if (outDir) await rm(outDir, { recursive: true, force: true });
+});
+
+const page = (html: string, init?: ResponseInit, path = "/x"): RenderedPage => ({
+ path,
+ referrers: [],
+ filename: "x/index.html",
+ emitted: true,
+ response: new Response(html, {
+ ...init,
+ headers: { "content-type": "text/html", ...Object.fromEntries(new Headers(init?.headers)) }
+ }),
+ duration: 1,
+ html
+});
+
+describe("sitemap()", () => {
+ it("lists every indexable rendered page as an absolute, sorted entry", async () => {
+ outDir = await mkdtemp(join(tmpdir(), "sitemap-"));
+ const transport = site({
+ "/": html(
+ `b a o f s`
+ ),
+ "/a": html("a"),
+ "/b": html("b"),
+ "/old": new Response(null, { status: 301, headers: { location: "/a" } }),
+ "/feed.xml": new Response("", { headers: { "content-type": "application/xml" } }),
+ "/secret": html(``)
+ });
+ const result = await runPrerender({
+ transport,
+ outDir,
+ integrations: [sitemap({ hostname: "https://example.com" })]
+ });
+ expect(result.files.map(file => file.filename)).toEqual(["sitemap.xml"]);
+ expect(await readFile(join(outDir, "sitemap.xml"), "utf8")).toBe(
+ `
+
+
+ https://example.com/
+
+
+ https://example.com/a
+
+
+ https://example.com/b
+
+
+`
+ );
+ });
+
+ it("applies the filter, per-page metadata, trailing slashes and a custom filename", async () => {
+ outDir = await mkdtemp(join(tmpdir(), "sitemap-"));
+ const transport = site({
+ "/": html(`a d`),
+ "/about": html("about"),
+ "/draft": html("draft")
+ });
+ await runPrerender({
+ transport,
+ outDir,
+ integrations: [
+ sitemap({
+ hostname: "https://example.com/",
+ filename: "seo/sitemap.xml",
+ trailingSlash: true,
+ filter: page => page.path !== "/draft",
+ entry: page =>
+ page.path === "/"
+ ? { priority: 1, changefreq: "daily", lastmod: new Date("2026-01-02T03:04:05Z") }
+ : { lastmod: "2026-01-01" }
+ })
+ ]
+ });
+ const xml = await readFile(join(outDir, "seo/sitemap.xml"), "utf8");
+ expect(xml).toContain(
+ "https://example.com/\n 2026-01-02T03:04:05.000Z\n daily\n 1.0"
+ );
+ expect(xml).toContain(
+ "https://example.com/about/\n 2026-01-01"
+ );
+ expect(xml).not.toContain("draft");
+ });
+
+ it("requires a hostname", () => {
+ expect(() => sitemap({ hostname: "" })).toThrow(/hostname/);
+ });
+});
+
+describe("indexable", () => {
+ it("honors robots meta in any attribute order and the X-Robots-Tag header, head only", () => {
+ expect(indexable(page(``))).toBe(false);
+ expect(indexable(page(``))).toBe(false);
+ expect(indexable(page(``))).toBe(false);
+ expect(indexable(page(``))).toBe(true);
+ expect(indexable(page(``))).toBe(true);
+ // a meta in the body is not a directive (and the text "noindex" certainly is not)
+ expect(
+ indexable(page(`noindex `))
+ ).toBe(true);
+ expect(indexable(page("ok", { headers: { "x-robots-tag": "noindex, nofollow" } }))).toBe(false);
+ });
+
+ it("drops redirects, non-HTML responses and query spellings", () => {
+ expect(indexable({ ...page("x"), redirect: { from: "/x", to: "/y", status: 301 } })).toBe(
+ false
+ );
+ expect(indexable(page("x", undefined, "/x?page=2"))).toBe(false);
+ const json = page("{}");
+ json.response.headers.set("content-type", "application/json");
+ expect(indexable(json)).toBe(false);
+ });
+});
+
+describe("formatSitemap", () => {
+ it("escapes locs and clamps priorities", () => {
+ expect(formatSitemap([{ loc: "https://e.com/a?b=1&c=<2>", priority: 7 }])).toContain(
+ "https://e.com/a?b=1&c=<2>\n 1.0"
+ );
+ });
+});