diff --git a/scripts/check_links.py b/scripts/check_links.py
index 39a847f..e1675b8 100755
--- a/scripts/check_links.py
+++ b/scripts/check_links.py
@@ -23,7 +23,12 @@
5. No page renders the same H2 twice. A generic block heading and an authored
one drifted into the same words on 32 pages, twice into an exact match, and
nothing in the build could see it.
- 6. Every icon and manifest in resolves, and each icon matches any
+ 6. robots.txt and the sitemap agree. The Sitemap line must name the same
+ origin the build used, and no URL in the sitemap may be blocked by a
+ Disallow rule. A robots.txt that blocks a page the sitemap advertises is
+ the one SEO error that silently un-indexes a site while every other check
+ still passes, and the two files are generated in different places here.
+ 7. Every icon and manifest in resolves, and each icon matches any
sizes="" it declares. A favicon is the one asset nobody notices is broken,
because the browser silently falls back to a blank page glyph. The web
manifest is parsed too: its icons are referenced from nowhere else, so
@@ -91,6 +96,51 @@ def resolves(href: str) -> bool:
or (DIST / f"{path}.html").exists())
+def robots_agrees_with_sitemap() -> list[str]:
+ """Disallow rules that contradict the sitemap, and a mismatched origin."""
+ robots, sitemap = DIST / "robots.txt", DIST / "sitemap.xml"
+ if not robots.exists():
+ return ["robots.txt was not generated"]
+ if not sitemap.exists():
+ return ["sitemap.xml was not generated"]
+ text = robots.read_text(encoding="utf-8")
+ locs = re.findall(r"([^<]+)", sitemap.read_text(encoding="utf-8"))
+ problems: list[str] = []
+
+ declared = re.findall(r"(?im)^\s*Sitemap:\s*(\S+)", text)
+ if not declared:
+ problems.append("robots.txt names no Sitemap")
+ for url in declared:
+ if locs and url.rsplit("/", 1)[0] != locs[0].rstrip("/").rsplit("/", 1)[0] \
+ and not url.startswith(re.match(r"https?://[^/]+", locs[0]).group(0)):
+ problems.append(f"robots.txt Sitemap {url} is not on the origin the sitemap uses "
+ f"({re.match(r'https?://[^/]+', locs[0]).group(0)})")
+
+ # Only the wildcard group can affect Googlebot's view of a sitemap URL here;
+ # a rule for a named agent is deliberate and not this check's business.
+ rules: list[str] = []
+ in_star = False
+ for line in text.splitlines():
+ line = line.split("#")[0].strip()
+ if not line:
+ continue
+ key, _, value = line.partition(":")
+ key, value = key.strip().lower(), value.strip()
+ if key == "user-agent":
+ in_star = value == "*"
+ elif key == "disallow" and in_star and value:
+ rules.append(value)
+
+ for loc in locs:
+ path = re.sub(r"^https?://[^/]+", "", loc) or "/"
+ for rule in rules:
+ pattern = re.escape(rule).replace(r"\*", ".*")
+ if re.match(pattern, path):
+ problems.append(f"sitemap lists {path} but robots.txt disallows {rule}")
+ break
+ return problems
+
+
def main() -> int:
if not DIST.exists():
print("dist/ not found: run npm run build first", file=sys.stderr)
@@ -102,6 +152,7 @@ def main() -> int:
bad_card: Counter[str] = Counter()
dupe_h2: Counter[str] = Counter()
leaked_status: Counter[str] = Counter()
+ robots_bad: Counter[str] = Counter()
internal = outbound = cards = 0
for page in pages:
@@ -192,6 +243,9 @@ def main() -> int:
bad_card[f"{where} -> /{src} is {size[0]}x{size[1]},"
f" declared {declared.group(0)}"] += 1
+ for problem in robots_agrees_with_sitemap():
+ robots_bad[problem] += 1
+
print(f"{len(pages)} pages | {internal} internal links | "
f"{outbound} outbound links | {cards} asset refs")
failed = False
@@ -200,7 +254,8 @@ def main() -> int:
("outbound links missing rel=nofollow", follows),
("card, icon or manifest assets missing or mis-sized", bad_card),
("pages rendering the same H2 twice", dupe_h2),
- ("pages printing an internal record status", leaked_status)):
+ ("pages printing an internal record status", leaked_status),
+ ("robots.txt disagreeing with the sitemap", robots_bad)):
if counter:
failed = True
print(f"\n{label}: {sum(counter.values())}")
diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts
index e6a1269..1c7b15b 100644
--- a/src/pages/robots.txt.ts
+++ b/src/pages/robots.txt.ts
@@ -1,3 +1,30 @@
+// robots.txt. Nothing is disallowed, and that is deliberate.
+//
+// The pages this site keeps out of search are kept out with a noindex meta tag,
+// which only works if a crawler is allowed to fetch the page and read it. The
+// seven unwritten records and the 404 are noindex for that reason. Adding a
+// Disallow for them would have the opposite of the intended effect: the tag
+// would never be seen, and the URLs could still be indexed from links pointing
+// at them, with no description.
+//
+// There is also nothing else to block. The site is static, has no search, no
+// parameters, no session URLs and no faceted navigation, so there are no crawl
+// traps. The bundled CSS and JS under /_astro must stay fetchable or Google
+// cannot render the pages.
+//
+// scripts/check_links.py fails the build if a Disallow rule here ever blocks a
+// URL the sitemap advertises, or if the Sitemap line names a different origin
+// from the one the build used.
import type { APIRoute } from 'astro';
+
export const GET: APIRoute = ({ site }) =>
- new Response(`User-agent: *\nAllow: /\n\nSitemap: ${new URL('/sitemap.xml', site)}\n`, { headers: { 'Content-Type': 'text/plain' } });
+ new Response(
+ [
+ 'User-agent: *',
+ 'Allow: /',
+ '',
+ `Sitemap: ${new URL('/sitemap.xml', site)}`,
+ '',
+ ].join('\n'),
+ { headers: { 'Content-Type': 'text/plain; charset=utf-8' } },
+ );