diff --git a/scripts/check_links.py b/scripts/check_links.py index 39a847f..e1675b8 100755 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -23,7 +23,12 @@ 5. No page renders the same H2 twice. A generic block heading and an authored one drifted into the same words on 32 pages, twice into an exact match, and nothing in the build could see it. - 6. Every icon and manifest in resolves, and each icon matches any + 6. robots.txt and the sitemap agree. The Sitemap line must name the same + origin the build used, and no URL in the sitemap may be blocked by a + Disallow rule. A robots.txt that blocks a page the sitemap advertises is + the one SEO error that silently un-indexes a site while every other check + still passes, and the two files are generated in different places here. + 7. Every icon and manifest in resolves, and each icon matches any sizes="" it declares. A favicon is the one asset nobody notices is broken, because the browser silently falls back to a blank page glyph. The web manifest is parsed too: its icons are referenced from nowhere else, so @@ -91,6 +96,51 @@ def resolves(href: str) -> bool: or (DIST / f"{path}.html").exists()) +def robots_agrees_with_sitemap() -> list[str]: + """Disallow rules that contradict the sitemap, and a mismatched origin.""" + robots, sitemap = DIST / "robots.txt", DIST / "sitemap.xml" + if not robots.exists(): + return ["robots.txt was not generated"] + if not sitemap.exists(): + return ["sitemap.xml was not generated"] + text = robots.read_text(encoding="utf-8") + locs = re.findall(r"([^<]+)", sitemap.read_text(encoding="utf-8")) + problems: list[str] = [] + + declared = re.findall(r"(?im)^\s*Sitemap:\s*(\S+)", text) + if not declared: + problems.append("robots.txt names no Sitemap") + for url in declared: + if locs and url.rsplit("/", 1)[0] != locs[0].rstrip("/").rsplit("/", 1)[0] \ + and not url.startswith(re.match(r"https?://[^/]+", locs[0]).group(0)): + problems.append(f"robots.txt Sitemap {url} is not on the origin the sitemap uses " + f"({re.match(r'https?://[^/]+', locs[0]).group(0)})") + + # Only the wildcard group can affect Googlebot's view of a sitemap URL here; + # a rule for a named agent is deliberate and not this check's business. + rules: list[str] = [] + in_star = False + for line in text.splitlines(): + line = line.split("#")[0].strip() + if not line: + continue + key, _, value = line.partition(":") + key, value = key.strip().lower(), value.strip() + if key == "user-agent": + in_star = value == "*" + elif key == "disallow" and in_star and value: + rules.append(value) + + for loc in locs: + path = re.sub(r"^https?://[^/]+", "", loc) or "/" + for rule in rules: + pattern = re.escape(rule).replace(r"\*", ".*") + if re.match(pattern, path): + problems.append(f"sitemap lists {path} but robots.txt disallows {rule}") + break + return problems + + def main() -> int: if not DIST.exists(): print("dist/ not found: run npm run build first", file=sys.stderr) @@ -102,6 +152,7 @@ def main() -> int: bad_card: Counter[str] = Counter() dupe_h2: Counter[str] = Counter() leaked_status: Counter[str] = Counter() + robots_bad: Counter[str] = Counter() internal = outbound = cards = 0 for page in pages: @@ -192,6 +243,9 @@ def main() -> int: bad_card[f"{where} -> /{src} is {size[0]}x{size[1]}," f" declared {declared.group(0)}"] += 1 + for problem in robots_agrees_with_sitemap(): + robots_bad[problem] += 1 + print(f"{len(pages)} pages | {internal} internal links | " f"{outbound} outbound links | {cards} asset refs") failed = False @@ -200,7 +254,8 @@ def main() -> int: ("outbound links missing rel=nofollow", follows), ("card, icon or manifest assets missing or mis-sized", bad_card), ("pages rendering the same H2 twice", dupe_h2), - ("pages printing an internal record status", leaked_status)): + ("pages printing an internal record status", leaked_status), + ("robots.txt disagreeing with the sitemap", robots_bad)): if counter: failed = True print(f"\n{label}: {sum(counter.values())}") diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index e6a1269..1c7b15b 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -1,3 +1,30 @@ +// robots.txt. Nothing is disallowed, and that is deliberate. +// +// The pages this site keeps out of search are kept out with a noindex meta tag, +// which only works if a crawler is allowed to fetch the page and read it. The +// seven unwritten records and the 404 are noindex for that reason. Adding a +// Disallow for them would have the opposite of the intended effect: the tag +// would never be seen, and the URLs could still be indexed from links pointing +// at them, with no description. +// +// There is also nothing else to block. The site is static, has no search, no +// parameters, no session URLs and no faceted navigation, so there are no crawl +// traps. The bundled CSS and JS under /_astro must stay fetchable or Google +// cannot render the pages. +// +// scripts/check_links.py fails the build if a Disallow rule here ever blocks a +// URL the sitemap advertises, or if the Sitemap line names a different origin +// from the one the build used. import type { APIRoute } from 'astro'; + export const GET: APIRoute = ({ site }) => - new Response(`User-agent: *\nAllow: /\n\nSitemap: ${new URL('/sitemap.xml', site)}\n`, { headers: { 'Content-Type': 'text/plain' } }); + new Response( + [ + 'User-agent: *', + 'Allow: /', + '', + `Sitemap: ${new URL('/sitemap.xml', site)}`, + '', + ].join('\n'), + { headers: { 'Content-Type': 'text/plain; charset=utf-8' } }, + );