From f04e16d62cb35663604f968c26c59e875b60800e Mon Sep 17 00:00:00 2001 From: fosterwei Date: Wed, 30 Sep 2026 01:25:16 -0700 Subject: [PATCH] Make robots.txt and the sitemap unable to contradict each other The file itself was already right: 200, text/plain, Allow: /, and a Sitemap line on the www host after the canonical fix. Nothing needed changing in what it says. What was missing is anything stopping it from going wrong. The two files are generated in different places, robots.txt.ts and sitemap.xml.ts, and nothing compared them. A Disallow that blocks a URL the sitemap advertises is the one SEO mistake that silently removes pages from search while every other check here still passes: the build succeeds, the links resolve, the pages render. check_links.py gains a seventh rule. Every sitemap URL is tested against the wildcard group's Disallow rules, with * expanded, and the Sitemap line must name the origin the sitemap actually uses. Verified by injecting five faults: a plain prefix Disallow (caught 30 URLs), a wildcard Disallow (4), a Sitemap line pointing at a vercel.app host, and no Sitemap line at all. The fifth is the one it must not catch, a Disallow under a named user-agent rather than the wildcard, which is a deliberate choice about one crawler and not this check's business. The generator now says why nothing is disallowed, because the reasoning is not obvious and the natural instinct is wrong. The pages kept out of search are kept out with a noindex tag, which only works if the crawler is allowed to fetch the page and read it. Disallowing the seven unwritten records would mean the tag is never seen and the URLs could still be indexed from links, with no description. /_astro must stay fetchable or Google cannot render anything. Also checked and found clean: the apex host redirects robots.txt to www, trailing-slash URLs 308 to the canonical form, and the .html duplicates that Vercel serves carry a canonical pointing at the clean URL, so they consolidate rather than compete. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/check_links.py | 59 +++++++++++++++++++++++++++++++++++++++-- src/pages/robots.txt.ts | 29 +++++++++++++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/scripts/check_links.py b/scripts/check_links.py index 39a847f..e1675b8 100755 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -23,7 +23,12 @@ 5. No page renders the same H2 twice. A generic block heading and an authored one drifted into the same words on 32 pages, twice into an exact match, and nothing in the build could see it. - 6. Every icon and manifest in resolves, and each icon matches any + 6. robots.txt and the sitemap agree. The Sitemap line must name the same + origin the build used, and no URL in the sitemap may be blocked by a + Disallow rule. A robots.txt that blocks a page the sitemap advertises is + the one SEO error that silently un-indexes a site while every other check + still passes, and the two files are generated in different places here. + 7. Every icon and manifest in resolves, and each icon matches any sizes="" it declares. A favicon is the one asset nobody notices is broken, because the browser silently falls back to a blank page glyph. The web manifest is parsed too: its icons are referenced from nowhere else, so @@ -91,6 +96,51 @@ def resolves(href: str) -> bool: or (DIST / f"{path}.html").exists()) +def robots_agrees_with_sitemap() -> list[str]: + """Disallow rules that contradict the sitemap, and a mismatched origin.""" + robots, sitemap = DIST / "robots.txt", DIST / "sitemap.xml" + if not robots.exists(): + return ["robots.txt was not generated"] + if not sitemap.exists(): + return ["sitemap.xml was not generated"] + text = robots.read_text(encoding="utf-8") + locs = re.findall(r"([^<]+)", sitemap.read_text(encoding="utf-8")) + problems: list[str] = [] + + declared = re.findall(r"(?im)^\s*Sitemap:\s*(\S+)", text) + if not declared: + problems.append("robots.txt names no Sitemap") + for url in declared: + if locs and url.rsplit("/", 1)[0] != locs[0].rstrip("/").rsplit("/", 1)[0] \ + and not url.startswith(re.match(r"https?://[^/]+", locs[0]).group(0)): + problems.append(f"robots.txt Sitemap {url} is not on the origin the sitemap uses " + f"({re.match(r'https?://[^/]+', locs[0]).group(0)})") + + # Only the wildcard group can affect Googlebot's view of a sitemap URL here; + # a rule for a named agent is deliberate and not this check's business. + rules: list[str] = [] + in_star = False + for line in text.splitlines(): + line = line.split("#")[0].strip() + if not line: + continue + key, _, value = line.partition(":") + key, value = key.strip().lower(), value.strip() + if key == "user-agent": + in_star = value == "*" + elif key == "disallow" and in_star and value: + rules.append(value) + + for loc in locs: + path = re.sub(r"^https?://[^/]+", "", loc) or "/" + for rule in rules: + pattern = re.escape(rule).replace(r"\*", ".*") + if re.match(pattern, path): + problems.append(f"sitemap lists {path} but robots.txt disallows {rule}") + break + return problems + + def main() -> int: if not DIST.exists(): print("dist/ not found: run npm run build first", file=sys.stderr) @@ -102,6 +152,7 @@ def main() -> int: bad_card: Counter[str] = Counter() dupe_h2: Counter[str] = Counter() leaked_status: Counter[str] = Counter() + robots_bad: Counter[str] = Counter() internal = outbound = cards = 0 for page in pages: @@ -192,6 +243,9 @@ def main() -> int: bad_card[f"{where} -> /{src} is {size[0]}x{size[1]}," f" declared {declared.group(0)}"] += 1 + for problem in robots_agrees_with_sitemap(): + robots_bad[problem] += 1 + print(f"{len(pages)} pages | {internal} internal links | " f"{outbound} outbound links | {cards} asset refs") failed = False @@ -200,7 +254,8 @@ def main() -> int: ("outbound links missing rel=nofollow", follows), ("card, icon or manifest assets missing or mis-sized", bad_card), ("pages rendering the same H2 twice", dupe_h2), - ("pages printing an internal record status", leaked_status)): + ("pages printing an internal record status", leaked_status), + ("robots.txt disagreeing with the sitemap", robots_bad)): if counter: failed = True print(f"\n{label}: {sum(counter.values())}") diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index e6a1269..1c7b15b 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -1,3 +1,30 @@ +// robots.txt. Nothing is disallowed, and that is deliberate. +// +// The pages this site keeps out of search are kept out with a noindex meta tag, +// which only works if a crawler is allowed to fetch the page and read it. The +// seven unwritten records and the 404 are noindex for that reason. Adding a +// Disallow for them would have the opposite of the intended effect: the tag +// would never be seen, and the URLs could still be indexed from links pointing +// at them, with no description. +// +// There is also nothing else to block. The site is static, has no search, no +// parameters, no session URLs and no faceted navigation, so there are no crawl +// traps. The bundled CSS and JS under /_astro must stay fetchable or Google +// cannot render the pages. +// +// scripts/check_links.py fails the build if a Disallow rule here ever blocks a +// URL the sitemap advertises, or if the Sitemap line names a different origin +// from the one the build used. import type { APIRoute } from 'astro'; + export const GET: APIRoute = ({ site }) => - new Response(`User-agent: *\nAllow: /\n\nSitemap: ${new URL('/sitemap.xml', site)}\n`, { headers: { 'Content-Type': 'text/plain' } }); + new Response( + [ + 'User-agent: *', + 'Allow: /', + '', + `Sitemap: ${new URL('/sitemap.xml', site)}`, + '', + ].join('\n'), + { headers: { 'Content-Type': 'text/plain; charset=utf-8' } }, + );