diff --git a/.claude/skills/report/SKILL.md b/.claude/skills/report/SKILL.md index 03a2e98..a75e5a0 100644 --- a/.claude/skills/report/SKILL.md +++ b/.claude/skills/report/SKILL.md @@ -39,6 +39,13 @@ Structure, in order: 6. **Open items** — split by who acts: owner (dashboard/env/merge), orchestrator (cross-repo), this repo's next pass. +Where the pass consumed a sync spec, per-item dispositions use +exactly these five words: `applied` / `ported-as-contract` / +`already-present` / `not-applicable-because` / `open`. `open` means +the detect fires but the item is deliberately out of this session's +scope — name it under Open items with who acts. Do not invent a +sixth word; the orchestrator's tooling reads these five. + Anti-patterns, all observed in the fleet and all rejected on receipt: "should work" (test it or mark it unverified); summary claims without artifacts; green CI presented as deploy proof when diff --git a/scripts/smoke_live.py b/scripts/smoke_live.py index acbdf03..bf955da 100644 --- a/scripts/smoke_live.py +++ b/scripts/smoke_live.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Post-deploy checks against a *live* satellite. - python scripts/smoke_live.py https://leaflet.2plot.dev + python scripts/smoke_live.py https://boilerplate.2plot.dev Everything here fails silently in production if it isn't checked. A wrong canonical host doesn't error, it deindexes; a stub body doesn't error, it @@ -11,11 +11,21 @@ Run in CD after every deploy, and by hand against any satellite you're upgrading. Exit code is the number of failed checks, capped at 125. +Much of the fleet runs on Render's free tier, which sleeps after ~15 minutes +idle and answers the first probe with a loading page or a hang — so the +battery wakes the host up first (a `/healthz` poll, LESSONS §21) and `fetch` +retries transport errors and 5xx. Both are tunable without editing this file: + + SMOKE_WAKE_ATTEMPTS /healthz probes before giving up (default 24) + SMOKE_WAKE_INTERVAL_S seconds between probes (default 10) + SMOKE_FETCH_RETRIES attempts per request inside fetch (default 3) + Only the standard library, so it runs anywhere without an install step. """ from __future__ import annotations +import html as html_lib import os import re import sys @@ -56,8 +66,13 @@ # the element. CHROME = re.compile(r'<[a-z]+ class="dv-banner') TIMEOUT = 30 -# Attempts per URL for NETWORK-level errors only — see `fetch`. -RETRIES = 3 +# Generous on purpose: a free-tier cold start routinely takes 60-90s, and the +# only cost of a wide window is paid when the host is actually down — a warm +# host passes the first probe. 24 x 10s covers the slow tail with room; a +# satellite on an even slower tier stretches it via the env vars above. +RETRIES = max(1, int(os.getenv("SMOKE_FETCH_RETRIES") or 3)) +WAKE_ATTEMPTS = max(1, int(os.getenv("SMOKE_WAKE_ATTEMPTS") or 24)) +WAKE_INTERVAL_S = max(0.0, float(os.getenv("SMOKE_WAKE_INTERVAL_S") or 10)) def _ssl_context() -> ssl.SSLContext: @@ -84,7 +99,11 @@ def _ssl_context() -> ssl.SSLContext: def fetch( - url: str, user_agent: str = BROWSER_UA, accept: Optional[str] = None + url: str, + user_agent: str = BROWSER_UA, + accept: Optional[str] = None, + retries: Optional[int] = None, + timeout: float = TIMEOUT, ) -> Tuple[int, str, Dict[str, str]]: """Returns (status, body, headers). @@ -92,42 +111,60 @@ def fetch( content-negotiates, so which *type* came back is the thing being checked, and `Vary` is what stops a CDN handing cached HTML to the next agent. - NETWORK-LEVEL failures are retried; HTTP statuses are not. The distinction - matters because this script makes ~17 requests in a burst against a host on - Render's free tier, which sleeps after ~15 minutes idle — so one cold start - or one dropped connection used to surface as `FAIL canonical on /`, - a check that had never actually run. A misdiagnosed failure is worse than a - slow one: it sends you looking at canonical tags that were correct all - along. A real 404 or 500 still fails on the first response, immediately. + TRANSPORT errors and 5xx are retried with backoff; other statuses are + verdicts and are not. The distinction matters because this script makes + ~40 requests in a burst against hosts on Render's free tier — one dropped + connection used to surface as `FAIL canonical on /`, a check that + had never actually run, sending you to look at canonical tags that were + correct all along (LESSONS §21; same ladder as network_smoke.py). A 404 is + a real answer, and retrying it would only slow the battery down; a check + still failing after every attempt is a real failure. + + `errors="surrogateescape"`, not `"replace"`: this function also fetches + the social card, and the card check reads the PNG's IHDR chunk for the + real pixel dimensions. `"replace"` substitutes U+FFFD for every invalid + byte and is one-way, so the header would be gone before it could be read. + surrogateescape round-trips exactly through + `body.encode("utf-8", "surrogateescape")`, and behaves identically to a + plain decode for text. """ headers = {"User-Agent": user_agent} if accept is not None: headers["Accept"] = accept request = urllib.request.Request(url, headers=headers) - last: Exception | None = None - for attempt in range(RETRIES): + attempts = RETRIES if retries is None else max(1, retries) + last: Tuple[int, str, Dict[str, str]] = (0, "no attempt was made", {}) + for attempt in range(attempts): if attempt: time.sleep(2 * attempt) try: with urllib.request.urlopen( - request, timeout=TIMEOUT, context=SSL_CONTEXT + request, timeout=timeout, context=SSL_CONTEXT ) as response: - body = response.read().decode("utf-8", "replace") + body = response.read().decode("utf-8", "surrogateescape") return response.status, body, dict(response.headers) except urllib.error.HTTPError as exc: # The STATUS is the answer; the body is a bonus. Reading it can - # itself raise — a peer that 502s mid-body raises IncompleteRead + # itself raise — a host that 502s mid-body raises IncompleteRead # here — and an exception escaping `fetch` takes the whole script - # down. That turns one sick peer into a dead CD run, which is - # exactly what the fatal/warn split in `check()` exists to prevent. + # down, turning one sick response into a dead CD run. try: - body = exc.read().decode("utf-8", "replace") + body = exc.read().decode("utf-8", "surrogateescape") except Exception: # noqa: BLE001 - truncated or already-closed body body = "" - return exc.code, body, dict(exc.headers or {}) + last = (exc.code, body, dict(exc.headers or {})) + if exc.code < 500: + return last + reason = f"HTTP {exc.code}" except Exception as exc: # noqa: BLE001 - DNS, TLS, timeouts all land here - last = exc - return 0, f"{type(last).__name__}: {last}", {} + last = (0, f"{type(exc).__name__}: {exc}", {}) + reason = type(exc).__name__ + if attempt + 1 < attempts: + # Visible on purpose: a green run whose log shows retries is a + # host worth watching, and CD output is the only place that shows. + print(f" retry {attempt + 1}/{attempts - 1} for {url} — {reason}", + flush=True) + return last def header(headers: Dict[str, str], name: str) -> str: @@ -138,6 +175,38 @@ def header(headers: Dict[str, str], name: str) -> str: return "" +def post(url: str, payload: str = "{}") -> int: + """POST for the auth-wiring probe; returns the status, 0 on transport. + + No retry ladder on purpose: a 4xx here IS the answer (invalid token, + anonymous signout — both prove the route is registered and callable), + so only a transport failure reads as 0. + """ + request = urllib.request.Request( + url, + data=payload.encode("utf-8"), + headers={"User-Agent": BROWSER_UA, "Content-Type": "application/json"}, + method="POST", + ) + try: + # context= must match fetch()'s — this line shipped WITHOUT it, so on + # any Python without OS trust-store integration (macOS: the fleet's + # whole local-dev half) every auth POST died in the TLS handshake, + # returned 0, and the check accused the app of the exact + # configure_app regression it exists to detect. CI never saw it + # (Linux verifies fine); no wired test can see it (they monkeypatch + # post) — hence the SOURCE pin in tests/test_auth_wiring.py. + # Found by flexlayout during the F1 kit adoption (154688e). + with urllib.request.urlopen( + request, timeout=TIMEOUT, context=SSL_CONTEXT + ) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + except (urllib.error.URLError, TimeoutError, OSError): + return 0 + + def check(name: str, passed: bool, detail: str = "", fatal: bool = True) -> None: """Record one check. ``fatal=False`` warns instead of failing the deploy. @@ -166,16 +235,81 @@ def check(name: str, passed: bool, detail: str = "", fatal: bool = True) -> None print(f"::warning title=peer unreachable::{name} — {detail}") +def wake(base: str) -> bool: + """Poll `/healthz` until the host actually answers. LESSONS §21. + + A sleeping free-tier host greets its first visitor with Render's loading + page or a hang, and the first visitor after a deploy is this battery — so + without this loop the opening checks fail on a perfectly healthy site. + Requiring `ok: true` rather than any 200 keeps the loading page (and a + CDN error page, which can also be a 200) from counting as awake. + + Each probe is single-shot with a short timeout: the loop IS the retry + ladder here, and per-probe printing is what makes a slow start readable + in the CD log rather than a silent multi-minute stall. + """ + url = f"{base}/healthz" + for attempt in range(1, WAKE_ATTEMPTS + 1): + status, body, _ = fetch(url, retries=1, timeout=10) + if status == 200 and re.search(r'"ok"\s*:\s*true', body): + print(f" wake attempt {attempt}/{WAKE_ATTEMPTS}: up") + return True + detail = f"HTTP {status}" if status else body[:80] + print(f" wake attempt {attempt}/{WAKE_ATTEMPTS}: {detail}", flush=True) + if attempt < WAKE_ATTEMPTS: + time.sleep(WAKE_INTERVAL_S) + return False + + def main(base: str) -> int: base = base.rstrip("/") host = urlparse(base).netloc print(f"Smoke-testing {base}\n") + # --- 0. Wake the host before asserting anything about it --------------- + print("Wake-up") + if not wake(base): + # ONE clear failure, not a cascade: forty per-check failures against a + # host that never answered all say the same thing and bury it. + check( + "host answered /healthz", + False, + f"never woke after {WAKE_ATTEMPTS} probes ~{WAKE_INTERVAL_S:g}s " + "apart — nothing else was tested", + ) + print(f"\n0/{checks_run} checks passed") + print("\nFailed:") + for name in failures: + print(f" - {name}") + return min(len(failures), 125) + # --- 1. The site is up, and llms.txt is the index it should be --------- print("Core surfaces") status, home, _ = fetch(f"{base}/") check("home page responds 200", status == 200, f"got {status}") + # --- Auth wiring: the two-call split, proven from outside -------------- + # dash-clerk-auth wires either side of Dash(...): register() is the UI + # half, configure_app(app) registers /api/auth/* and per-request + # identity. A fork that drops the second call still LOOKS signed in + # (components render, ClerkJS runs) while every server render reads + # signed-out and sign-out never revokes — flexlayout shipped exactly + # that, and no local suite can see it because Clerk is off in test + # environments. From outside the tell is unambiguous: registered, these + # POSTs answer 2xx/4xx; unregistered, the path falls through to Dash's + # GET-only page catch-all and answers 405 (or 404). Gated on the + # package's inline bootstrap being in the served shell, so clerk-off + # hosts skip rather than fail. + if "dashClerkAuth" in home: + for endpoint in ("session", "signout"): + status = post(f"{base}/api/auth/{endpoint}") + check( + f"POST /api/auth/{endpoint} is a registered route", + status not in (0, 404, 405), + f"got {status} — the configure_app(app) half of the auth " + "wiring is missing: components without a server", + ) + status, llms, llms_headers = fetch(f"{base}/llms.txt") check("/llms.txt responds 200", status == 200, f"got {status}") check("/llms.txt lists pages", "## Pages" in llms or "# " in llms) @@ -189,17 +323,10 @@ def main(base: str) -> int: "sitemap line missing or pointing elsewhere", ) # The artifact fingerprint. pip metadata is invisible from outside, so - # these robots.txt stanzas are how a live host is proven to run the - # intended dash-improve-my-llms: 2.3.2 introduced the OAI-SearchBot / - # ChatGPT-User / PerplexityBot allowlist, 2.3.3 added Claude-User and - # Claude-SearchBot. - # - # PER-SITE: most satellites also expect `ClaudeBot -> Disallow: /`, the - # 2.3.3 training-crawler split. This host runs `block_ai_training=False` - # ON PURPOSE (run.py's RobotsConfig — for MIT-licensed component docs, - # being in the training corpus is how a model recommends the library), and - # under that config the package emits no ClaudeBot stanza at all. The - # absence is asserted below so a silent flip of that flag is still caught. + # these robots.txt pairs are how a live host is proven to run the intended + # dash-improve-my-llms: 2.3.2 allowed OAI-SearchBot; 2.3.3 moved ClaudeBot + # (the training crawler) to Disallow while allowing the user-triggered and + # search fetchers Claude-User / Claude-SearchBot. robots_lines = robots.splitlines() def robots_rule(agent: str) -> str: @@ -212,8 +339,7 @@ def robots_rule(agent: str) -> str: for agent, expected, since in ( ("OAI-SearchBot", "Allow: /", "2.3.2"), - ("ChatGPT-User", "Allow: /", "2.3.2"), - ("PerplexityBot", "Allow: /", "2.3.2"), + ("ClaudeBot", "Disallow: /", "2.3.3"), ("Claude-User", "Allow: /", "2.3.3"), ("Claude-SearchBot", "Allow: /", "2.3.3"), ): @@ -224,12 +350,6 @@ def robots_rule(agent: str) -> str: f"got {got}: this host runs a pre-{since} artifact", ) - check( - "/robots.txt keeps this site's deliberate open-training posture", - "User-agent: ClaudeBot" not in robots_lines, - "a ClaudeBot stanza appeared — block_ai_training flipped to True?", - ) - status, sitemap, _ = fetch(f"{base}/sitemap.xml") check("/sitemap.xml responds 200", status == 200, f"got {status}") page_urls = re.findall(r"([^<]+)", sitemap) @@ -261,6 +381,117 @@ def robots_rule(agent: str) -> str: "served the JavaScript stub", ) + # --- 3b. The social card actually exists, and is the shape we claim ---- + # This is the ONLY check that can see either failure. The card is on the + # CDN, so no offline test can fetch it; and its dimensions are hard-coded + # in three places (lib/constants.py, index.html, the CDN object), so + # replacing the uploaded file with a different shape leaves every test + # green while the platform reserves the wrong box and crops into it. + # + # A blank preview is also self-inflicting: platforms cache a failed scrape, + # so the first share after a bad upload poisons the link for everyone. + print("\nSocial card") + card_urls = re.findall(r']+property="og:image"[^>]+content="([^"]*)"', home) + check("og:image is declared exactly once", len(card_urls) == 1, f"got {card_urls}") + if card_urls and card_urls[0]: + card_url = card_urls[0] + check("og:image is not served by the app", "/assets/" not in card_url, + f"{card_url} — a cold container blanks the preview, cached") + status, body, headers = fetch(card_url) + check("og:image resolves", status == 200, f"got {status}") + ctype = header(headers, "Content-Type") + check("og:image is a real image", ctype.startswith("image/"), ctype or "none") + + declared = { + prop: re.findall( + rf']+property="{prop}"[^>]+content="([^"]*)"', home) + for prop in ("og:image:width", "og:image:height") + } + # PNG stores its dimensions in the IHDR chunk: bytes 16..24 of the + # file. Read from the RESPONSE, so what is checked is what a scraper + # would actually receive rather than what the repo believes. + raw = body.encode("utf-8", "surrogateescape") + if raw[1:4] == b"PNG" and len(raw) > 24: + actual_w = int.from_bytes(raw[16:20], "big") + actual_h = int.from_bytes(raw[20:24], "big") + check( + "og:image dimensions match the declared width/height", + declared["og:image:width"] == [str(actual_w)] + and declared["og:image:height"] == [str(actual_h)], + f"file is {actual_w}x{actual_h}, tags say " + f"{declared['og:image:width']}x{declared['og:image:height']}", + ) + ratio = actual_w / actual_h if actual_h else 0 + check("og:image suits summary_large_image (~1.91:1)", + 1.7 <= ratio <= 2.05, f"{actual_w}x{actual_h} is {ratio:.2f}:1") + else: + check("og:image is not empty", False, + "an EMPTY og:image renders a blank card — worse than none") + + # --- 3c. Crawler/browser identity parity (the 2.5.0 Tier-B standard) --- + # Every SEO defect measured across the fleet in 2026-08 was one bug in + # different clothes: the head a crawler received had drifted from the + # head a browser received — 4-7 icon links vs zero, "site | page" vs a + # bare page name, og:image vs nothing. Content may differ between the + # two documents (that is what the prerender is for); identity may not. + # This block is the single assertion that would have caught all of it. + print("\nCrawler/browser identity parity") + + def identity(html: str) -> Dict[str, object]: + # Icons compare as the SET of declared sizes, not a raw link count: + # Dash auto-injects one extra favicon link (with a cache-busting + # query) into the browser head, so counts differ by one forever + # while the actual identity — which sizes a consumer can pick from + # — is what the two heads must agree on. + icon_links = re.findall(r']+rel="(?:icon|apple-touch-icon)"[^>]*>', html) + # Unescape before comparing: one side may write an apostrophe as + # ' and the other verbatim — same identity, different escaping. + unescape = html_lib.unescape + return { + "icon sizes": sorted( + {s for link in icon_links for s in re.findall(r'sizes="([^"]+)"', link)} + ), + "title": unescape( + (re.findall(r"(.*?)", html, re.S) or [""])[0].strip() + ), + "og:image": sorted({ + unescape(u) + for u in re.findall(r'property="og:image"[^>]+content="([^"]*)"', html) + }), + "twitter:card": sorted({ + unescape(v) + for v in re.findall(r'name="twitter:card"[^>]+content="([^"]*)"', html) + }), + } + + for url in [f"{base}/"] + page_urls[:3]: + path = urlparse(url).path or "/" + _status, crawler_html, _ = fetch(url, CRAWLER_UA) + _status, browser_html, _ = fetch(url, BROWSER_UA) + seen_c, seen_b = identity(crawler_html), identity(browser_html) + for field in ("icon sizes", "title", "og:image", "twitter:card"): + check( + f"{path}: crawler and browser agree on {field}", + seen_c[field] == seen_b[field] and seen_c[field] not in (0, "", []), + f"crawler={seen_c[field]!r} browser={seen_b[field]!r}", + ) + check( + f"{path}: crawlers get an icon >=192px", + 'sizes="192x192"' in crawler_html or 'sizes="512x512"' in crawler_html, + "no >=192px icon link in the crawler head — Google's preferred size", + ) + + # Google falls back to /favicon.ico when the page it crawled + # declares no icon. Dash's page catch-all used to answer it with the app + # shell — 200 text/html where an image belongs, a poisoned fallback. + status, favicon_body, _ = fetch(f"{base}/favicon.ico") + check("/favicon.ico resolves", status == 200, f"got {status}") + check( + "/favicon.ico is an image, not the app shell", + not favicon_body.lstrip().lower().startswith(" bool: ) +def _in_repo(rel: str) -> bool: + return ".." not in rel and not rel.startswith("/") + + def _machine_fence(kind: str, text: str, where: str) -> None: """The shared pin for machine fences (```yaml sync-verbatim in specs, ```yaml byte-owned in DIVERGENCES.md): exactly one block, `- path` lines with `#` comments, every path repo-relative and real at HEAD. Empty is valid — an empty block is a statement, a missing one is an - omission. `# requires: ` lines (the fan-out's adoption gate, - 1.6.23) are validated like paths — a typo'd gate gates nothing.""" + omission. Gate lines (the fan-out's adoption gates) are validated + like paths — a typo'd gate gates nothing: + + `# requires: ` (1.6.23) — the block applies only where + exists. For paths no pre-existing file can occupy; + where one can, the gate must name a contract instead + (sync/README.md — flows' pre-existing CLAUDE.md, 1.6.28). + `# requires-contract: :: ` (1.6.28) — the block + applies only where exists AND contains . The + clause must be real in THIS repo's copy at HEAD too. + `- # requires: ` (1.6.28) — per-file gate: the + fan-out skips this one copy where is absent, instead + of gating the whole block (clerkhook: a lockdown fork has no + lib/auth_demos.py, legitimately, and must still receive the + rest).""" fences = re.findall( r"^```yaml " + kind + r"[ \t]*\n(.*?)^```[ \t]*$", text, re.M | re.S ) @@ -52,10 +69,34 @@ def _machine_fence(kind: str, text: str, where: str) -> None: f"found {len(fences)}" ) for raw in fences[0].splitlines(): - required = re.match(r"#\s*requires:\s*(.+)$", raw.strip()) + stripped = raw.strip() + if re.match(r"#\s*requires-contract:", stripped): + gate = re.match( + r"#\s*requires-contract:\s*(.+?)\s*::\s*(.+)$", stripped + ) + assert gate, ( + f"{where} {kind}: {raw!r} — `# requires-contract:` takes " + "` :: `; a malformed gate gates nothing" + ) + req, clause = gate.group(1).strip(), gate.group(2).strip() + assert _in_repo(req), ( + f"{where} {kind}: `# requires-contract:` path {req!r} " + "escapes the repo" + ) + assert (REPO / req).is_file(), ( + f"{where} {kind}: `# requires-contract:` names {req!r} " + "which does not exist at HEAD — a typo'd gate gates nothing" + ) + assert clause in (REPO / req).read_text(), ( + f"{where} {kind}: `# requires-contract:` clause {clause!r} " + f"is not in this repo's own {req} — a typo'd clause gates " + "nothing" + ) + continue + required = re.match(r"#\s*requires:\s*(.+)$", stripped) if required: req = required.group(1).strip() - assert ".." not in req and not req.startswith("/"), ( + assert _in_repo(req), ( f"{where} {kind}: `# requires:` path {req!r} escapes the repo" ) assert (REPO / req).is_file(), ( @@ -63,20 +104,36 @@ def _machine_fence(kind: str, text: str, where: str) -> None: "not exist at HEAD — a typo'd gate gates nothing" ) continue - entry = raw.split("#", 1)[0].strip() + entry, _, comment = raw.partition("#") + entry = entry.strip() if not entry: continue assert entry.startswith("- "), ( f"{where} {kind}: {raw!r} is not a `- path` line" ) path = entry[2:].strip() - assert ".." not in path and not path.startswith("/"), ( + assert _in_repo(path), ( f"{where} {kind}: {path!r} escapes the repo" ) assert (REPO / path).is_file(), ( f"{where} {kind}: {path!r} does not exist at HEAD " "— the machine would act on nothing or the wrong thing" ) + # A per-file gate is the WHOLE trailing comment, `requires: ` + # from its first character; prose comments that merely mention the + # word stay prose. + per_file = re.match(r"\s*requires:\s*(.+)$", comment) + if per_file: + gate_path = per_file.group(1).strip() + assert _in_repo(gate_path), ( + f"{where} {kind}: per-file gate on {path!r} escapes the " + f"repo: {gate_path!r}" + ) + assert (REPO / gate_path).is_file(), ( + f"{where} {kind}: per-file gate on {path!r} names " + f"{gate_path!r} which does not exist at HEAD — a typo'd " + "gate gates nothing" + ) def test_kit_files_exist_and_are_not_ignored():