diff --git a/build-docs.py b/build-docs.py index 30122c20..91920d98 100644 --- a/build-docs.py +++ b/build-docs.py @@ -18,6 +18,10 @@ --serve Build and serve locally (English only, for development) --skip-gen Skip running gen_redirects.py (use existing configs) --no-api-copy Skip copying API docs to localized sites + --skip-api Reuse existing content/api instead of regenerating API metadata + (~30-40% faster). LOCAL markdown iteration ONLY, with --serve/--lang; + never for testing, CI/CD, or release builds. + --permissive Don't fail the English build on DocFX warnings (local iteration) """ import argparse @@ -81,6 +85,62 @@ def run_command(cmd: list[str], description: str, check: bool = True, fail_on_wa return result.returncode +DOCFX: list[str] = [] # cache; populated on first ensure_docfx() + + +def _local_docfx_available() -> bool: + """True if a dotnet tool manifest in cwd or an ancestor declares docfx. + + Mirrors dotnet's manifest discovery (cwd upward, `.config/` or legacy path, + stop at an isRoot manifest). Filesystem-only — does not verify the tool is + restored; a missing restore surfaces as a clear `dotnet docfx` error at build time. + """ + for d in (Path.cwd(), *Path.cwd().parents): + manifest = next((m for m in (d / ".config" / "dotnet-tools.json", + d / "dotnet-tools.json") if m.is_file()), None) + if manifest is None: + continue + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + if "docfx" in {k.lower() for k in data.get("tools", {})}: + return True + if data.get("isRoot"): + break # root manifest without docfx: dotnet stops searching here too + return False + + +def resolve_docfx() -> list[str]: + """Resolve how to invoke docfx: $DOCFX override, then a repo-pinned local tool + (`dotnet docfx`), then a global/PATH `docfx`. Raises if none is available.""" + override = os.environ.get("DOCFX") + if override: + return override.split() + if shutil.which("dotnet") and _local_docfx_available(): + return ["dotnet", "docfx"] + if shutil.which("docfx"): + return ["docfx"] + raise SystemExit( + "Error: docfx not found.\n" + " Local: dotnet tool install docfx (then `dotnet tool restore`)\n" + " Global: dotnet tool install -g docfx" + ) + + +def ensure_docfx() -> list[str]: + """Return the docfx invocation prefix, resolving (and caching) it on first use.""" + global DOCFX + if not DOCFX: + DOCFX = resolve_docfx() + return DOCFX + + +def run_docfx(args: list[str], description: str, **kwargs) -> int: + """Run docfx with the resolved invocation prefix (global/PATH or `dotnet docfx`).""" + return run_command([*ensure_docfx(), *args], description, **kwargs) + + def get_available_languages() -> list[str]: """Get list of available languages from metadata/languages.json or scan localizedContent/.""" manifest_path = Path("metadata/languages.json") @@ -151,7 +211,7 @@ def prepare_localized_content(lang: str, sync: bool = False) -> int: ) -def build_language(lang: str, sync: bool = False) -> int: +def build_language(lang: str, sync: bool = False, skip_api: bool = False, permissive: bool = False) -> int: """Build documentation for a specific language.""" config_path = f"localizedContent/{lang}/docfx.json" @@ -164,14 +224,20 @@ def build_language(lang: str, sync: bool = False) -> int: result = prepare_localized_content(lang, sync=sync) if result != 0: return result - + # Build the documentation — fail on DocFX warnings only for English (the # authored source). Localized content is Crowdin-managed and may carry - # translation warnings that must not block deployment. - return run_command( - ["docfx", config_path], + # translation warnings that must not block deployment. `permissive` lifts the + # English gate too, for local iteration where transient warnings are expected + # (warnings are still printed, just not fatal); full/CI builds leave it off. + # + # `docfx build` skips API metadata regeneration and reuses the existing + # content/api/*.yml (the _apiSource DLLs don't change between content edits), + # which is ~30-40% faster; bare `docfx` regenerates metadata then builds. + return run_docfx( + [*(["build"] if skip_api else []), config_path], f"Building {lang} documentation", - fail_on_warnings=(lang == "en") + fail_on_warnings=(lang == "en" and not permissive) ) @@ -279,7 +345,9 @@ def main() -> int: parser.add_argument("--list", action="store_true", help="List available languages") parser.add_argument("--serve", action="store_true", help="Build English and serve locally") parser.add_argument("--skip-gen", action="store_true", help="Skip gen_redirects.py") - parser.add_argument("--no-api-copy", action="store_true", help="Skip copying API docs") + parser.add_argument("--no-api-copy", action="store_true", help="Skip copying API docs to localized sites") + parser.add_argument("--skip-api", action="store_true", help="LOCAL markdown iteration only (requires --serve/--lang): reuse existing content/api, ~30-40%% faster. NEVER for testing/CI/CD/releases") + parser.add_argument("--permissive", action="store_true", help="Don't treat English DocFX warnings as build failures (for local iteration; keep full/CI builds strict)") parser.add_argument("--sync", action="store_true", help="Sync English fallback for missing/outdated translations (for local dev)") args = parser.parse_args() @@ -292,7 +360,39 @@ def main() -> int: suffix = " (default)" if lang == "en" else "" print(f" {lang}{suffix}") return 0 - + + # Resolve docfx now so a missing install exits before any build work happens. + ensure_docfx() + + # --skip-api is strictly a fast LOCAL iteration aid for editing markdown: it reuses + # the existing content/api/*.yml instead of regenerating API metadata. It must NEVER + # be used for full builds, testing, CI/CD, or releases (those must regenerate the API). + # Guard it conservatively: require an explicit single-target (--serve or --lang), + # forbid --all / the default all-languages build, and require API metadata to already + # exist so we never silently ship a site with missing or stale API docs. + if args.skip_api: + if args.all or not (args.serve or args.lang): + print( + "Error: --skip-api is for fast LOCAL iteration only and must target a single build.\n" + " Use it with --serve or --lang (e.g. `--serve --skip-api`, `--lang en --skip-api`).\n" + " Never use it for --all, testing, CI/CD, or release builds — omit it so API\n" + " metadata is regenerated.", + file=sys.stderr, + ) + return 1 + if not list(Path("content/api").glob("*.yml")): + print( + "Error: --skip-api reuses existing API metadata, but content/api/*.yml is missing.\n" + " Run a full build once (e.g. `python3 build-docs.py --lang en`) to generate it.", + file=sys.stderr, + ) + return 1 + print("\n" + "=" * 60) + print(" WARNING: --skip-api active — reusing existing content/api/*.yml.") + print(" Fast LOCAL markdown iteration ONLY; API docs may be stale.") + print(" NEVER use --skip-api for testing, CI/CD, or release builds.") + print("=" * 60) + # Run gen_redirects.py first (unless skipped) if not args.skip_gen: result = run_command( @@ -315,7 +415,7 @@ def main() -> int: if args.serve: # Build English only and serve - result = build_language("en", sync=True) + result = build_language("en", sync=True, skip_api=args.skip_api, permissive=args.permissive) if result != 0: return result @@ -329,8 +429,8 @@ def main() -> int: shutil.copy(manifest_src, manifest_dest) print("Copied languages.json to _site/en/") - return run_command( - ["docfx", "serve", "_site/en"], + return run_docfx( + ["serve", "_site/en"], "Serving documentation locally" ) @@ -354,7 +454,7 @@ def main() -> int: # Build all requested languages for lang in build_langs: - result = build_language(lang, sync=args.sync) + result = build_language(lang, sync=args.sync, skip_api=args.skip_api, permissive=args.permissive) if result != 0: return result diff --git a/build_scripts/README.md b/build_scripts/README.md new file mode 100644 index 00000000..bd0a0137 --- /dev/null +++ b/build_scripts/README.md @@ -0,0 +1,238 @@ +# Build tools + +This document covers new build tools: +- `te_script_runner.py`: standalone runner and module for other tools that need to execute C# scripts +- `csharp_doctest.py`: compiles and runs annotated `csharp` code blocks in markdown files +- `check_links.py`: dead link checker for built site + +Existing docfx and localization orchestration can be found in [../README.md](../README.md) + +## Prerequisites + +Required on PATH: + +- `python3` -- 3.10+ (the scripts use 3.10 syntax; validated on 3.14). +- `uv` -- provides `uvx`, for lint and type-check. +- `te` -- the Tabular Editor CLI, for the doc-validation scripts; you should use a build aligned with TE3 release for checking docs. +- `docfx` -- or `dotnet` with the pinned local docfx tool, to build the site. + +Run all scripts from the docs repo root; +paths like `_site` and `content/` resolve relative to the current directory. + +## Contributing and development notes + +Scripts have no build phase. +All Python build scripts are linted and type-checked: + +```shell +$ uvx ruff check --select F,B,SIM,I,UP +$ uvx mypy --strict +``` + +Scripts named herein are all built with a functional core, imperative shell style. +Build small components that deal in pure data, orchestrate these with command dispatch. +Expose useful functions in the command dispatch for testing and future ad-hoc use cases. + +## Doc validation + +### Code block validation and output generation + +`te_script_runner.py` and `csharp_doctest.py` contribute to validating code blocks in docs. +Both need `te` on PATH. + +Current state only validates semantic bridge docs. +The code block annotations can be used in any doc. + +#### `te_script_runner.py` -- generic runner + +Runs (or compile-checks) C# snippets against a throwaway, empty `.bim` model. +Output: +- stdout: the snippets' `Output()` only (te messages at level `output`), newline-joined +- stderr: te's own stderr stream (always passed through), plus the runner's diagnostics parsed from te's JSON -- `[compile-error]`, `[runtime-error]`, and `[]` for other non-output messages + +```shell +$ python3 build_scripts/te_script_runner.py run -e '' # execute (also -f , or stdin) +$ python3 build_scripts/te_script_runner.py check -e '' # compile only (te --dry-run) +$ python3 build_scripts/te_script_runner.py run -e '' -f -e '' # execute multiple scripts in order; also works for `check` +``` + +Operation: +1. Creates a new temporary directory +2. Inits a new empty model in that directory +3. Creates files for all scripts in temp directory (this is necessary to enforce strict ordering) +4. Executes all scripts in order with `te` binary; 1 execution of `te`, no `--save`, so all script results are transient +5. Cleans up temp directory + +Additional sub-commands for testing, validation, and ad-hoc use cases: + +- `python3 build_scripts/te_script_runner.py init`: set up the temp directory and model, returning the path; does not automatically clean up: that's your responsibility if you use this +- `python3 build_scripts/te_script_runner.py summarize`: parses and creates output based on `te`'s output + +Importable in other Python scripts: `run_snippets(snippets, dry_run=..., last_only=...) -> Result`. +`last_only` reports only the final snippet's `Output()`, rather than all snippets outputs. + +#### `csharp_doctest.py` -- doc orchestrator + +Checks annotated C# fences in a markdown file. +The annotation is invisible in rendered docfx output: + +``` + ```csharp {compile} + ```csharp {run id= setup= after= output=} +``` + +Commands (each validates first and bails on a malformed doc): + +```shell +$ python3 build_scripts/csharp_doctest.py validate # check annotation grammar + coverage counts, no CLI calls +$ python3 build_scripts/csharp_doctest.py compile # compile every {compile}/{run} block +$ python3 build_scripts/csharp_doctest.py compare # diff each {run} Output() against its fence +$ python3 build_scripts/csharp_doctest.py update # run {run} blocks, rewrite output blocks in place +``` + +Dealing with C# code blocks in a markdown doc: + +- code block without `csharp` in fence: skipped, but an annotation on such a block is a validation error +- code block with `csharp` and no annotation: skip +- `{compile}`: compile-check only (catches API drift; never executes) +- `{run ...}`: will execute the code block with `te` binary + - `id=`: unique name for this block in the document + - `setup=`: prepends a preamble (see `SETUP_SCRIPTS`, e.g. the sample Metric View); must be registered in the script before it can be used + - `output=`: if true, then this code block must be followed by literal `**Output**` on its own line, followed by a fenced code block; will check or update the output to what the script generates + - `after=`: replays earlier run blocks for their state only (not their output); for docs where many code blocks are provided and expected to be run in sequence + +The grammar for these `run` blocks is explicit: +every option must be provided in every block. + +Exit codes: +- `0`: all blocks in the doc pass +- `1`: a block failed (output mismatch, compile error, runtime error) +- `2`: malformed doc or bad usage (validate bails before any CLI calls). + +Output (note: the opposite split from `te_script_runner.py` -- here the whole report, errors included, is on stdout): +- stdout: the full report -- per-block verdicts, diffs, and compile/runtime error detail +- stderr: te's own stderr (passed through) and harness/operational errors (bad usage, unreadable file) + +Sweep all docs with annotated code blocks for valid annotations, bailing on first error: + +```shell +$ (set -eu; rg '```csharp \{' --glob content/**/*.md --files-with-matches | while read f; do python3 build_scripts/csharp_doctest.py validate $f; done) +``` + +Substitute `compile`, `compare`, or `update` for `validate` in above to run in the same way (bailing on first error). + +This orchestrator uses a thread pool with a thread per code block. +Each invocation of `te` as a sub-process takes ~1-2s, +so parallelizing nets a significant performance gain. + +#### Test fixtures (`test-fixtures/`) + +Small, self-contained inputs that exercise each code path: +a manual regression corpus you run by hand (there are no unit tests). +Run a command against a fixture and eyeball the result. + +Markdown fixtures drive `csharp_doctest.py`: +- `doc-valid.md`: one of every block kind (skip / `{compile}` / `{run}`); everything passes. +- `doc-mismatch.md`: a `{run}` whose `Output()` differs from its fence, so `compare` fails (and `update` would rewrite it; don't call update and commit, instead revert if you do update). +- `doc-compile-drift.md`: a `{run}` calling a nonexistent API, so `compile` fails. +- `err-*.md`: each holds exactly one grammar error (missing option, no output fence, annotation on a non-csharp fence, unknown `after=`, unknown annotation), so `validate` bails. + +`te-*.json` are canned `te --output-format json` outputs that drive `te_script_runner.py summarize` (its pure parser, no `te` needed): +the executed-run and `--dry-run` schemas, each in a success and a failure (compile / runtime) variant. + +```shell +$ python3 build_scripts/csharp_doctest.py validate test-fixtures/doc-valid.md # passes +$ python3 build_scripts/csharp_doctest.py compare test-fixtures/doc-mismatch.md # fails with a diff (nonzero exit) +$ python3 build_scripts/csharp_doctest.py compile test-fixtures/doc-compile-drift.md # fails on the drifted API +$ python3 build_scripts/csharp_doctest.py validate test-fixtures/err-unknown-after.md # bails with the grammar error +$ python3 build_scripts/te_script_runner.py summarize test-fixtures/te-runtime-error.json +``` + +### Link validation + +`check_links.py` validates all links (`href`/`src`) in the built `_site`. +It walks the generated HTML, +resolves each reference to a local file or an external URL, +visits every unique target once, and reports references to broken targets. + +Build the docs site first with `python3 build-docs.py --lang en` or `python3 build-docs.py --all`. + +```shell +$ python3 build_scripts/check_links.py validate # check _site: authored content, on-disk + external; requires built _site/ +$ python3 build_scripts/check_links.py validate stats # add per-host external-fetch diagnostics +$ python3 build_scripts/check_links.py validate local # on-disk checks only, skip external fetching +$ python3 build_scripts/check_links.py validate _site under=en # only check links from pages under _site/en +$ python3 build_scripts/check_links.py validate all # also include generated API and localized pages; noisy, likely unnecessary; requires build with `--all` +``` + +Broken link checks: +- local links (to something defined in this docs repo): the target file must exist + - detect old root links (e.g. ``) and resolve against the live docs site to check redirects; a dead one is an error, not a warning +- external links (to something not defined in this docs repo): must return a 2xx/3xx status; a bad status (e.g. 404) or a transport error (DNS, TLS/certificate, timeout, refused connection) fails +- fragments: ensure the `#anchor` is defined in the body of the target page (local file or fetched external page) +- text links: ensure the literal `:~:text=` string is in the body of the target page (local file or fetched external page) + +Internal failures are errors (nonzero exit): +own-site links, i.e. local files and root-absolute links (the latter checked over HTTP). +External failures are warnings (network issues may be transient, or the target blocks bots), +listed at the end as URLs to verify by hand, each with two counts: +how many docs reference it and how many total times it appears. +Fragment/text failures keep their `#anchor` / `:~:text=` in that list (the bare URL works; +the fragment is what broke); a wholly unreachable URL is listed bare. + +Exit codes: `1` if any internal (own-site) reference is broken, else `0`. External warnings never fail the run, so third-party link rot will not break CI. + +Options to modify output: + +- `local`: skip external URL fetching (on-disk checks only) +- `all`: include generated API and localized pages (default: authored `content/*.md` only) +- `stats`: print per-host external-fetch diagnostics +- `under=`: only check links from pages under `/` + +Every built HTML page under `_site` is walked (to index fragment anchors and collect links site-wide). +But by default only *failures on authored English content* are reported: +pages under `_site/en/` that map back to `content/*.md`, +excluding generated API reference (`en/api/`). +The other-language pages are near-duplicate translations, +so they and the generated API are hidden unless you pass `all` (which floods the report with those duplicates). + +`extract`, `resolve`, `enumerate`, and `fetch` expose the internal stages for testing. + +This script uses a rudimentary scheduler to avoid flooding a single host and getting a 429 storm. +We distribute work across a thread pool and the scheduler interleaves requests to different hosts. +There is a per-host maximum for in-flight requests, +and a per-host cooldown when we encounter 429s, +to avoid slamming a host that has already told us to back off. +On a 429 we honor `Retry-After` (otherwise exponential backoff: 1, 2, 4, ... seconds) +and also decrement that host's in-flight cap as ongoing backpressure; +after a per-URL retry limit we give up and record the last result. + +Reachability uses a `HEAD` request; +a body-downloading `GET` runs only when a fragment or `:~:text=` directive must be verified, +so it never pulls installers or images just to check a link. +A failed `HEAD` falls back to `GET` (some hosts reject `HEAD`), +and known HEAD-hostile hosts skip straight to `GET`. + +#### Reading the `stats` table + +`stats` prints one row per host, worst-first. Columns: + +- `total`: external URLs seen for the host +- `ok`: reachable, and any `#anchor` / `:~:text=` check passed; `total == ok` means all links to this domain were good +- `frag`: reachable (url+path), but a fragment or text check failed +- `bad`: unreachable, total (`= 401 + 403 + 404 + oth + net`) +- `reqs`: fetch attempts, including retries; `total == reqs` means everything succeeded on first fetch +- `429`: rate-limited responses seen +- `401` / `403` / `404`: per-host counts of those statuses +- `oth`: other failing HTTP (5xx, and 4xx that is not 401/403/404) +- `net`: non-HTTP failures (DNS, TLS, timeout, connection reset) +- `wait(s)`: total cooldown time applied to the host (rate limiting on our side for 429s) +- `fb`: HEAD requests that fell back to GET (candidates for the GET-only list) +- `cap`: ending in-flight cap (below the start value means it was throttled down) + +#### Interactive control (long runs) + +A full external run takes minutes; the terminal can query or stop it: + +- Status line (phase, counts, in-flight, cooling hosts) on **Ctrl-T** (macOS/BSD `SIGINFO`), **Ctrl-\\** (`SIGQUIT`, Linux/macOS), or **Ctrl-Break** (Windows `SIGBREAK`). +- **Ctrl-C** stops cleanly and prints a partial report over what was fetched; a second **Ctrl-C** force-quits. diff --git a/build_scripts/check_links.py b/build_scripts/check_links.py new file mode 100644 index 00000000..7e61d244 --- /dev/null +++ b/build_scripts/check_links.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +""" +Standalone dead-link checker for the generated `_site`. + +Walks the built HTML, resolves every href/src to a local file or an external +URL, visits each unique target once, and reports references to broken targets +(missing files, missing fragments). Local failures are errors (nonzero exit); +external failures are warnings (network issues are often transient or the +target blocks bots), emitted as a copy-paste list of URLs to verify by hand. + +The functional core (extract, resolve, validate) is pure; the imperative shell +does filesystem, HTTP, and reporting. +""" + +import http.client +import os +import signal +import sys +import urllib.error +import urllib.request +from collections import Counter, defaultdict, deque +from collections.abc import Callable +from dataclasses import dataclass, field +from html.parser import HTMLParser +from pathlib import Path +from queue import Empty, Queue +from threading import Event, Thread +from time import monotonic +from types import FrameType +from typing import Any, NamedTuple +from urllib.parse import unquote, urldefrag, urlsplit + +# Interactive control: a shared progress view that `main` reads for a Ctrl-T status line, and a stop flag it sets on +# Ctrl-C so the running command can wind down and still emit a partial report. Commands populate the view as they go. + +_STOP_POLL = 1.0 # max seconds the fetch scheduler blocks between stop-flag checks, so Ctrl-C stays responsive + +# Terminal keys that print a status line, by platform: Ctrl-T (BSD/macOS), Ctrl-\ (POSIX; overrides the quit/core +# dump), Ctrl-Break (Windows). We bind whichever the platform defines, so a status key exists everywhere. +_STATUS_SIGNALS = ("SIGINFO", "SIGQUIT", "SIGBREAK") + +_SignalHandler = Callable[[int, FrameType | None], None] + + +@dataclass +class _Progress: + """Live counters for the running command. `main` reads them for the status signal and sets `stop` on Ctrl-C; + commands advance the phase, bump the counters, and check `stop` to bail out early for a partial report.""" + + phase: str = "starting" + pages: int = 0 # built HTML pages scanned (enumerate phase) + links: int = 0 # link references seen while scanning + targets: int = 0 # unique targets discovered to check + urls_total: int = 0 # external URLs queued to fetch + urls_done: int = 0 # external URLs fetched so far + dispatched: int = 0 # URLs handed to the queue but not yet resolved (>= the few workers actively fetching) + workers: int = 0 # size of the fetch worker pool + cooling: dict[str, float] = field(default_factory=dict) # host -> monotonic deadline it is eligible again + started: float = 0.0 # monotonic start time, for elapsed + stop: Event = field(default_factory=Event) + + +def _status_line(progress: _Progress, now: float) -> str: + """A one-line snapshot of our own work for the Ctrl-T status signal (about the run, not OS resource stats).""" + p = progress + head = f"[check_links] phase={p.phase} elapsed={now - p.started:.0f}s" + if p.phase != "fetching external": + return f"{head} pages={p.pages} links={p.links} targets={p.targets}" + cooling = sorted( + ((host, deadline - now) for host, deadline in p.cooling.items() if deadline > now), + key=lambda hw: hw[1], + reverse=True, + ) + detail = "".join(f" {host}({wait:.1f}s)" for host, wait in cooling) # wide when many hosts cool at once; fine + return ( + f"{head} fetched={p.urls_done}/{p.urls_total} dispatched={p.dispatched} " + f"workers={p.workers} cooling={len(cooling)}{detail}" + ) + + +def _install_signal_handlers(on_stop: _SignalHandler, on_status: _SignalHandler) -> dict[int, Any]: + """Wire SIGINT (Ctrl-C, clean stop) and every available status signal (see `_STATUS_SIGNALS`), returning the + prior handlers for restoration. A no-op off the main thread, where handlers cannot be installed.""" + previous: dict[int, Any] = {} + try: + previous[signal.SIGINT] = signal.signal(signal.SIGINT, on_stop) + for name in _STATUS_SIGNALS: + signum = getattr(signal, name, None) + if signum is not None: + previous[signum] = signal.signal(signum, on_status) + except ValueError: + pass # not the main thread; interactive control is simply unavailable + return previous + + +def _restore_signal_handlers(previous: dict[int, Any]) -> None: + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +def _normalize_text(text: str) -> str: + """Collapse whitespace and case-fold, so text-fragment queries match the page regardless of layout or case.""" + return " ".join(text.split()).casefold() + + +class _LinkAnchorParser(HTMLParser): + """Collects href/src references and fragment targets (`id`, and `name` on `a`), and optionally the page text.""" + + def __init__(self, collect_text: bool = False) -> None: + super().__init__(convert_charrefs=True) + self.links: list[tuple[str, int]] = [] # (href, 1-based line in the source HTML) + self.anchors: set[str] = set() + self._collect_text = collect_text + self._chunks: list[str] = [] + self._skip_depth = 0 # inside