diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 509af6a2b1..aa166ddf1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,42 @@ jobs: key: ${{ github.ref }} path: .cache - run: pip install -r requirements.txt - - run: sudo apt update && sudo apt-get install imagemagick pngquant + - run: sudo apt update && sudo apt-get install imagemagick pngquant libimage-exiftool-perl - run: find . -name '*.png' -execdir bash -c 'mogrify -resize '750x>' "$1"' - {} \; - run: find . -name '*.png' -execdir bash -c 'pngquant "$1" --ext .png --force' - {} \; - - run: mkdocs gh-deploy --force + # Nothing is published unless every check below passes. These are the + # pull request checks (tools/docs-checks/), run again right before + # publishing, so a package update since the PR (like selectolax 1.0.1, + # which dropped every image) can't publish a broken site. The checks all + # run even if one fails, so the log shows every problem. A failure stops + # the publish step, and Notify on Failure posts to Discord. + - name: Markdown (front matter, headings, alt text, includes) + run: python tools/docs-checks/check_markdown.py docs + - name: No secrets in text + if: ${{ !cancelled() }} + run: python tools/docs-checks/check_secrets.py docs + - name: Photos (GPS location data) + if: ${{ !cancelled() }} + env: + CHANGED_FILES: "" + run: python tools/docs-checks/check_images.py docs/assets + - name: Build the site + if: ${{ !cancelled() }} + env: + DISABLE_MKDOCS_2_WARNING: "true" + run: mkdocs build -d site + - name: Links, images, stylesheets, menus, and redirects in the built site + if: ${{ !cancelled() }} + run: python tools/docs-checks/check_links.py site docs + - name: Custom domain file (CNAME) is in the build + if: ${{ !cancelled() }} + run: test -s docs/CNAME && cmp docs/CNAME site/CNAME + # Same as `mkdocs gh-deploy --force`, but publishes the build checked + # above instead of building again. Runs only if every step passed. + - name: Publish the checked site to GitHub Pages + run: | + message="Deployed $(git rev-parse --short HEAD) with MkDocs version: $(python -c 'import mkdocs; print(mkdocs.__version__)')" + ghp-import --no-jekyll --push --force --message "$message" site notify: name: Notify on Failure diff --git a/AGENTS.md b/AGENTS.md index 71ec1488b3..4b84e22f90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,8 +67,10 @@ Read the full style guide, but these are the ones worth checking twice: Every pull request runs `tools/docs-checks/` (see its README): front matter, heading levels, alt text, snippet includes, secrets, GPS data in photos, and links and anchors on the built site. Run them locally before -opening a PR. They can't read text inside images, so check screenshots -and recordings for passwords, API keys, and personal details by eye. +opening a PR. The same checks run before every deploy to the live wiki, +and a failure blocks the deploy. They can't read text inside images, so +check screenshots and recordings for passwords, API keys, and personal +details by eye. ## Keep the style guide current diff --git a/mkdocs.yml b/mkdocs.yml index f44d334395..d0cc70ef11 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,9 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.snippets: base_path: "docs" + # Fail the build when an included file is missing, instead of silently + # publishing a page without that content. + check_paths: true - admonition - pymdownx.arithmatex: generic: true diff --git a/tools/docs-checks/README.md b/tools/docs-checks/README.md index 73bf12c69d..01795068e1 100644 --- a/tools/docs-checks/README.md +++ b/tools/docs-checks/README.md @@ -4,12 +4,21 @@ These scripts run on every pull request into `dev` or `main` (`.github/workflows/docs-checks.yml`). Problems show up as comments on the changed lines in the PR. Fix them and push again; the checks re-run. +They run again before every deploy to the live wiki +(`.github/workflows/ci.yml`), and the deploy publishes the same build the +link check looked at. If any check fails, nothing is published, the live wiki +stays as it was, and the team gets a Discord alert. This catches problems +that appear between the PR and the deploy, such as a build package update +that drops images or breaks links. The build itself also fails if a +`--8<--` snippet include points at a missing file (`check_paths` in +`mkdocs.yml`). + | Check | Fails the PR on | How to fix | | --- | --- | --- | | `check_markdown.py` | Missing front matter `title:` or `description:`, heading levels that skip (`##` to `####`), more than one `#` H1, images without alt text, snippet includes that point at a missing file or start inside the included page's front matter | Add the field, use the next heading level, describe the image inside `![...]`, or start the include after the included page's closing `---` | | `check_secrets.py` | ESPHome API encryption keys, generated OTA passwords, passwords typed into YAML, and access tokens | Use `!secret` (or a placeholder like `your-wifi-password`). If the value was real, change it on the device or account too | | `check_images.py` | Photos with GPS location data (phones add it by default) | `exiftool -gps:all= -overwrite_original docs/assets/` | -| `check_links.py` | Broken links, links to a missing `#anchor`, filename case mismatches (GitHub Pages is case-sensitive), images without alt text, and a built site with far fewer images than the Markdown, checked on the built site | Fix the link target or anchor; match the file name's case exactly. If images went missing, look for a recently updated build package and pin it in `requirements.txt` | +| `check_links.py` | Broken links, links to a missing `#anchor`, filename case mismatches (GitHub Pages is case-sensitive), and images without alt text, checked on the built site. Also stylesheets, scripts, and menu links that point at missing files, redirects in `mkdocs.yml` whose target is missing, and images the build dropped: a page that should have images but has none, or a site with less than 90% of the images its Markdown should produce | Fix the link target or anchor; match the file name's case exactly. If images went missing, look for a recently updated build package and pin it in `requirements.txt` | Warnings (shown, but they don't fail the PR): em dashes and the filler words the style guide bans, and images or recordings over 1.5 MB that the PR adds diff --git a/tools/docs-checks/check_links.py b/tools/docs-checks/check_links.py index 8d74f254d2..e9f739d178 100644 --- a/tools/docs-checks/check_links.py +++ b/tools/docs-checks/check_links.py @@ -6,9 +6,13 @@ links (../page/, https://wiki.apolloautomation.com/...), so this checks every link, #anchor, image, and video in page content against the built site. Filename case must match exactly, because GitHub Pages is case-sensitive. -Fails on any broken link, missing anchor, or image without alt text, and -when the built site has far fewer images than the Markdown (a plugin -dropping them, as selectolax 1.0.1 did to mkdocs-glightbox in October 2026). +Fails on any broken link, missing anchor, or image without alt text. Also +fails when the build lost something without saying so: +- images: a page that should have images has none, or the site has less + than 90% of the images its Markdown (with snippet includes expanded) has. + In October 2026, selectolax 1.0.1 made mkdocs-glightbox drop every image. +- stylesheets, scripts, and menu links that point at missing files +- redirects in mkdocs.yml whose page wasn't generated (missing target) """ import os, re, sys, posixpath from html.parser import HTMLParser @@ -22,7 +26,7 @@ class Page(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) - self.ids, self.refs = set(), [] + self.ids, self.refs, self.site_refs = set(), [], [] self.in_article, self.stack = False, 0 def handle_starttag(self, tag, attrs): @@ -33,6 +37,12 @@ def handle_starttag(self, tag, attrs): self.ids.add(a["name"]) if tag == "article" and "md-content__inner" in (a.get("class") or ""): self.in_article, self.stack = True, 0 + if tag == "link" and "stylesheet" in (a.get("rel") or "") and a.get("href"): + self.site_refs.append(("stylesheet", a["href"])) + elif tag == "script" and a.get("src"): + self.site_refs.append(("script", a["src"])) + elif tag == "a" and "md-nav__link" in (a.get("class") or "") and a.get("href"): + self.site_refs.append(("menu link", a["href"])) if not self.in_article: return if tag == "article": @@ -132,23 +142,103 @@ def src_md(url): if frag and kind == "a" and t in pages and frag not in pages[t].ids: bad_anchor.append({"page": src_md(url), "ref": ref}) +# Stylesheets, scripts, and menu links are on every page, so report each once. +broken_site = {} +for url, pr in pages.items(): + for kind, ref in pr.site_refs: + r = resolve(url, ref) + if r is not None and not exists(r[0]): + broken_site.setdefault((kind, r[0]), src_md(url)) + + +# Images the Markdown should produce, with --8<-- snippet includes expanded +# and fenced code blocks skipped. +INCLUDE = re.compile(r'^\s*--8<--\s+"([^"]+)"\s*$') +FENCE = re.compile(r"^\s*(```|~~~)") +IMAGE = re.compile(r"!\[[^\]]*\]\(| 10 or not os.path.isfile(path): + return [] + lines = open(path, encoding="utf-8", errors="replace").read().splitlines() + lines = lines[(start - 1 if start else 0):(end if end else None)] + out = [] + for line in lines: + m = INCLUDE.match(line) + if not m: + out.append(line) + continue + parts = m.group(1).split(":") + s = int(parts[1]) if len(parts) > 1 and parts[1] else None + e = int(parts[2]) if len(parts) > 2 and parts[2] else None + out.extend(expand(parts[0], s, e, depth + 1)) + return out + + +def expected_images(rel): + n, fenced = 0, False + for line in expand(rel): + if FENCE.match(line): + fenced = not fenced + elif not fenced: + n += len(IMAGE.findall(line)) + return n + + +expected_total = built_total = 0 +pages_without_images = [] +for url, pr in pages.items(): + src = src_md(url) + if not src.startswith("docs/"): + continue + want = expected_images(src[len("docs/"):]) + got = sum(1 for kind, _, _ in pr.refs if kind == "img") + expected_total += want + built_total += got + if want and not got: + pages_without_images.append((src, want)) +images_dropped = expected_total > 0 and built_total < 0.9 * expected_total + +# Redirect pages that mkdocs-redirects didn't generate (it skips missing targets). +missing_redirects = [] +if os.path.isfile("mkdocs.yml"): + in_maps = False + for line in open("mkdocs.yml", encoding="utf-8").read().splitlines(): + if line.strip().startswith("redirect_maps:"): + in_maps, indent = True, len(line) - len(line.lstrip()) + continue + if in_maps: + if line.strip() and not line.strip().startswith("#") and len(line) - len(line.lstrip()) <= indent: + break + m = re.match(r"""\s*['"]?([^'":]+\.md)['"]?\s*:""", line) + if m: + key = m.group(1) + page = key[: -len("index.md")] if key.endswith("index.md") else key[:-3] + "/" + if ("/" + page + "index.html").replace("//", "/") not in ALL: + missing_redirects.append(key) + for b in broken: print(f"::error file={b['page']}::Broken {b['kind']} link: {b['ref']}") for a in bad_anchor: print(f"::error file={a['page']}::Link to a missing #anchor: {a['ref']}") for m in missing_alt: print(f"::error file={m['page']}::Image without alt text: {m['src']}") -md_imgs = 0 -for r, _, fs in os.walk(DOCS): - for f in fs: - if f.endswith(".md"): - md_imgs += len(re.findall(r"!\[[^\]]*\]\(| 0 and built_imgs < md_imgs // 2 +for (kind, path), page in sorted(broken_site.items()): + print(f"::error file=mkdocs.yml::Broken {kind} on every page (first seen on {page}): {path}") +for src, want in pages_without_images: + print(f"::error file={src}::The Markdown has {want} image(s), but the built page has none.") if images_dropped: - print(f"::error::The built site has {built_imgs} images, but the Markdown has about {md_imgs}. A build plugin is probably removing images; check recently updated packages.") + print(f"::error::The built site has {built_total} images, but the Markdown should produce about {expected_total}. A build plugin is probably removing images; check recently updated packages.") +for key in missing_redirects: + print(f"::error file=mkdocs.yml::The redirect for {key} wasn't generated. Its target page is probably missing.") for wrong, right in sorted(set(case_mismatch)): print(f"::error::Link case doesn't match the file (GitHub Pages is case-sensitive): {wrong} should be {right}") -total = len(broken) + len(bad_anchor) + len(missing_alt) + len(set(case_mismatch)) + images_dropped -print(f"check_links: {len(pages)} pages, {built_imgs} images, {len(broken)} broken, {len(bad_anchor)} bad anchors, {len(missing_alt)} missing alt, {len(set(case_mismatch))} case mismatches") +total = (len(broken) + len(bad_anchor) + len(missing_alt) + len(set(case_mismatch)) + len(broken_site) + + len(pages_without_images) + images_dropped + len(missing_redirects)) +print(f"check_links: {len(pages)} pages, {built_total} of {expected_total} expected images, {len(broken)} broken, " + f"{len(bad_anchor)} bad anchors, {len(missing_alt)} missing alt, {len(set(case_mismatch))} case mismatches, " + f"{len(broken_site)} broken stylesheets/scripts/menu links, {len(pages_without_images)} pages missing images, " + f"{len(missing_redirects)} missing redirects") sys.exit(1 if total else 0)