render.py

#!/usr/bin/env python3
"""render.py -- page images and bbox crops.

Turns a (doc, page, bbox) triple from the data contract in README.md into a PNG
a human can look at. Used as a library by review.py and as a CLI so the crop
arithmetic can be eyeballed without the GUI.

The one piece of arithmetic that must be right
----------------------------------------------
`pdftotext -bbox-layout` reports word boxes in POINTS with a TOP-LEFT origin,
measured in the page's **MEDIA box** -- not its crop box. That is easy to get
backwards, and when a PDF's two boxes differ the crops come out subtly offset,
which still looks like a crop, so nobody notices.

Measured, not assumed. A synthetic PDF with MediaBox [0 0 400 400] and CropBox
[50 100 350 380], with text whose glyph box starts 100pt from the media left,
makes pdftotext emit `<page width="400" height="400">` and `xMin="100"` -- media
box on both counts. `pdftoppm` also uses the media box unless told `-cropbox`,
so the correct pairing is **plain pdftoppm against the media box**, and passing
`-cropbox` is the bug rather than the fix. It matters here: some source
documents carry crop boxes offset from their media boxes.

So:

  * render with pdftoppm's default box, and take page dimensions from the
    MediaBox that pdfinfo reports for that specific page (they vary per page --
    one observed document has page 1 at 612x791 and page 20 at 612.24x791.04);
  * take the scale from the ACTUAL rendered pixel size divided by the media box
    size in points, per axis, rather than trusting dpi/72 -- pdftoppm rounds to
    whole pixels, and a rounded page is a fractional-pixel drift at the far edge
    (that same page 20 at 200dpi measures 2.778322 px/pt against a nominal
    2.777778);
  * a rotated page (/Rotate 90 or 270) would need the bbox rotated too, which
    no source page here requires, so we refuse loudly instead of guessing.

    px = (pt - media_origin_pt) * image_px / media_size_pt

Sources
-------
A crop is requested as `(source_id, page, bbox)`. Source ids are resolved
through `sources.json` -- the registry `extract.py` owns:

    {"id": "core-rules", "file": "Core Rules.pdf",
     "system": "core", "pages": 353, "text_layer": "native", "precedence": 100}

If that file does not exist yet, resolution degrades to a built-in alias table
and then to a filename match against the staging directory, so this module is
usable before extract.py lands. A source id may also be a bare filename, or --
with allow_path=True, never from HTTP -- a path.

For a loose PNG (the deferred OCR case) there is no point space: bboxes are
taken to be pixel coordinates in that image and `dpi`/`page` are ignored.

Duplicate documents
-------------------
The library contains two byte-identical pairs under different filenames, so two
source ids can name one file. `same_file()` / `duplicate_groups()` exist so that
callers never report one document agreeing with itself as corroboration.

CLI
---
    render.py sources
    render.py size  --source core-rules --page 65
    render.py page  --source core-rules --page 65 [--dpi 150] [--out p.png]
    render.py crop  --source core-rules --page 65 --bbox 32.4,130.6,82.9,137.6
    render.py probe --source core-rules --page 65 --bbox ...  # the pixel maths
    render.py words --source core-rules --page 65 --bbox ...  # what the text
                                                             # layer claims
    render.py dupes

`probe` + `words` together are the alignment check: `words` says what the text
layer claims is in the region, the PNG from `crop` says what is actually there.
(`--doc` is accepted as a synonym for `--source`.)
"""

from __future__ import annotations

import argparse
import hashlib
import html.parser
import io
import json
import os
import re
import shutil
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path

try:
    from PIL import Image, ImageDraw
except ImportError:  # pragma: no cover - PIL is a documented prerequisite
    Image = None
    ImageDraw = None

POINTS_PER_INCH = 72.0

DEFAULT_PAGE_DPI = 150
DEFAULT_CROP_DPI = 200

# Context around a bbox, in points.
#
# Horizontal is deliberately large. A numeric cell without its row label is not
# checkable -- you can see "2/5" but not what it is 2/5 OF. Measured against a
# sample cost table on p65: the row label ends at x=82.9 and the first value
# column sits at x=143.8, so 60pt of context falls ~1pt short of the label and
# produces a crop that looks fine and answers nothing. 120pt reaches it.
#
# Vertical is about two and a half body-text lines, enough to show the rows
# either side so a row-offset error is visible.
DEFAULT_MARGIN_X_PT = 120.0
DEFAULT_MARGIN_Y_PT = 24.0

MIN_DPI, MAX_DPI = 36, 600

HERE = Path(__file__).resolve().parent

IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".gif"}

# Fallback tokens for use before sources.json exists. sources.json always wins.
# Populate with short-code -> filename entries for whatever corpus is staged,
# e.g. {"CORE": "Core Rules.pdf"}.
DOC_ALIASES: dict[str, str] = {}


class RenderError(RuntimeError):
    """Anything that stops us producing an honest image."""


# --------------------------------------------------------------------------
# locations
# --------------------------------------------------------------------------


def staging_dir() -> Path:
    """Where the source PDFs and loose page PNGs live.

    Defaults to ./sources, matching the --staging-dir convention used
    elsewhere in this toolset. Override with EXTRACTOR_STAGING_DIR.
    """
    env = os.environ.get("EXTRACTOR_STAGING_DIR")
    if env:
        return Path(env).expanduser()
    return Path("./sources")


def cache_root() -> Path:
    return Path(os.environ.get("EXTRACTOR_CACHE", HERE / "cache")).expanduser()


def data_dir() -> Path:
    """Where sources.json / extracted/ / decisions.jsonl live -- this repo."""
    return Path(os.environ.get("EXTRACTOR_DATA", HERE)).expanduser()


def sources_path() -> Path:
    return data_dir() / "sources.json"


_sources_cache: tuple[float, dict[str, dict]] | None = None


def load_sources() -> dict[str, dict]:
    """The `sources.json` registry, keyed by id. Empty dict if absent.

    extract.py owns this file; we only ever read it, and we work without it.
    Accepts either a bare list of entries or {"sources": [...]}.
    """
    global _sources_cache
    path = sources_path()
    try:
        mtime = path.stat().st_mtime
    except OSError:
        _sources_cache = None
        return {}
    if _sources_cache and _sources_cache[0] == mtime:
        return _sources_cache[1]

    try:
        raw = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}  # a half-written registry must not take the GUI down

    entries = raw.get("sources", []) if isinstance(raw, dict) else raw
    if isinstance(raw, dict) and not entries:
        # also tolerate {"<id>": {...}, ...}
        entries = [
            {"id": k, **v} for k, v in raw.items() if isinstance(v, dict)
        ]

    out: dict[str, dict] = {}
    for e in entries:
        if isinstance(e, dict) and e.get("id"):
            out[str(e["id"])] = e
    _sources_cache = (mtime, out)
    return out


def source_entry(source_id: str) -> dict | None:
    srcs = load_sources()
    if source_id in srcs:
        return srcs[source_id]
    lowered = source_id.lower()
    for k, v in srcs.items():
        if k.lower() == lowered:
            return v
    return None


def resolve_source(source_id: str, *, allow_path: bool = False) -> Path:
    """Map a source id to a file on disk.

    Order: sources.json -> built-in alias -> exact filename in staging ->
    unique case-insensitive substring match.

    allow_path=True additionally accepts a filesystem path. Never pass input
    from the network with allow_path=True -- the point of the id whitelist is
    that an HTTP query string cannot name arbitrary files.
    """
    if not source_id:
        raise RenderError("no source given")

    stage = staging_dir()

    entry = source_entry(source_id)
    if entry:
        # `aliases` holds the other filenames the same bytes live under -- the
        # registry deduplicates by md5, so an alias is a second name for this
        # one document, not a second document.
        candidates = [entry.get("path"), entry.get("file")]
        aliases = entry.get("aliases")
        if isinstance(aliases, list):
            candidates += [a for a in aliases if isinstance(a, str)]
        for candidate in candidates:
            if not candidate:
                continue
            p = Path(candidate).expanduser()
            p = p if p.is_absolute() else stage / p
            if p.is_file():
                return p.resolve()
        raise RenderError(
            f"sources.json lists {source_id!r} as {entry.get('file')!r}, "
            f"which is not in {stage}"
        )

    if allow_path:
        p = Path(source_id).expanduser()
        if p.is_file():
            return p.resolve()

    if "/" in source_id or "\\" in source_id or source_id.startswith("."):
        raise RenderError(f"not an allowed source id: {source_id!r}")

    alias = DOC_ALIASES.get(source_id.upper())
    if alias and (stage / alias).is_file():
        return (stage / alias).resolve()

    direct = stage / source_id
    if direct.is_file():
        return direct.resolve()

    needle = source_id.lower()
    hits = [
        p
        for p in sorted(stage.glob("*"))
        if p.is_file()
        and (p.suffix.lower() == ".pdf" or p.suffix.lower() in IMAGE_SUFFIXES)
        and needle in p.name.lower()
    ]
    if len(hits) == 1:
        return hits[0].resolve()
    if len(hits) > 1:
        names = ", ".join(p.name for p in hits)
        raise RenderError(f"source id {source_id!r} is ambiguous: {names}")
    raise RenderError(f"no source matching {source_id!r} in {stage}")


# `source.doc` in the original single-book contract; same resolution.
resolve_doc = resolve_source


def source_meta(source_id: str) -> dict:
    """What the GUI needs to label a crop: id, system, text_layer, path.

    Unknown-to-the-registry sources still get a usable answer, flagged so the
    UI can say "not in sources.json" rather than silently implying `native`.
    """
    entry = source_entry(source_id) or {}
    meta = {
        "id": source_id,
        "system": entry.get("system"),
        "text_layer": entry.get("text_layer"),
        "precedence": entry.get("precedence"),
        "file": entry.get("file"),
        "pages": entry.get("pages"),
        "registered": bool(entry),
    }
    try:
        path = resolve_source(source_id)
        meta["path"] = str(path)
        meta["filename"] = path.name
        meta["fingerprint"] = file_fingerprint(path, entry)
        meta["resolved"] = True
    except RenderError as exc:
        meta["resolved"] = False
        meta["error"] = str(exc)
    return meta


def known_sources() -> list[dict]:
    """Every source id resolve_source() would accept, for the CLI and the GUI."""
    out: list[dict] = []
    seen: set[str] = set()

    for sid in load_sources():
        out.append(source_meta(sid))
        seen.add(sid.lower())

    if not out:  # registry absent: fall back to the staging directory
        stage = staging_dir()
        for token, filename in DOC_ALIASES.items():
            if (stage / filename).is_file() and token.lower() not in seen:
                seen.add(token.lower())
                out.append(source_meta(token))
        known_paths = {d.get("path") for d in out}
        for p in sorted(stage.glob("*.pdf")):
            if str(p.resolve()) not in known_paths:
                out.append(source_meta(p.name))
    return out


# --------------------------------------------------------------------------
# duplicate documents -- one file must never look like two agreeing sources
# --------------------------------------------------------------------------

_fp_cache: dict[str, str] = {}


def file_fingerprint(path: Path, entry: dict | None = None) -> str:
    """Content identity for "is this the same document twice?".

    Prefers the full `md5` that extract.py already computed and recorded in
    sources.json -- it is authoritative and free. Falls back to size plus md5
    of the first and last 1MB, because hashing ~600MB of PDFs on every GUI
    start is not worth it, and that is still enough to catch the byte-identical
    pairs the README warns about.
    """
    if entry and entry.get("md5"):
        return f"md5:{entry['md5']}"
    path = Path(path)
    key = str(path)
    st = path.stat()
    stamped = f"{key}|{st.st_size}|{st.st_mtime_ns}"
    if _fp_cache.get(stamped):
        return _fp_cache[stamped]

    chunk = 1 << 20
    h = hashlib.md5(usedforsecurity=False)
    h.update(str(st.st_size).encode())
    with path.open("rb") as f:
        h.update(f.read(chunk))
        if st.st_size > chunk:
            f.seek(max(0, st.st_size - chunk))
            h.update(f.read(chunk))
    fp = f"{st.st_size}-{h.hexdigest()[:16]}"
    _fp_cache[stamped] = fp
    return fp


def same_file(a: str, b: str) -> bool:
    """True if two source ids resolve to the same bytes.

    Callers use this so that "two sources agree" can never be one document
    agreeing with itself -- some corpora stage duplicate copies of the same
    file under different names (see README).
    """
    if a == b:
        return True
    try:
        pa, pb = resolve_source(a), resolve_source(b)
    except RenderError:
        return False
    if pa == pb:
        return True
    fa = file_fingerprint(pa, source_entry(a))
    fb = file_fingerprint(pb, source_entry(b))
    if fa.startswith("md5:") and fb.startswith("md5:"):
        return fa == fb
    if pa.stat().st_size != pb.stat().st_size:
        return False
    return file_fingerprint(pa) == file_fingerprint(pb)


def duplicate_groups() -> list[list[str]]:
    """Groups of source ids that share a file. Only groups of 2+ are returned."""
    by_fp: dict[str, list[str]] = {}
    for meta in known_sources():
        if meta.get("resolved") and meta.get("fingerprint"):
            by_fp.setdefault(meta["fingerprint"], []).append(meta["id"])
    return [ids for ids in by_fp.values() if len(ids) > 1]


# --------------------------------------------------------------------------
# page geometry
# --------------------------------------------------------------------------


def is_image_source(path: Path) -> bool:
    return path.suffix.lower() in IMAGE_SUFFIXES


def _run(cmd: list[str]) -> str:
    try:
        proc = subprocess.run(cmd, capture_output=True, text=True, check=False)
    except FileNotFoundError as exc:
        raise RenderError(f"{cmd[0]} not found on PATH") from exc
    if proc.returncode != 0:
        raise RenderError(
            f"{cmd[0]} failed ({proc.returncode}): {proc.stderr.strip() or '(no stderr)'}"
        )
    return proc.stdout


_BOX_RE = re.compile(
    r"^(?:Page\s+\d+\s+)?(\w+Box):\s+"
    r"(-?[\d.]+)\s+(-?[\d.]+)\s+(-?[\d.]+)\s+(-?[\d.]+)\s*$",
    re.M,
)
_ROT_RE = re.compile(r"^(?:Page\s+\d+\s+)?rot:\s+(-?\d+)\s*$", re.M)
_PAGES_RE = re.compile(r"^Pages:\s+(\d+)\s*$", re.M)


def page_count(pdf: Path) -> int:
    if is_image_source(Path(pdf)):
        return 1
    m = _PAGES_RE.search(_run(["pdfinfo", str(pdf)]))
    return int(m.group(1)) if m else 0


@dataclass(frozen=True)
class Geometry:
    """Everything needed to turn points into pixels for one rendered page."""

    doc: str            # resolved path, as a string
    page: int
    dpi: int
    # media box in PDF user space, as pdfinfo reports it -- the space
    # pdftotext -bbox coordinates are expressed in. For a rotated page the
    # box is stored TRANSPOSED to the displayed dimensions (see page_geometry),
    # because both pdftoppm's raster and pdftotext's word boxes live in the
    # displayed page, not the media box.
    box_x0: float
    box_y0: float
    box_x1: float
    box_y1: float
    img_w: int          # actual rendered pixels
    img_h: int
    is_image: bool = False
    rot: int = 0        # /Rotate; nonzero means bboxes arrive in READING space

    @property
    def width_pt(self) -> float:
        return self.box_x1 - self.box_x0

    @property
    def height_pt(self) -> float:
        return self.box_y1 - self.box_y0

    @property
    def scale_x(self) -> float:
        return self.img_w / self.width_pt

    @property
    def scale_y(self) -> float:
        return self.img_h / self.height_pt

    def to_px(self, bbox: tuple[float, float, float, float]) -> tuple[int, int, int, int]:
        """(xMin, yMin, xMax, yMax) in points, top-left origin -> pixel box.

        The x offset is the media box's left edge. There is no y offset: the
        text-layer y is already measured DOWN from the top of the media box,
        which is where the raster starts too.
        """
        if self.is_image:
            l, t, r, b = (float(v) for v in bbox)
        else:
            x0, y0, x1, y1 = (float(v) for v in bbox)
            l = (x0 - self.box_x0) * self.scale_x
            r = (x1 - self.box_x0) * self.scale_x
            t = y0 * self.scale_y
            b = y1 * self.scale_y
        # floor the top-left, ceil the bottom-right: never crop a pixel of the
        # thing we were asked to show.
        import math

        return (
            int(math.floor(min(l, r))),
            int(math.floor(min(t, b))),
            int(math.ceil(max(l, r))),
            int(math.ceil(max(t, b))),
        )


def page_box(pdf: Path, page: int) -> tuple[float, float, float, float, int]:
    """MediaBox (x0, y0, x1, y1) in points plus the page rotation.

    MediaBox specifically: it is the box pdftotext -bbox measures in and the box
    pdftoppm rasterises by default. Preferring CropBox here would misalign every
    crop from a PDF whose boxes differ.
    """
    out = _run(["pdfinfo", "-box", "-f", str(page), "-l", str(page), str(pdf)])
    boxes = {m.group(1): tuple(float(g) for g in m.groups()[1:]) for m in _BOX_RE.finditer(out)}
    box = boxes.get("MediaBox") or boxes.get("CropBox")
    if not box:
        raise RenderError(f"pdfinfo reported no box for page {page} of {pdf.name}")
    rot_m = _ROT_RE.search(out)
    rot = int(rot_m.group(1)) if rot_m else 0
    return (*box, rot)  # type: ignore[return-value]


def page_geometry(doc: str | Path, page: int, dpi: int = DEFAULT_PAGE_DPI, *,
                  allow_path: bool = True) -> Geometry:
    """Resolve + render (cached) and report the exact points->pixels mapping."""
    pdf = doc if isinstance(doc, Path) else resolve_doc(str(doc), allow_path=allow_path)
    png = render_page(pdf, page, dpi=dpi)
    with Image.open(png) as im:
        img_w, img_h = im.size

    if is_image_source(pdf):
        return Geometry(str(pdf), page, dpi, 0.0, 0.0, float(img_w), float(img_h),
                        img_w, img_h, is_image=True)

    x0, y0, x1, y1, rot = page_box(pdf, page)
    rot %= 360
    if rot:
        # MEASURED on a rotated page of a real scanned source (/Rotate 270, MediaBox 792x612):
        # pdftoppm renders the DISPLAYED page (612x792 there), and pdftotext's
        # word boxes live in that same displayed space -- its page width/height
        # report is the unrotated media box, but the word coordinates are not.
        # So for a quarter turn the geometry box is the media box transposed,
        # and no new offset arises; a rotated page with a nonzero media-box
        # origin has never been seen and is refused rather than guessed at.
        if (x0, y0) != (0.0, 0.0):
            raise RenderError(
                f"page {page} of {pdf.name}: /Rotate {rot} with media box origin "
                f"({x0}, {y0}) -- mapping unverified, refusing"
            )
        if rot in (90, 270):
            x1, y1 = y1, x1
    return Geometry(str(pdf), page, dpi, x0, y0, x1, y1, img_w, img_h, rot=rot)


def points_to_pixels(pt: float, dpi: int) -> float:
    """Nominal conversion. Real crops use Geometry.scale_*, which is measured."""
    return pt * dpi / POINTS_PER_INCH


# --------------------------------------------------------------------------
# rendering
# --------------------------------------------------------------------------


def _cache_path(pdf: Path, page: int, dpi: int) -> Path:
    stem = re.sub(r"[^A-Za-z0-9._-]+", "_", pdf.stem)[:48]
    tag = hashlib.sha1(str(pdf).encode("utf-8")).hexdigest()[:8]
    return cache_root() / "pages" / f"{stem}-{tag}" / f"r{dpi}" / f"p{page:04d}.png"


def render_page(doc: str | Path, page: int, dpi: int = DEFAULT_PAGE_DPI, *,
                force: bool = False, allow_path: bool = True) -> Path:
    """Render one page to PNG and return its path. Cached on disk.

    Re-rendering a page of a 23MB PDF per HTTP request is the difference
    between a usable GUI and an unusable one, so the cache is not optional.
    """
    pdf = doc if isinstance(doc, Path) else resolve_doc(str(doc), allow_path=allow_path)
    if is_image_source(pdf):
        return pdf  # a loose page image is already the render

    if not (MIN_DPI <= dpi <= MAX_DPI):
        raise RenderError(f"dpi {dpi} outside {MIN_DPI}..{MAX_DPI}")
    if page < 1:
        raise RenderError(f"page {page} is not a page number")

    out = _cache_path(pdf, page, dpi)
    if out.is_file() and not force and out.stat().st_mtime >= pdf.stat().st_mtime:
        return out

    out.parent.mkdir(parents=True, exist_ok=True)
    # pdftoppm -singlefile takes a PREFIX and appends ".png" to it verbatim,
    # so the produced name is str(prefix) + ".png" -- not Path.with_suffix(),
    # which would eat a dotted temp tag.
    prefix = out.parent / f"{out.stem}.tmp{os.getpid()}"
    # No -cropbox: pdftoppm's default box is the media box, which is the space
    # pdftotext -bbox reports coordinates in. See the module docstring.
    _run([
        "pdftoppm", "-png", "-singlefile",
        "-r", str(dpi), "-f", str(page), "-l", str(page),
        str(pdf), str(prefix),
    ])
    produced = Path(str(prefix) + ".png")
    if not produced.is_file():
        raise RenderError(f"pdftoppm produced nothing for page {page} of {pdf.name}")
    shutil.move(str(produced), str(out))  # atomic-enough within one filesystem
    return out


def _resolve_margins(margin, margin_x, margin_y) -> tuple[float, float]:
    mx = DEFAULT_MARGIN_X_PT if margin is None else float(margin)
    my = DEFAULT_MARGIN_Y_PT if margin is None else float(margin)
    if margin_x is not None:
        mx = float(margin_x)
    if margin_y is not None:
        my = float(margin_y)
    return max(0.0, mx), max(0.0, my)


def _reading_to_displayed(bbox, geom: Geometry):
    """Map a READING-space bbox onto the displayed page a rotated render shows.

    extract.py stores bboxes for rotated pages in reading orientation (its
    apply_rotation), so a crop request arrives in that space and must be
    turned back before to_px. Exact inverse of extract.py's transform --
    verified round-trip on a real /Rotate 270 page's headings, whose displayed
    coordinates come back to the pdftotext originals to the hundredth.
    """
    rx0, ry0, rx1, ry1 = bbox
    w, h = geom.width_pt, geom.height_pt  # displayed dimensions
    if geom.rot == 270:
        return (ry0, h - rx1, ry1, h - rx0)
    if geom.rot == 90:
        return (w - ry1, rx0, w - ry0, rx1)
    return (w - rx1, h - ry1, w - rx0, h - ry0)  # 180


#: The quarter turn that brings a rotated page's crop to reading orientation.
#: Empirical, per the extract.py anchor: on /Rotate 270 pages the displayed
#: text reads bottom-to-top, so the crop turns 90 degrees clockwise -- PIL's
#: ROTATE_270 (it names the CCW angle). 90 and 180 are the matching inverses.
_READING_TURN = {90: "ROTATE_90", 180: "ROTATE_180", 270: "ROTATE_270"}


def crop_bbox_image(
    doc: str | Path,
    page: int,
    bbox,
    *,
    dpi: int = DEFAULT_CROP_DPI,
    margin: float | None = None,
    margin_x: float | None = None,
    margin_y: float | None = None,
    highlight: bool = True,
    allow_path: bool = True,
):
    """Crop `bbox` (points, top-left origin) with context. Returns a PIL Image.

    The bbox itself is outlined so the reviewer can see which of the visible
    values is the one being asserted -- context is useless if you cannot tell
    the subject from its neighbours.
    """
    if Image is None:
        raise RenderError("PIL is required for cropping")

    bbox = tuple(float(v) for v in bbox)
    if len(bbox) != 4:
        raise RenderError(f"bbox needs 4 numbers, got {len(bbox)}")

    geom = page_geometry(doc, page, dpi, allow_path=allow_path)
    if geom.rot:
        bbox = _reading_to_displayed(bbox, geom)
    mx, my = _resolve_margins(margin, margin_x, margin_y)

    tight = geom.to_px(bbox)
    padded = geom.to_px((bbox[0] - mx, bbox[1] - my, bbox[2] + mx, bbox[3] + my))

    left = max(0, padded[0])
    top = max(0, padded[1])
    right = min(geom.img_w, padded[2])
    bottom = min(geom.img_h, padded[3])
    if right <= left or bottom <= top:
        raise RenderError(
            f"bbox {bbox} maps outside page {page} "
            f"({geom.width_pt:.1f}x{geom.height_pt:.1f}pt)"
        )

    with Image.open(render_page(doc, page, dpi=dpi, allow_path=allow_path)) as src:
        crop = src.convert("RGB").crop((left, top, right, bottom))

    if highlight:
        overlay = Image.new("RGBA", crop.size, (0, 0, 0, 0))
        d = ImageDraw.Draw(overlay)
        box = (tight[0] - left, tight[1] - top, tight[2] - left, tight[3] - top)
        d.rectangle(box, fill=(255, 214, 0, 46))
        d.rectangle(box, outline=(220, 30, 30, 235), width=max(1, round(dpi / 100)))
        crop = Image.alpha_composite(crop.convert("RGBA"), overlay).convert("RGB")

    if geom.rot:
        crop = crop.transpose(getattr(Image.Transpose, _READING_TURN[geom.rot]))

    return crop


def crop_bbox(doc, page, bbox, *, out: Path | None = None, **kw) -> Path:
    """crop_bbox_image() written to a PNG. Returns the path."""
    img = crop_bbox_image(doc, page, bbox, **kw)
    if out is None:
        digest = hashlib.sha1(
            f"{doc}|{page}|{bbox}|{sorted(kw.items())}".encode("utf-8")
        ).hexdigest()[:16]
        out = cache_root() / "crops" / f"{digest}.png"
    out = Path(out)
    out.parent.mkdir(parents=True, exist_ok=True)
    img.save(out, format="PNG")
    return out


def crop_bbox_png(doc, page, bbox, **kw) -> bytes:
    """PNG bytes, for serving straight down a socket."""
    buf = io.BytesIO()
    crop_bbox_image(doc, page, bbox, **kw).save(buf, format="PNG")
    return buf.getvalue()


def page_png(doc, page, dpi: int = DEFAULT_PAGE_DPI, **kw) -> bytes:
    return render_page(doc, page, dpi=dpi, **kw).read_bytes()


# --------------------------------------------------------------------------
# text layer -- the machine-checkable half of "does the crop show the value?"
# --------------------------------------------------------------------------


class _BBoxParser(html.parser.HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.words: list[tuple[str, tuple[float, float, float, float]]] = []
        self._box = None
        self._buf: list[str] = []

    def handle_starttag(self, tag, attrs):
        if tag == "word":
            a = dict(attrs)
            try:
                self._box = (
                    float(a["xmin"]), float(a["ymin"]),
                    float(a["xmax"]), float(a["ymax"]),
                )
            except (KeyError, TypeError, ValueError):
                self._box = None
            self._buf = []

    def handle_data(self, data):
        if self._box is not None:
            self._buf.append(data)

    def handle_endtag(self, tag):
        if tag == "word" and self._box is not None:
            text = "".join(self._buf).strip()
            if text:
                self.words.append((text, self._box))
            self._box = None
            self._buf = []


def page_words(doc: str | Path, page: int, *, allow_path: bool = True):
    """Every word on the page with its bbox in points, from the text layer."""
    pdf = doc if isinstance(doc, Path) else resolve_doc(str(doc), allow_path=allow_path)
    if is_image_source(pdf):
        return []
    tag = hashlib.sha1(str(pdf).encode("utf-8")).hexdigest()[:8]
    cached = cache_root() / "text" / f"{tag}" / f"p{page:04d}.xml"
    if not cached.is_file() or cached.stat().st_mtime < pdf.stat().st_mtime:
        cached.parent.mkdir(parents=True, exist_ok=True)
        _run(["pdftotext", "-bbox-layout", "-f", str(page), "-l", str(page),
              str(pdf), str(cached)])
    parser = _BBoxParser()
    parser.feed(cached.read_text(encoding="utf-8", errors="replace"))
    return parser.words


def words_in_bbox(doc, page, bbox, *, margin: float = 0.0, allow_path: bool = True):
    """Words whose box overlaps `bbox` (grown by `margin` points)."""
    x0, y0, x1, y1 = (float(v) for v in bbox)
    x0, y0, x1, y1 = x0 - margin, y0 - margin, x1 + margin, y1 + margin
    hits = []
    for text, (wx0, wy0, wx1, wy1) in page_words(doc, page, allow_path=allow_path):
        if wx1 > x0 and wx0 < x1 and wy1 > y0 and wy0 < y1:
            hits.append((text, (wx0, wy0, wx1, wy1)))
    return hits


# --------------------------------------------------------------------------
# CLI
# --------------------------------------------------------------------------


def _parse_bbox(s: str) -> tuple[float, float, float, float]:
    parts = [p for p in re.split(r"[,\s]+", s.strip()) if p]
    if len(parts) != 4:
        raise argparse.ArgumentTypeError("bbox must be 4 numbers: xMin,yMin,xMax,yMax")
    try:
        return tuple(float(p) for p in parts)  # type: ignore[return-value]
    except ValueError as exc:
        raise argparse.ArgumentTypeError(f"bbox must be numeric: {exc}") from exc


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest="cmd", required=True)

    def common(p, dpi_default):
        p.add_argument("--source", "--doc", dest="source", required=True,
                       help="source id from sources.json, or a filename / path")
        p.add_argument("--page", type=int, default=1)
        p.add_argument("--dpi", type=int, default=dpi_default)

    sub.add_parser("sources", aliases=["docs"], help="list resolvable sources")
    sub.add_parser("dupes", help="source ids that share one file")

    p = sub.add_parser("size", help="page + raster geometry")
    common(p, DEFAULT_PAGE_DPI)

    p = sub.add_parser("page", help="render a whole page")
    common(p, DEFAULT_PAGE_DPI)
    p.add_argument("--out")
    p.add_argument("--force", action="store_true")

    for name, helptext in (("crop", "crop a bbox with context"),
                           ("probe", "print the crop arithmetic, render nothing")):
        p = sub.add_parser(name, help=helptext)
        common(p, DEFAULT_CROP_DPI)
        p.add_argument("--bbox", type=_parse_bbox, required=True)
        p.add_argument("--margin", type=float, default=None, help="points, both axes")
        p.add_argument("--margin-x", type=float, default=None)
        p.add_argument("--margin-y", type=float, default=None)
        if name == "crop":
            p.add_argument("--out")
            p.add_argument("--no-highlight", action="store_true")

    p = sub.add_parser("words", help="text-layer words inside a bbox")
    common(p, DEFAULT_CROP_DPI)
    p.add_argument("--bbox", type=_parse_bbox, required=True)
    p.add_argument("--margin", type=float, default=0.0)

    args = ap.parse_args(argv)

    try:
        if args.cmd in ("sources", "docs"):
            reg = sources_path()
            print(f"registry: {reg}" + ("" if reg.is_file() else "  (absent -- "
                  "falling back to filename matching)"))
            print(f"staging:  {staging_dir()}\n")
            for d in known_sources():
                flag = "" if d.get("resolved") else "  !! UNRESOLVED"
                print(f"{d['id']:<28} {str(d.get('system') or '-'):<6} "
                      f"{str(d.get('text_layer') or '-'):<7} "
                      f"{str(d.get('pages') or '?'):>4}p  "
                      f"{d.get('filename') or d.get('file') or ''}{flag}")
            for group in duplicate_groups():
                print(f"\n  duplicate file: {' == '.join(group)}")
            return 0

        if args.cmd == "dupes":
            groups = duplicate_groups()
            if not groups:
                print("no duplicate files among resolvable sources")
            for group in groups:
                print(" == ".join(group))
            return 0

        if args.cmd == "size":
            g = page_geometry(args.source, args.page, args.dpi)
            print(f"file      {g.doc}")
            print(f"page      {g.page}   dpi {g.dpi}")
            print(f"media box [{g.box_x0:g} {g.box_y0:g} {g.box_x1:g} {g.box_y1:g}] pt"
                  f"  -> {g.width_pt:g} x {g.height_pt:g} pt")
            print(f"raster    {g.img_w} x {g.img_h} px")
            print(f"scale     x {g.scale_x:.6f}  y {g.scale_y:.6f} px/pt"
                  f"   (nominal dpi/72 = {args.dpi / 72:.6f})")
            return 0

        if args.cmd == "page":
            path = render_page(args.source, args.page, dpi=args.dpi, force=args.force)
            if args.out:
                shutil.copyfile(path, args.out)
                path = Path(args.out)
            print(path)
            return 0

        if args.cmd == "probe":
            g = page_geometry(args.source, args.page, args.dpi)
            mx, my = _resolve_margins(args.margin, args.margin_x, args.margin_y)
            tight = g.to_px(args.bbox)
            b = args.bbox
            padded = g.to_px((b[0] - mx, b[1] - my, b[2] + mx, b[3] + my))
            print(f"page      {g.width_pt:g} x {g.height_pt:g} pt "
                  f"-> {g.img_w} x {g.img_h} px @ {g.dpi} dpi")
            print(f"scale     x {g.scale_x:.6f}  y {g.scale_y:.6f} px/pt")
            print(f"bbox pt   {b[0]:.2f}, {b[1]:.2f}, {b[2]:.2f}, {b[3]:.2f}"
                  f"   ({b[2]-b[0]:.2f} x {b[3]-b[1]:.2f} pt)")
            print(f"bbox px   {tight}   ({tight[2]-tight[0]} x {tight[3]-tight[1]} px)")
            print(f"margin    {mx:g} x {my:g} pt")
            print(f"crop px   {padded}   "
                  f"({padded[2]-padded[0]} x {padded[3]-padded[1]} px, before clamping)")
            return 0

        if args.cmd == "crop":
            out = crop_bbox(
                args.source, args.page, args.bbox,
                dpi=args.dpi, margin=args.margin,
                margin_x=args.margin_x, margin_y=args.margin_y,
                highlight=not args.no_highlight,
                out=Path(args.out) if args.out else None,
            )
            print(out)
            return 0

        if args.cmd == "words":
            hits = words_in_bbox(args.source, args.page, args.bbox, margin=args.margin)
            if not hits:
                print("(no text-layer words in that region)")
            for text, box in hits:
                print(f"[{box[0]:7.2f} {box[1]:7.2f} {box[2]:7.2f} {box[3]:7.2f}]  {text}")
            return 0

    except RenderError as exc:
        print(f"render.py: {exc}", file=sys.stderr)
        return 2

    return 1


if __name__ == "__main__":
    raise SystemExit(main())