diff --git a/agents/README.md b/agents/README.md index 44e6ab47dad..75c2aaca091 100644 --- a/agents/README.md +++ b/agents/README.md @@ -30,7 +30,7 @@ The model is **Claude Haiku 5.5 on Vertex AI** (`claude-haiku-5-5`) by default. | `anyplot/data/generate.py` | The formula source behind `generate_data`: the `Generate` spec (one or two variable ranges, one to ten series), the whitelist evaluator over Python's `ast` (NumPy float64, never `eval`), the canonical CSV it writes, and `looks_like_formula`, the dataset route's `422 formula` guard | | `anyplot/plugins/` | `ScopeGuardPlugin`, `BudgetPlugin`, `ToolSafetyPlugin` and the request ledger they share | | `anyplot/output_guard.py` | The output sanitiser of every root message and plot-event line (links, HTML and long code blocks out, at most 1,200 characters) and the reply cap the ScopeGuard runs on the root's final reply before it is stored: a reply over 1,200 characters becomes the fixed refusal, the same in every language, or is cut at 1,200 characters when the turn ran the plot pipeline | -| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8/G9/G10), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, `serial.py`, the one render semaphore in front of every backend, and `watermark.py`, the footer strip the artifact route and the feedback bundle add to the raw render | +| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8/G9/G10/G11/G12), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, `serial.py`, the one render semaphore in front of every backend, and `watermark.py`, the footer strip the artifact route and the feedback bundle add to the raw render | | `anyplot/render/fonts/` | JetBrains Mono 2.304 Regular and Bold for the footer strip, unmodified, with `OFL.txt` (SIL Open Font License 1.1) and a notice with their SHA-256 values ([fonts README](anyplot/render/fonts/README.md)) | | `renderer/` | The `anyplot-renderer` service: `main.py` (`POST /render`, `GET /status`, one render slot), `executor.py` (one `sandbox do` per theme with the kill path, the byte watchdog and the launcher retry), `auth.py` (the caller check), `settings.py` (`RENDERER_*`), `wire.py` (the JSON contract the `remote` backend shares), `Dockerfile` and `cloudbuild.yaml`. No ADK import and no `__init__.py`, so `adk web agents` does not list it as an agent | | `anyplot/opening.py`, `session_state.py`, `services.py`, `briefs.py` | Opening a session and taking in a dataset, the server-set session state, the process-wide stores, and the spec and dataset briefs | @@ -403,7 +403,7 @@ Before the first case the harness renders the catalogue file of the first case o --renderer remote --render-url https://RENDERER_URL --gcloud-token ``` -- The full matrix is the 123 cases that are not stress cases. Cap the spend with `--budget-usd`: +- The full matrix is the 127 cases that are not stress cases. Cap the spend with `--budget-usd`: ```bash uv run --extra agents python -m agents.evals.matrix --cases full --budget-usd 4 \ diff --git a/agents/anyplot/language.py b/agents/anyplot/language.py index 28ec5f536dd..ab16f26c110 100644 --- a/agents/anyplot/language.py +++ b/agents/anyplot/language.py @@ -18,6 +18,7 @@ """ import re +from dataclasses import dataclass from typing import Literal @@ -74,6 +75,53 @@ """The English name of each code in `_NAMES`: server text, so a prompt may name the language outside a fence.""" +@dataclass(frozen=True) +class NumberFormat: + """How the text in a plot writes numbers in one language: its decimal and thousands marks and its short words + for large numbers, each as `(power of ten, word)`, smallest first.""" + + decimal: str + thousands: str + suffixes: tuple[tuple[int, str], ...] + + +NUMBER_FORMATS: dict[str, NumberFormat] = { + "en": NumberFormat(".", ",", ((3, "k"), (6, "M"), (9, "B"))), + "de": NumberFormat(",", ".", ((3, "Tsd."), (6, "Mio."), (9, "Mrd."))), + "fr": NumberFormat(",", " ", ((3, "k"), (6, "M"), (9, "Md"))), + "es": NumberFormat(",", ".", ((3, "mil"), (6, "M"), (9, "mil M"))), + "it": NumberFormat(",", ".", ((3, "mila"), (6, "Mln"), (9, "Mld"))), + "pt": NumberFormat(",", ".", ((3, "mil"), (6, "mi"), (9, "bi"))), + "nl": NumberFormat(",", ".", ((3, "k"), (6, "mln"), (9, "mld"))), + "pl": NumberFormat(",", " ", ((3, "tys."), (6, "mln"), (9, "mld"))), + "ru": NumberFormat(",", " ", ((3, "тыс."), (6, "млн"), (9, "млрд"))), + "vi": NumberFormat(",", ".", ((3, "N"), (6, "Tr"), (9, "Tỷ"))), + "zh": NumberFormat(".", ",", ((4, "万"), (8, "亿"))), + "ja": NumberFormat(".", ",", ((4, "万"), (8, "億"))), +} +"""The number format of each language in `ENGLISH_NAMES`, the server's own table (Q4 of the live test of 2026-10-14: +a German plot showed "6.0 M" and "5.30"). `locale` is a banned import in plot code, so the adapter formats with +`FuncFormatter` or an f-string and these marks (`number_format_table`), and gate G12 measures the decimal mark. +Chinese and Japanese group large numbers by ten thousand (万) and a hundred million (亿, 億). A language +outside the table has no number rule and no G12.""" + + +def number_format(language: str) -> NumberFormat | None: + """The number format of a session language (`de`, `Deutsch` and `German` share one), or None outside the table.""" + return NUMBER_FORMATS.get(canonical(language)) + + +def number_format_table() -> str: + """The table as Markdown rows for the adapter's static instruction: code, language, marks and words.""" + names = {" ": "space", " ": "narrow no-break space", " ": "no-break space"} + rows = ["| Plot language | Decimal mark | Thousands mark | Large numbers |", "|---|---|---|---|"] + for code, spec in NUMBER_FORMATS.items(): + words = ", ".join(f"10^{power}: {word}" for power, word in spec.suffixes) + thousands = names.get(spec.thousands, f"`{spec.thousands}`") + rows.append(f"| `{code}` ({ENGLISH_NAMES[code]}) | `{spec.decimal}` | {thousands} | {words} |") + return "\n".join(rows) + + def normalise(value: str) -> str: """The stored form of a language. diff --git a/agents/anyplot/pipeline.py b/agents/anyplot/pipeline.py index ce09d0579bf..240dedfd9df 100644 --- a/agents/anyplot/pipeline.py +++ b/agents/anyplot/pipeline.py @@ -358,7 +358,8 @@ class Candidate: adaptation_rules: list[str] = field(default_factory=list) """The ADAPTATION validator rule ids behind `adaptation_lines`, for the attribution log.""" advisory_gates: list[str] = field(default_factory=list) - """The advisory probe gate ids (`G3`, `G5`, `G7`, `G8`, `G9`, `G10`) behind `advisory_lines`, for the attribution log.""" + """The advisory probe gate ids (`G3`, `G5`, `G7`, `G8`, `G9`, `G10`, `G11`, `G12`) behind `advisory_lines`, for the + attribution log.""" review_score: int | None = None """The catalogue review's score of this render, when it had one (`AGENT_REVIEWER_MODE=catalogue`).""" review_lines: list[str] = field(default_factory=list) @@ -933,7 +934,8 @@ def finish(run: Run) -> PlotResult: if shipped.canvas_line: residual.append(shipped.canvas_line) # The measured findings are facts of the shipped render, reviewed or not: the ADAPTATION findings and - # every advisory gate line (clipping G3, annotations outside G5, overlap G7, G8, G9, missing glyphs G10). + # every advisory gate line (clipping G3, annotations outside G5, overlap G7, G8, G9, missing glyphs G10, + # crowded per-mark labels G11, the plot language's decimal mark G12). # No review score hides them (until 2026-10-14 only G10 stayed above the ship bar). residual += shipped.adaptation_lines residual += shipped.advisory_lines @@ -1436,7 +1438,7 @@ async def _attempts( started = time.monotonic() rendered = await services.backend.render(job) render_s = time.monotonic() - started - report = evaluate(rendered, themes=job.themes, library=view.library, rows=rows) + report = evaluate(rendered, themes=job.themes, library=view.library, rows=rows, plot_language=run.language) if report.passed_host_gates: run.repair_hints = list(report.repair_hints) attribution( diff --git a/agents/anyplot/policy.py b/agents/anyplot/policy.py index 48cc8221455..fcbf592f835 100644 --- a/agents/anyplot/policy.py +++ b/agents/anyplot/policy.py @@ -40,6 +40,8 @@ import yaml +from .language import number_format_table + PACKAGE_DIR = Path(__file__).resolve().parent PROMPTS_DIR = PACKAGE_DIR / "prompts" @@ -263,6 +265,7 @@ def adapter_instruction(library: str) -> str: # The rubric holds the VQ-03 marker-size and alpha table (`vq03_table`), so it is not repeated. return _join( own_prompt("adapter.md"), + "## Number formats by plot language\n\n" + number_format_table(), highlights, "## Excerpt: which weaknesses to fix\n\n" + weaknesses, "## Excerpt: visual-sizing fixes\n\n" + sizing, diff --git a/agents/anyplot/prompts/adapter.md b/agents/anyplot/prompts/adapter.md index 27aa3a23242..6321e2b4ce2 100644 --- a/agents/anyplot/prompts/adapter.md +++ b/agents/anyplot/prompts/adapter.md @@ -67,6 +67,8 @@ These rules replace the catalogue's own rules where they differ: - **Keep the plot.** Same plot type, same variant, same encodings and the same style. Change only what the user's data, the hints, the feedback and the change request need. Add no encodings, reference lines, highlights or callouts that nobody asked for. - **Callouts and statistics.** A label or callout that states a number must compute it from `df` in an f-string (`f"r = {df[x].corr(df[y]):.2f}"`, written with the column variables of the code). A callout target comes from a rule over `df` (the maximum, the latest row), and a conditional claim is drawn only when `df` meets it. Remove a callout that cannot be derived. - **Density.** Adapt marker sizes, line widths and alpha to the user's row count (see the VQ-03 table below). +- **Per-mark labels and decorations.** "The same style" covers elements whose count stays fixed (a title, a legend, one callout), not elements the catalogue draws once per bar or point of its few example marks: value labels, dots on bar ends, shadows. Their count grows with the user's data, so keep them only while each label fits the space of its mark and the plot stays legible, which rarely holds past about 25 marks; with more marks, drop them, or label only the extremes (the largest and the smallest) when the plot needs numbers. The server measures this (a `VQ-03` gate line counts the labels, the decorations and the pixels each label has). +- **Numbers.** Write every number in the plot text (tick labels, value labels, annotations, a number in the title) with the plot language's decimal and thousands marks and its words for large numbers, from the table "Number formats by plot language" below: `de` writes `5,30` and `6,5 Mio.`, `en` writes `5.30` and `6.5 M`. Python's `locale` is not available: format with an f-string, which writes `,` for thousands and `.` for decimals, then swap both marks through a placeholder: `f"{v:,.2f}".replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`, with `DECIMAL` and `THOUSANDS` set once to the table's marks (`de`: `","` and `"."`; `fr`: `","` and `" "`; `en` keeps the f-string as it is), through a `matplotlib.ticker.FuncFormatter` for the axes and the same function for value labels (`ax.bar_label(container, labels=[...])`, never `fmt="%.2f"`, which always writes a point). Years, dates, identifiers and the cells of the user's data stay as they are. A plot language the table does not list keeps the catalogue's formats. The server checks the decimal mark of every numeric tick label and annotation (a `VQ-06` gate line). - **Dates.** Use `matplotlib.dates.AutoDateLocator` with `ConciseDateFormatter` instead of a fixed locator, unless the change request asks for a specific one. - **Safe code.** Use only the imports the code already has, plus, when needed, `numpy`, `pandas`, the plotting library, scipy's computing modules (`scipy.stats`, `scipy.interpolate`, `scipy.signal`, `scipy.optimize`, `scipy.fft`, `scipy.linalg`, ...) and these standard modules: `math`, `cmath`, `statistics`, `decimal`, `fractions`, `numbers`, `datetime`, `calendar`, `zoneinfo`, `collections`, `itertools`, `functools`, `operator`, `bisect`, `heapq`, `typing`, `warnings`, `textwrap`, `colorsys` and `re`. Never import `io`, `sys`, `pathlib`, `random`, `locale` or `string`. No file or network access, no `os` beyond the THEME line, no `.format(...)` (use f-strings), no `eval`, `exec`, `getattr`, `operator.attrgetter` or other reflection, no names with double underscores or private names of a module (`module._name`), no `class` statements, and no string literal longer than 200 characters. diff --git a/agents/anyplot/prompts/reviewer.md b/agents/anyplot/prompts/reviewer.md index 4804b726fdb..f8b858efc4d 100644 --- a/agents/anyplot/prompts/reviewer.md +++ b/agents/anyplot/prompts/reviewer.md @@ -40,7 +40,7 @@ Do not judge anything else: not the catalogue title format, not the realism of t These are never defects: -- The house style: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had, such as a shadow, an outline, a grid or a muted reference line. Data claims are not styling: a callout at an example value is still DQ-03. +- The house style: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had, such as a shadow, an outline, a grid or a muted reference line. Data claims are not styling: a callout at an example value is still DQ-03. Nor is an element drawn once per bar or point: value labels, dots on bar ends or shadows that the catalogue drew on a few example marks and the user's data repeats on many are judged like any other element (VQ-03 when they crowd the marks). - Column names, units and category labels written the way the user's data writes them, in every plot language; a text whose language you cannot judge. - A gate note on its own (see "The request"). - Something that could be better but fails no criterion above. diff --git a/agents/anyplot/prompts/scorer.md b/agents/anyplot/prompts/scorer.md index 44953f87f99..f615cb6252a 100644 --- a/agents/anyplot/prompts/scorer.md +++ b/agents/anyplot/prompts/scorer.md @@ -22,7 +22,7 @@ The rubric below was written for catalogue implementations that generate their o 3. **No catalogue title.** The mandated catalogue title (`{spec-id} · {language} · {library} · anyplot.ai`) does not apply: a title that names the user's data is correct, and the 5d note on the long mandated title is about that format only. SC-04 asks whether the title and the legend describe the user's data. The adapter keeps the user's column names on purpose: an axis title, legend title or label that is a bound column name (the bindings block lists them) or a category from the data, exactly as the data writes it, is correct and never a VQ-06 or SC-04 deduction, even when it is terse, abbreviated, has no unit or is in another language than the plot language. Units and category labels keep the spelling of the user's data too. Only a text the code added, a sentence or phrase rather than a name from the data, that is plainly in another language than the plot language is a VQ-06 defect. 4. **One theme.** Apply the theme-readability checklist (5c) only for the attached render's theme, and name that theme, or `code`, in every defect line instead of `both`. -The house style is never a defect: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had. +The house style is never a defect: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had. This covers elements whose count stays fixed, not those drawn once per bar or point: value labels, dots on bar ends or shadows that the catalogue drew on its few example marks and the user's data repeats on many are judged like any other element (VQ-03 when they crowd the marks, VQ-02 when they overlap). ## Your answer diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 9fb0df93a0b..935dc31f10e 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -1,4 +1,4 @@ -"""Host gates on a render: R1-R3 decide, G3/G5/G7/G8/G9/G10 advise. +"""Host gates on a render: R1-R3 decide, G3/G5/G7/G8/G9/G10/G11/G12 advise. | Gate | Checks | Effect | |---|---|---| @@ -11,6 +11,8 @@ | G8 | probe: more point marks than data rows (fabricated data) | advisory: a DQ-03 line | | G9 | probe: scatter marks plus line vertices over `MAX_PLOTTED_POINTS` (computed mass data) | advisory: a DQ-03 line | | G10 | probe: characters without a glyph in any font of their text, plain text and mathtext (the harness records matplotlib's own reports) | advisory: a VQ-01 line | +| G11 | probe (`density`): value labels at a data coordinate on a bar's end or point, dots on bar ends and shadowed bars, per axes: more than `MAX_VALUE_LABELS` labels, bar labels wider than `LABEL_SLOT_FILL` of the space per bar, or more than `MAX_DECORATIONS` decorations | advisory: a VQ-03 line | +| G12 | probe (`labels`, `tick_axes`): numeric annotations, and tick labels of numeric axes (never dates or categories), whose decimal mark is not the plot language's (`language.NUMBER_FORMATS`; `foreign_decimal`, mathtext read as drawn) | advisory: a VQ-06 line | A pipeline run renders one theme (`PipelineArgs.theme`) and the theme toggle renders the other one later, so `evaluate` judges exactly the themes of the job it is given: @@ -36,6 +38,7 @@ from core.canvas import check_canvas +from ..language import NumberFormat, english_name, number_format from ..schemas import MAX_LINE_CHARS from .contract import THEMES, RenderResult, Theme from .png import PngRejected, harden, pad_to @@ -71,6 +74,15 @@ of 50 columns, so the owner raised it to 500,000 (2026-10-14): a dense plot of the user's own data, such as 20,000 rows of 24 series (480,000 points), passes, while the stress cases' 610,000 and 1,000,000 points still trip it.""" +MAX_VALUE_LABELS = 25 +"""G11: value labels (texts at a data coordinate on a bar's end or a point) one axes may carry. Q3 of the live test of +2026-10-14: the catalogue's bar-grouped labels, dots and shadows, meant for its 12 bars, on 52 bars of 26 cantons.""" +LABEL_SLOT_FILL = 0.9 +"""G11: a bar label wider than this share of the space between neighbouring bar centres crowds its neighbours.""" +MIN_CRAMPED_LABELS = 6 # fewer labels than this are a few callouts, never crowded +MAX_DECORATIONS = 25 # G11: dots on bar ends and shadowed bars one axes may carry +MAX_DENSITY_AXES = 8 # `harness.DENSITY_AXES_LIMIT`: more entries are a tampered probe, read no further +MAX_CANVAS_PX = 10_000.0 # a pixel size past any canvas the gates accept is a tampered probe G10_SAMPLE = 3 # code points the G10 line names G10_NAME_CHARS = 15 # name text per label: a label is then at most 26 characters and the line fits MAX_NOTE_CHARS MAX_CODEPOINT = 0x10FFFF @@ -149,7 +161,7 @@ class GateReport: failed_gates: list[str] = field(default_factory=list) """The ids of the gates that failed or reported, one entry per theme and finding: `R1`, `R1-timeout`, `R1-` for a renderer limit (`R1-memory`, `R1-disk_budget`, ...), `R2`, - `R3`, then the probe gates `G3`, `G5`, `G7`, `G8`, `G9`, `G10` in the order the probe checks run + `R3`, then the probe gates `G3`, `G5`, `G7`, `G8`, `G9`, `G10`, `G11`, `G12` in the order the probe checks run (not sorted). Content-free, for the attribution log.""" @property @@ -256,11 +268,18 @@ def data_rows(data_csv: str) -> int: return max(0, len([line for line in data_csv.split("\n") if line]) - 1) -def evaluate(result: RenderResult, *, themes: Sequence[Theme], library: str, rows: int) -> GateReport: - """Run R1-R3 and the advisory gates over every theme of the job; a theme without an output fails R1.""" +def evaluate( + result: RenderResult, *, themes: Sequence[Theme], library: str, rows: int, plot_language: str | None = None +) -> GateReport: + """Run R1-R3 and the advisory gates over every theme of the job; a theme without an output fails R1. + + `plot_language` is the session language the plot text is written in; G12 checks the + decimal mark of its numbers when the language has a number format (`language.NUMBER_FORMATS`). + """ wanted = [theme for theme in THEMES if theme in themes] if not wanted: raise ValueError("a render job names at least one theme") + numbers = number_format(plot_language) if plot_language else None report = GateReport(passed_host_gates=True, canvas_ok=True) advisory: dict[str, list[Theme]] = {} missed: list[Theme] = [] @@ -307,7 +326,7 @@ def evaluate(result: RenderResult, *, themes: Sequence[Theme], library: str, row if verdict.defect_line and verdict.defect_line not in report.canvas_defects: report.canvas_defects.append(_line(verdict.defect_line)) report.padded_pngs[theme] = pad_to(hardened.data, verdict.target) - for gate, line in _probe_lines(output.probe, rows): + for gate, line in _probe_lines(output.probe, rows, numbers, plot_language): report.failed_gates.append(gate) advisory.setdefault(line, []).append(theme) # Both themes draw the same geometry, so the hint comes from the first theme that has one. @@ -424,12 +443,229 @@ def _text_kind(value: Any) -> str: return value if isinstance(value, str) and value in TEXT_KINDS else "text" -def _probe_lines(probe: dict[str, Any] | None, rows: int) -> list[tuple[str, str]]: - """Advisory `(gate, line)` pairs from one theme's probe; a malformed probe yields none, never an exception.""" - try: - return _probe_lines_unchecked(probe, rows) - except (ValueError, TypeError, OverflowError, AttributeError): +def _probe_lines( + probe: dict[str, Any] | None, rows: int, numbers: NumberFormat | None = None, plot_language: str | None = None +) -> list[tuple[str, str]]: + """Advisory `(gate, line)` pairs from one theme's probe; a malformed probe yields none, never an exception. + + Each gate is guarded on its own, so a malformed field of one gate never hides another's line. + """ + lines: list[tuple[str, str]] = [] + checks = ( + lambda: _probe_lines_unchecked(probe, rows), + lambda: _density_lines(probe), + lambda: _number_format_lines(probe, numbers, plot_language), + ) + for check in checks: + try: + lines += check() + except (ValueError, TypeError, OverflowError, AttributeError): + continue + return lines + + +def _count(value: Any) -> int | None: + """A non-negative int of the probe, or None (a bool, a float or a negative number is a tampered probe).""" + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _pixels(value: Any) -> float | None: + """A positive finite pixel size of the probe, or None.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + number = float(value) + return number if math.isfinite(number) and 0 < number <= MAX_CANVAS_PX else None + + +def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: + """The G11 line: per-mark value labels or decorations past what the plot's marks leave room for. + + The probe's `density` field (`harness.density_report`) describes each axes; the line + names the worst one. Labels crowd when more than `MAX_VALUE_LABELS` bar positions carry + one (`labelled_bars`: a stacked bar's segment labels count once), or, on an axes without + bars, more than that many numeric labels sit on points (`numeric_labels`: named points + are the point of scatter-annotated), + or when a bar label is wider than `LABEL_SLOT_FILL` of the space between bar centres + (from `MIN_CRAMPED_LABELS` labels on); decorations (dots on bar ends, shadowed bars) + when there are more than `MAX_DECORATIONS`. Only counts and pixels reach the line. + """ + entries = probe.get("density") if isinstance(probe, dict) else None + if not isinstance(entries, list): return [] + worst: tuple[int, str] | None = None + for entry in entries[:MAX_DENSITY_AXES]: + if not isinstance(entry, dict): + continue + bars, marks = _count(entry.get("bars")), _count(entry.get("marks")) + labelled, decorated = _count(entry.get("labelled")) or 0, _count(entry.get("decorated")) or 0 + if bars is None or marks is None: + continue + numeric = _count(entry.get("numeric_labels")) + positions = _count(entry.get("labelled_bars")) + # With bars the cap counts labelled bar positions, so a stacked bar's segment labels count once; without + # bars only numbers count: named points (scatter-annotated asks for 10 to 50) do not. + if bars: + capped = labelled if positions is None else min(positions, labelled) + else: + capped = labelled if numeric is None else min(numeric, labelled) + slot, label = _pixels(entry.get("slot_px")), _pixels(entry.get("label_px")) + cramped = ( + bars > 0 + and labelled >= MIN_CRAMPED_LABELS + and slot is not None + and label is not None + and label > slot * LABEL_SLOT_FILL + ) + many_labels = capped > MAX_VALUE_LABELS + many_decorations = decorated > MAX_DECORATIONS + if not (cramped or many_labels or many_decorations): + continue + observed: list[str] = [] + if cramped or many_labels: + room = f" ({round(label)} px wide on {round(slot)} px per bar)" if cramped and label and slot else "" + observed.append(f"{labelled if bars else capped} value labels{room}") + if many_decorations: + observed.append(f"{decorated} dots or shadows") + marks_text = f"{bars} bars" if bars else f"{marks} marks" + line = ( + f"VQ-03 (THEME): {marks_text} carry {' and '.join(observed)} → at most {MAX_VALUE_LABELS} labels, each " + "fitting its mark, and no per-mark dots or shadows past that: drop them or label only the extremes. " + "Likely cause: the catalogue's per-mark labels, dots or shadows repeated on every mark of the data." + ) + severity = labelled + decorated + if worst is None or severity > worst[0]: + worst = (severity, line) + return [] if worst is None else [("G11", worst[1])] + + +_NUMBER_LABEL = re.compile( + r"^[+\-−]?(?:[$€£¥]\s?)?[+\-−]?(?P\d[\d.,    ']*\d|\d)" + r"(?:\s?(?:%|‰|[$€£¥]|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?$" +) +"""A label that is one number: an optional sign and currency sign in either order, digits with marks, an optional +unit, word or trailing currency sign (`%`, `Mio.`, `k`, `mil M`, `5,30 €`, `-$5.30`). Dates (`14.10.`, +`2024-10-14`), ranges, times and words with digits never match.""" +_DAY_MONTH = re.compile(r"^(0[1-9]|[12]\d|3[01])[.](0[1-9]|1[0-2])$") +"""`14.10`: a day and month as `%d.%m` writes them, both zero-padded. Read as a date in an annotation, where no axis +says what it is; a decimal written with two digits each side (`10.12`) is the one number this lets through.""" +NUMBER_LABEL_KINDS = frozenset({"annotation", "text"}) +"""The label kinds whose numbers G12 checks besides every tick label (`x tick label`, `y tick label`, `z tick label`, +`colorbar tick label`, `harness.drawn_labels`); titles, axis titles and legend entries may name a version or a code.""" + + +NUMERIC_TICKS = "number" +"""`harness.tick_axes`'s kind for an axis whose tick labels are numbers; dates, categories and labels the code set +itself (`fixed`) are never checked.""" +_MATH_MARKUP = re.compile(r"\\mathdefault|\\mathrm|\\,|[{}$]") + + +def plain_text(text: str) -> str: + """A label as it reads on the canvas when it is one mathtext string: `$\\mathdefault{5.30}$` reads `5.30` and + `$5{,}30$` reads `5,30`. Any other text stays as it is, a currency dollar included.""" + stripped = text.strip() + if len(stripped) > 2 and stripped.startswith("$") and stripped.endswith("$") and stripped.count("$") == 2: + return _MATH_MARKUP.sub("", stripped).replace("\\ ", " ").strip() + return stripped + + +def _number_label_kind(label: dict[str, Any], ticks: list[Any] | None) -> str | None: + """`tick labels` or `annotations` for a label G12 checks, else None. + + An annotation or text always counts. A tick label counts only when the probe's + `tick_axes` (`harness.tick_axes`) names its axis numeric: an axis of dates (`14.10`) or + of the data's own categories (`3.11`) writes no decimals. Without `tick_axes` (a renderer + older than G12) no tick label counts. + """ + kind = label.get("kind") + if not isinstance(kind, str): + return None + if kind in NUMBER_LABEL_KINDS: + return "annotations" + if not kind.endswith("tick label") or ticks is None: + return None + index = label.get("axes") + if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(ticks): + return None + entry = ticks[index] + if not isinstance(entry, dict): + return None + names = ("x", "y") if kind == "colorbar tick label" else (kind[0],) + return "tick labels" if any(entry.get(name) == NUMERIC_TICKS for name in names) else None + + +def foreign_decimal(text: str, numbers: NumberFormat) -> bool: + """Whether a numeric label writes its decimals with the other mark than the plot language's. + + Only a label that is one number counts (`_NUMBER_LABEL`). With a decimal comma, a single + point followed by other than three digits (`5.30`, `6.0 M`) is foreign, and so is a point + after a comma (`1,234.5`) or more than one comma (`1,234,567`); a single point with three + digits (`1.234`) is a thousands mark and passes, unless the whole part is 0 (`0.125`). A + zero-padded day and month (`14.10`, `_DAY_MONTH`) is a date and passes. A decimal point mirrors this, except that + several points (`7.184.003`) pass, because a version-like label writes them too. + """ + shown = plain_text(text) + match = _NUMBER_LABEL.match(shown) + if match is None or _DAY_MONTH.match(shown): + return False + body = match.group("body") + marks = [char for char in body if char in ".,"] + if not marks: + return False + other = "." if numbers.decimal == "," else "," + own = numbers.decimal + if own in marks and other in marks: + return marks[-1] == other # the decimal mark comes last + if other not in marks: + return own == "," and marks.count(own) > 1 # `1,234,567` groups with commas: English + if marks.count(other) > 1: + return False # `7.184.003` or `1,234,567` group thousands the other way: ambiguous, never named + whole, decimals = body.rsplit(other, 1) + # `0.125` is never a thousands group: a leading 0 makes the mark a decimal one. + return len(decimals) != 3 or whole == "0" + + +def _number_format_lines( + probe: dict[str, Any] | None, numbers: NumberFormat | None, plot_language: str | None +) -> list[tuple[str, str]]: + """The G12 line: numeric tick labels or annotations whose decimal mark is not the plot language's. + + Reads the probe's `labels` field (`{"kind", "text", "axes"}` per drawn text) and its + `tick_axes`, which say which tick labels are numbers (`_number_label_kind`). No label text + reaches the line: a tick label can be a cell of the user's data. The language is named + by its English name from the server's table, never by the session's free-text value. + """ + labels = probe.get("labels") if isinstance(probe, dict) else None + if numbers is None or not isinstance(labels, list): + return [] + ticks = probe.get("tick_axes") if isinstance(probe, dict) else None + ticks = ticks if isinstance(ticks, list) else None + numeric = foreign = 0 + kinds: set[str] = set() + for label in labels: + kind = _number_label_kind(label, ticks) if isinstance(label, dict) else None + if kind is None: + continue + text = label.get("text") + if not isinstance(text, str) or _NUMBER_LABEL.match(plain_text(text)) is None: + continue + numeric += 1 + if foreign_decimal(text, numbers): + foreign += 1 + kinds.add(kind) + if not foreign: + return [] + name = english_name(plot_language or "") or "the plot language" + own, other = ("comma", "point") if numbers.decimal == "," else ("point", "comma") + thousands = {".": "a point", ",": "a comma"}.get(numbers.thousands, "a space") + words = ", ".join(word for _, word in numbers.suffixes) + return [ + ( + "G12", + f"VQ-06 (THEME): {foreign} of {numeric} numeric {' and '.join(sorted(kinds))} use a decimal {other}; " + f"{name} writes a decimal {own} → decimal {own}, {thousands} for thousands, {words} for large numbers. " + 'Likely cause: a default formatter or fmt="%.2f" instead of a FuncFormatter or f-string with these marks.', + ) + ] def _probe_lines_unchecked(probe: dict[str, Any] | None, rows: int) -> list[tuple[str, str]]: diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 4a6cce3d64d..0428d2df7c5 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -105,7 +105,10 @@ The probe feeds the advisory gates G3 (clipped text), G5 (annotation outside its axes), G7 (overlapping text: tick labels among themselves, any other text with any), G8 (more point marks than data rows), G9 (more plotted points than any dataset within the parser's caps can hold) and G10 -(characters without a glyph in any font of the text). It is +(characters without a glyph in any font of the text); the `density` field each save adds after the +probe (`density_report`: per axes, the bars and points and the value labels and decorations on them) +feeds G11, and `tick_axes` (what each axis's tick labels write: numbers, dates, categories) tells G12 +which tick labels are numbers. It is advisory because the code under test could tamper with it; the host never fails a render on it. The same holds for the `HARNESS` lines: the code under test shares stdout, so the host reads them as hints (a retry decision, a memory figure), never @@ -119,6 +122,7 @@ import logging import math import os +import re import resource import shutil import sys @@ -184,6 +188,11 @@ POLAR_BIN_DEG = 5 # width of one angle bin of the plotted points in the probe's `polar` field POLAR_RINGS = 10 # radius bins: tenths of the visible radius range, centre to edge POLAR_AXES_LIMIT = 4 # polar axes the probe describes per figure +DENSITY_AXES_LIMIT = 8 # axes the probe's `density` field describes per figure +DENSITY_LABEL_LIMIT = 500 # data-anchored texts per axes matched against the marks +DENSITY_MARK_LIMIT = 2_000 # bar ends and points per axes a text is matched against +DENSITY_BAR_SLACK = 0.12 # a label's anchor sits within this share of the axes' value extent past its bar's end +DENSITY_POINT_SLACK = (0.02, 0.08) # ... and within these shares of the axes' width and height of its point GLYPH_SAMPLE_LIMIT = 16 # code points the probe lists; the host shows a few of them MISSING_GLYPHS: set[int] = set() # code points reported by matplotlib's text layout, this process MISSING_MATH_GLYPHS: set[int] = set() # code points reported by mathtext (no fallback inside `$…$`) @@ -1285,6 +1294,314 @@ def polar_report(figure): return report +def _in_view(x, y, box, slack=1.0): + """Whether a display point lies within the axes box (a mark or anchor outside it is clipped, never drawn).""" + return box.x0 - slack <= x <= box.x1 + slack and box.y0 - slack <= y <= box.y1 + slack + + +def _bar_ends(axes, box): + """Every visible bar of the axes inside its view as `(centre, end, width, base)` in display pixels along its + orientation, plus a flag whether it draws a path effect (a shadow), and the orientation of the first bar container. + + For a vertical bar the centre is its x centre, the end the y of the side away from the + baseline, the base the y of the baseline side and the width its pixel width; a + horizontal bar swaps the axes. End and base come from the bar's data coordinates, so an + inverted or logarithmic value axis puts the end where the bar ends on the canvas. A bar + with a missing value (NaN geometry) or a rectangle outside the axes box (a zoomed view) + is left out: it draws nothing, and a NaN would void the probe's JSON. A bar whose + rectangle meets the view counts, even when its centre lies past a value-axis limit. + """ + bars = [] + orientation = None + for container in axes.containers: + if type(container).__name__ != "BarContainer": + continue + horizontal = getattr(container, "orientation", "vertical") == "horizontal" + orientation = orientation or ("horizontal" if horizontal else "vertical") + for patch in container.patches: + if not patch.get_visible(): + continue + extent = patch.get_window_extent() + x, y, w, h = patch.get_x(), patch.get_y(), patch.get_width(), patch.get_height() + if horizontal: + (end, _), (base, _) = axes.transData.transform([(x + w, y), (x, y)]) + bar = ((extent.y0 + extent.y1) / 2, end, extent.y1 - extent.y0, base) + else: + (_, end), (_, base) = axes.transData.transform([(x, y + h), (x, y)]) + bar = ((extent.x0 + extent.x1) / 2, end, extent.x1 - extent.x0, base) + # Drawn when its rectangle meets the view: a bar zoomed past on its value axis still shows its end. + meets = extent.x0 <= box.x1 and extent.x1 >= box.x0 and extent.y0 <= box.y1 and extent.y1 >= box.y0 + if not all(math.isfinite(float(value)) for value in bar) or not meets: + continue + bars.append((*(float(value) for value in bar), bool(patch.get_path_effects()))) + return bars, orientation + + +def _point_marks(axes, box): + """The display positions of the axes' visible point marks (scatter marks, and the markers of a `plot` call that + draws no line, `ax.plot(x, y, "o")`) and of the vertices of its lines of three or more, finite and inside the + axes box only.""" + import numpy as np + + scatter, vertices = [], [] + for collection in axes.collections: + if type(collection).__name__ != "PathCollection" or not collection.get_visible(): + continue + offsets = np.asarray(collection.get_offsets(), dtype=float).reshape(-1, 2) + scatter.append(collection.get_offset_transform().transform(offsets)) + for line in axes.get_lines(): + if not line.get_visible(): + continue + xy = np.asarray(line.get_xydata(), dtype=float).reshape(-1, 2) + markers_only = line.get_linestyle() in ("None", "none", "", " ") and line.get_marker() not in (None, "None", "") + if markers_only: + scatter.append(line.get_transform().transform(xy)) + elif len(xy) >= 3: # two-vertex lines are rules, brackets and reference lines + vertices.append(line.get_transform().transform(xy)) + empty = np.empty((0, 2)) + + def kept(parts): + points = np.concatenate(parts) if parts else empty + inside = ( + np.isfinite(points).all(axis=1) + & (points[:, 0] >= box.x0 - 1) + & (points[:, 0] <= box.x1 + 1) + & (points[:, 1] >= box.y0 - 1) + & (points[:, 1] <= box.y1 + 1) + ) + return points[inside] + + return kept(scatter), kept(vertices) + + +def _label_anchors(axes, box, renderer): + """The visible texts of the axes placed at a data coordinate, with their anchors in display pixels. + + An annotation counts when its `xycoords` is "data" (`ax.annotate`, `ax.bar_label`), any + other text when it uses the data transform (`ax.text` at a data position). A callout in + axes coordinates is no value label. Like matplotlib's own draw, an annotation whose point + lies outside the axes is skipped unless it is set to draw there (`annotation_clip=False`), + and so is a plain text clipped to the axes (`clip_on=True`) outside them; a text at an + infinite position is skipped too. + """ + found = [] + for text in axes.texts: + if not text.get_visible() or not text.get_text().strip(): + continue + if type(text).__name__ == "Annotation": + if callable(text.xycoords): + # `bar_label(label_type="center")` anchors at a share of its bar's box: a callable returning it. + try: + frame = text.xycoords(renderer) + anchor = (frame.x0 + text.xy[0] * frame.width, frame.y0 + text.xy[1] * frame.height) + except Exception: + continue + elif text.xycoords == "data": + anchor = axes.transData.transform(text.xy) + else: + continue + clipped = text.get_annotation_clip() is not False + elif text.get_transform() == axes.transData: + anchor = axes.transData.transform(text.get_position()) + clipped = bool(text.get_clip_on()) + else: + continue + x, y = float(anchor[0]), float(anchor[1]) + if not (math.isfinite(x) and math.isfinite(y)) or (clipped and not _in_view(x, y, box)): + continue + found.append((text, x, y)) + if len(found) >= DENSITY_LABEL_LIMIT: + break + return found + + +def _on_bar_ends(scatter, bar_array, horizontal, value_extent): + """Which scatter marks sit on a bar's end, matched in chunks of `DENSITY_MARK_LIMIT` marks so the pairwise + arrays stay at most `DENSITY_MARK_LIMIT` squared, however many marks the plot draws.""" + import numpy as np + + on_bar = np.zeros(len(scatter), dtype=bool) + if not len(bar_array) or not len(scatter): + return on_bar + for start in range(0, len(scatter), DENSITY_MARK_LIMIT): + chunk = scatter[start : start + DENSITY_MARK_LIMIT] + along, across = (chunk[:, 1], chunk[:, 0]) if horizontal else (chunk[:, 0], chunk[:, 1]) + near_centre = np.abs(along[:, None] - bar_array[None, :, 0]) <= bar_array[None, :, 2] / 2 + 1 + near_end = np.abs(across[:, None] - bar_array[None, :, 1]) <= 0.03 * value_extent + on_bar[start : start + len(chunk)] = (near_centre & near_end).any(axis=1) + return on_bar + + +def density_report(figure, dpi): + """Per axes (at most `DENSITY_AXES_LIMIT`, only those with bars or point marks): how many marks it draws and how + many of them carry a value label or a decoration, with the space a label has. + + `bars` counts the visible bars, `marks` bars plus scatter marks plus line vertices + (point marks on a bar's end count as decorations instead), `labelled` the texts at a + data coordinate whose anchor sits on a mark (at or past a bar's end, inside a bar's + segment, or on a point; `_label_anchors`), `numeric_labels` those of them that are one + number (`_is_number_text`: a named point, `Firm 3`, is not), `labelled_bars` the bar + positions that carry a label (a stacked bar's segments share one), `decorated` the point marks + on a bar's end plus the bars that draw a path effect such as a shadow, `slot_px` the + space one bar label has along the category axis (`_label_slot`, from the labelled bars + only) and `label_px` the median extent of the bar labels along that axis. Pixels are the + saved image's. The host turns this into G11. + """ + import numpy as np + + scale = (dpi or figure.dpi) / figure.dpi + renderer = figure.canvas.get_renderer() + report = [] + for axes in figure.get_axes(): + if len(report) >= DENSITY_AXES_LIMIT: + break + try: + box = axes.get_window_extent(renderer) + bars, orientation = _bar_ends(axes, box) + scatter, vertices = _point_marks(axes, box) + except Exception: + continue + if not bars and not len(scatter) and not len(vertices): + continue + horizontal = orientation == "horizontal" + width, height = float(box.width), float(box.height) + value_extent = width if horizontal else height + bar_array = np.asarray([bar[:3] for bar in bars[:DENSITY_MARK_LIMIT]], dtype=float).reshape(-1, 3) + # A scatter mark on a bar's end is a decoration (the catalogue's dots on its bar tops), not a data mark. + on_bar = _on_bar_ends(scatter, bar_array, horizontal, value_extent) + points = np.concatenate([scatter[~on_bar], vertices])[:DENSITY_MARK_LIMIT] + segments = np.asarray([bar[3] for bar in bars[:DENSITY_MARK_LIMIT]], dtype=float) + slack = DENSITY_BAR_SLACK * value_extent + labelled, numeric, label_sizes, labelled_bars = 0, 0, [], [] + for text, x, y in _label_anchors(axes, box, renderer): + along, across = (y, x) if horizontal else (x, y) + hit = None + if len(bar_array): + # On a bar: past its end by at most `slack`, or inside its segment (`label_type="center"`). + low = np.minimum(bar_array[:, 1], segments) - slack + high = np.maximum(bar_array[:, 1], segments) + slack + matches = ( + (np.abs(along - bar_array[:, 0]) <= bar_array[:, 2] / 2 + 1) & (across >= low) & (across <= high) + ) + if matches.any(): + hit = int(np.argmax(matches)) + on_a_point = len(points) and bool( + ( + (np.abs(x - points[:, 0]) <= DENSITY_POINT_SLACK[0] * width + 2) + & (np.abs(y - points[:, 1]) <= DENSITY_POINT_SLACK[1] * height + 2) + ).any() + ) + if hit is None and not on_a_point: + continue + labelled += 1 + numeric += _is_number_text(text.get_text()) + if hit is not None: + labelled_bars.append((float(bar_array[hit, 0]), float(bar_array[hit, 2]))) + try: + extent = text.get_window_extent(renderer) + except Exception: + continue + size = float(extent.height if horizontal else extent.width) + if math.isfinite(size): + label_sizes.append(size) + slot = _label_slot(labelled_bars) + report.append( + { + "bars": len(bars), + "marks": len(bars) + int((~on_bar).sum()) + len(vertices), + "labelled": labelled, + "numeric_labels": numeric, + "labelled_bars": len({round(centre, 1) for centre, _ in labelled_bars}), + "decorated": int(on_bar.sum()) + sum(1 for bar in bars if bar[4]), + "slot_px": None if slot is None else round(slot * scale, 1), + "label_px": round(float(np.median(label_sizes)) * scale, 1) if label_sizes else None, + } + ) + return report + + +_NUMBER_TEXT = re.compile( + r"[$€£¥]?\s?[+\-−]?\s?[$€£¥]?\s?\d[\d.,'    ]*" + r"(?:\s?(?:%|‰|[$€£¥]|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?" +) +_MATH_MARKUP = re.compile(r"\\mathdefault|\\mathrm|\\,|[{}$]") + + +def _is_number_text(text): + """Whether a label is one number (`5.30`, `6,5 Mio.`, `12 %`, `5,30 €`), mathtext read as drawn; a name is not.""" + stripped = text.strip() + if len(stripped) > 2 and stripped.startswith("$") and stripped.endswith("$") and stripped.count("$") == 2: + stripped = _MATH_MARKUP.sub("", stripped).strip() + return _NUMBER_TEXT.fullmatch(stripped) is not None + + +def _label_slot(labelled_bars): + """The space one bar label has along the category axis: the median distance from each labelled bar's centre to + the nearest other labelled bar's centre, in display pixels, or None for fewer than two. + + Only labelled bars count, so unlabelled bars between them (an offset shadow series, a + chart that labels only its extremes) leave the space as wide as it looks. Distances + below half a bar width are one bar labelled twice (a stacked bar's segments) and are skipped. + """ + import numpy as np + + if len(labelled_bars) < 2: + return None + centres = np.asarray([centre for centre, _ in labelled_bars], dtype=float) + widths = np.asarray([bar_width for _, bar_width in labelled_bars], dtype=float) + distances = np.abs(centres[:, None] - centres[None, :]) + distances[distances < np.median(widths) / 2] = np.inf + nearest = distances.min(axis=1) + nearest = nearest[np.isfinite(nearest)] + return float(np.median(nearest)) if len(nearest) else None + + +def _tick_kind(axis): + """What an axis's tick labels write: `number` (a numeric scale and formatter), `date`, `category`, `fixed` + (labels the code set itself, `set_xticks(positions, labels)`: often the data's own categories), `none` (no labels) + or `other`.""" + major = axis.get_major_formatter() + formatter = type(major).__name__ + if formatter == "NullFormatter": + return "none" + # `set_ticklabels` (and `set_xticks(positions, labels)`) installs a FuncFormatter over a dict of the labels. + labels_dict = getattr(getattr(major, "func", None), "func", None) + if formatter == "FixedFormatter" or getattr(labels_dict, "__name__", "") == "_format_with_dict": + return "fixed" + # A date formatter over coordinates converted beforehand (`mdates.date2num`) leaves the converter unset. + if "Date" in formatter or "dates" in type(major).__module__: + return "date" + get_converter = getattr(axis, "get_converter", None) + converter = get_converter() if get_converter is not None else getattr(axis, "converter", None) + if converter is None: + return "number" + module = type(converter).__module__ + if "dates" in module: + return "date" + if "category" in module: + return "category" + return "other" + + +def tick_axes(figure): + """Per axes of `figure.get_axes()`, in that order (the `axes` index of `drawn_labels`), what each of its axes' + tick labels write (`_tick_kind`), so the host checks the decimal mark (G12) of numeric ticks only, never of a + date (`14.10`) or a category the data names (`3.11`).""" + report = [] + for axes in figure.get_axes(): + entry = {} + for name in ("x", "y", "z"): + axis = getattr(axes, f"{name}axis", None) + if axis is None: + continue + try: + entry[name] = _tick_kind(axis) + except Exception: + entry[name] = "other" + report.append(entry) + return report + + def install_probe(theme): try: from matplotlib.figure import Figure @@ -1305,6 +1622,16 @@ def savefig(self, *args, **kwargs): polar = [] if polar: report["polar"] = polar + try: + density = density_report(self, kwargs.get("dpi")) + except Exception: # nor does the density report + density = [] + if density: + report["density"] = density + try: + report["tick_axes"] = tick_axes(self) + except Exception: # nor does the tick-axes report + pass try: text = json.dumps(report, allow_nan=False) # NaN or Infinity (a text at an infinite coordinate) except (TypeError, ValueError) as exc: diff --git a/agents/anyplot/settings.py b/agents/anyplot/settings.py index f27fb46b0e5..72cfad5998b 100644 --- a/agents/anyplot/settings.py +++ b/agents/anyplot/settings.py @@ -217,7 +217,7 @@ class AgentSettings(BaseSettings): gate_repair: Literal["review", "first"] = "review" """When the measured findings of a render that passed the host gates start a repair (`AGENT_GATE_REPAIR`): - the advisory gate lines (G3, G5, G7, G8, G9, G10) and the ADAPTATION validator's soft findings. + the advisory gate lines (G3, G5, G7, G8, G9, G10, G11, G12) and the ADAPTATION validator's soft findings. `review`: they go to the round's self-review as findings, then to the review as gate notes, and a round whose render still has them after its review goes to the repair with them and the review's diff --git a/agents/evals/cases.py b/agents/evals/cases.py index e464aa4d935..f71fad70e4f 100644 --- a/agents/evals/cases.py +++ b/agents/evals/cases.py @@ -16,7 +16,7 @@ `select_cases` takes the harness's `--cases` value: `smoke` (the cases tagged `smoke`: one generated case per spec plus the hand-written `scatter-basic-matplotlib` and `bar-grouped-seaborn`), `full` (every case except the stress cases: the 120 generated -and the three hand-written ones, 123 since 2026-10-12), `stress` (the +and the seven hand-written ones, 127 since 2026-10-15), `stress` (the cases tagged `stress`, from `make_stress.py`), or comma-separated shell globs over case ids (`*-seaborn-*`, `pie-basic-*`, `stress-compute-*`). The catalogue snapshot comes from `snapshot.json` or, without it, from `plots//` in this diff --git a/agents/evals/fixtures/README.md b/agents/evals/fixtures/README.md index a79dab2b433..419f8c07293 100644 --- a/agents/evals/fixtures/README.md +++ b/agents/evals/fixtures/README.md @@ -12,15 +12,15 @@ Each directory under `cases/` is one case of the regression harness (`agents/eva A stress case's `case.json` also carries `bound_expected` (the gate, limit or error that must bound it), `needs_model`, `static` and `render` (what the static stage must measure), and for an injection case `marker`; `agents/evals/make_stress.py` documents each field. -There are 146 cases: +There are 150 cases: - **120 generated cases** `--`: 10 specs times matplotlib and seaborn times 6 perturbations (`renamed`, `x10`, `n12`, `n5000`, `date`, `decimal-comma`), written by `agents/evals/make_fixtures.py`. Never edit them by hand: change the generator and rerun `uv run --extra agents python -m agents.evals.make_fixtures`. Its docstring lists the specs, why each was chosen, and what each perturbation does. -- **3 hand-written cases**: `scatter-basic-matplotlib` and `bar-grouped-seaborn`, which the runtime tests also use, and `scatter-basic-matplotlib-german-message` (since 2026-10-12): German column names and a German follow-up message, `Mach die Punkte etwas größer.`, with no word about the language. The first plot comes from `create_plot`, which the scope judge never reads, so it is English; the follow-up is the session's first typed message and sets the session language to `de` (`detected`), so the second version must carry German plot text while the column names keep their spelling. The case names no `language` on purpose: it tests detection, not a choice. Each run's record holds the last `done` event's `language`. -- **23 stress cases** `stress-`, written by `agents/evals/make_stress.py` (rerun `uv run --extra agents python -m agents.evals.make_stress`): data at and one past the parser's caps, many categories, degenerate data, prompt injection, and requests that make the code compute mass data. They carry the tag `stress`; `--cases stress` selects them and `--cases full` leaves them out, so a full run keeps its 123 cases. +- **7 hand-written cases**: `scatter-basic-matplotlib` and `bar-grouped-seaborn`, which the runtime tests also use, and `scatter-basic-matplotlib-german-message` (since 2026-10-12): German column names and a German follow-up message, `Mach die Punkte etwas größer.`, with no word about the language. The first plot comes from `create_plot`, which the scope judge never reads, so it is English; the follow-up is the session's first typed message and sets the session language to `de` (`detected`), so the second version must carry German plot text while the column names keep their spelling. The case names no `language` on purpose: it tests detection, not a choice. Each run's record holds the last `done` event's `language`. Four more (since 2026-10-15, tags `density` or `number-format`) generalise the live tests of 2026-10-14/15 and choose their `language`: `bar-grouped-matplotlib-many-categories-de` (26 invented regions times two series, where the catalogue's per-bar labels, dots and shadows crowded 52 bars; gate G11), `bar-basic-seaborn-decimals-de` and its English control `bar-basic-seaborn-decimals-en` (bar values with two decimals, which seaborn's `bar_label` wrote as `5.30` in a German plot; gate G12), and `line-multi-matplotlib-millions-de` (two population series in the millions with the follow-up `Zeig die y-Achse in Millionen, zum Beispiel 6,5 Mio.`, after an axis that read `6.0 M`; gate G12). +- **23 stress cases** `stress-`, written by `agents/evals/make_stress.py` (rerun `uv run --extra agents python -m agents.evals.make_stress`): data at and one past the parser's caps, many categories, degenerate data, prompt injection, and requests that make the code compute mass data. They carry the tag `stress`; `--cases stress` selects them and `--cases full` leaves them out, so a full run keeps its 127 cases. The tag `smoke` marks the 12 cases of `--cases smoke`: one generated case per spec, cycling through the perturbations and both libraries, plus `scatter-basic-matplotlib` and `bar-grouped-seaborn`. -43 cases open their session with `locale` `de`: the 40 generated `renamed` and `decimal-comma` cases, `bar-grouped-seaborn` (tag `german-locale`) and the stress cases `cap-columns` and `mixed-decimals`. Since 2026-10-12 the locale sets no language, and their only turn is `create_plot`, so they reply and title their plots in English like every other case; their purpose is the data perturbation. `scatter-basic-matplotlib-german-message` is the case that exercises German replies and German plot text. +46 cases open their session with `locale` `de`: the 40 generated `renamed` and `decimal-comma` cases, `bar-grouped-seaborn` (tag `german-locale`), the three German block H cases (which also choose `language` `de`) and the stress cases `cap-columns` and `mixed-decimals`. Since 2026-10-12 the locale sets no language, and their only turn is `create_plot`, so they reply and title their plots in English like every other case; their purpose is the data perturbation. `scatter-basic-matplotlib-german-message` is the case that exercises German replies and German plot text. ## Stress cases: what bounds them diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json new file mode 100644 index 00000000000..399ac4b1bd1 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json @@ -0,0 +1,13 @@ +{ + "spec_id": "bar-basic", + "library": "seaborn", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "category", "column": "Stadt"}, + {"role": "value", "column": "CO2 pro Kopf (t)"} + ], + "expected": "accepted", + "tags": ["categorical", "decimals", "de", "language", "number-format"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-15 (seaborn bar labels 5.30 in a German session)" +} diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv new file mode 100644 index 00000000000..cbbec094630 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv @@ -0,0 +1,7 @@ +Stadt,CO2 pro Kopf (t) +Altbrunn,5.30 +Seefeld,4.85 +Hochmatt,6.12 +Talwil,3.95 +Rosshalden,7.41 +Birkenau,4.27 diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json new file mode 100644 index 00000000000..43a023a83b7 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json @@ -0,0 +1,13 @@ +{ + "spec_id": "bar-basic", + "library": "seaborn", + "locale": "en", + "language": "en", + "bindings": [ + {"role": "category", "column": "City"}, + {"role": "value", "column": "CO2 per capita (t)"} + ], + "expected": "accepted", + "tags": ["categorical", "decimals", "en", "language", "number-format"], + "source": "hand-written synthetic data, the English control of bar-basic-seaborn-decimals-de" +} diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv new file mode 100644 index 00000000000..b78854e33a5 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv @@ -0,0 +1,7 @@ +City,CO2 per capita (t) +Altbrunn,5.30 +Seefeld,4.85 +Hochmatt,6.12 +Talwil,3.95 +Rosshalden,7.41 +Birkenau,4.27 diff --git a/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json new file mode 100644 index 00000000000..5e5bc79dac8 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json @@ -0,0 +1,14 @@ +{ + "spec_id": "bar-grouped", + "library": "matplotlib", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "category", "column": "Region"}, + {"role": "group", "column": "Ereignis"}, + {"role": "value", "column": "Anzahl"} + ], + "expected": "accepted", + "tags": ["categorical", "many-categories", "de", "language", "density"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-14 (B: 26 regions x 2 series)" +} diff --git a/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv new file mode 100644 index 00000000000..b721b778920 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv @@ -0,0 +1,53 @@ +Region,Ereignis,Anzahl +Aarwald,Geburten,12895 +Aarwald,Todesfälle,12435 +Bergfeld,Geburten,4925 +Bergfeld,Todesfälle,4499 +Brunnau,Geburten,3722 +Brunnau,Todesfälle,2612 +Dornegg,Geburten,6956 +Dornegg,Todesfälle,7264 +Eichtal,Geburten,13097 +Eichtal,Todesfälle,14382 +Felsach,Geburten,935 +Felsach,Todesfälle,990 +Grünmatt,Geburten,1359 +Grünmatt,Todesfälle,1512 +Hasenbühl,Geburten,14066 +Hasenbühl,Todesfälle,13537 +Ilfingen,Geburten,1591 +Ilfingen,Todesfälle,1436 +Juraweid,Geburten,14541 +Juraweid,Todesfälle,14518 +Kaltbrunn,Geburten,8252 +Kaltbrunn,Todesfälle,7096 +Lindau-Ost,Geburten,4217 +Lindau-Ost,Todesfälle,3909 +Mooshalden,Geburten,12995 +Mooshalden,Todesfälle,9336 +Nesselried,Geburten,12904 +Nesselried,Todesfälle,11989 +Oberried,Geburten,14029 +Oberried,Todesfälle,15283 +Pfannstiel,Geburten,1151 +Pfannstiel,Todesfälle,1247 +Quellmatt,Geburten,15082 +Quellmatt,Todesfälle,12359 +Rebhalde,Geburten,7120 +Rebhalde,Todesfälle,6801 +Seewinkel,Geburten,4894 +Seewinkel,Todesfälle,4904 +Tannegg,Geburten,14311 +Tannegg,Todesfälle,15669 +Unterau,Geburten,6607 +Unterau,Todesfälle,6048 +Vorderwald,Geburten,15897 +Vorderwald,Todesfälle,13608 +Weissbach,Geburten,7805 +Weissbach,Todesfälle,6402 +Zelgli,Geburten,11394 +Zelgli,Todesfälle,10652 +Ahornried,Geburten,5982 +Ahornried,Todesfälle,6609 +Buchsee,Geburten,4698 +Buchsee,Todesfälle,4461 diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json new file mode 100644 index 00000000000..0dc265c7a01 --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json @@ -0,0 +1,14 @@ +{ + "spec_id": "line-multi", + "library": "matplotlib", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "x", "column": "Jahr"}, + {"role": "y1", "column": "Bevölkerung"}, + {"role": "series", "column": "Staatsangehörigkeit"} + ], + "expected": "accepted", + "tags": ["numeric", "long-format", "de", "language", "number-format"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-14 (A: an axis in millions showed 6.0 M in a German session)" +} diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt new file mode 100644 index 00000000000..ceaf272addc --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt @@ -0,0 +1 @@ +Zeig die y-Achse in Millionen, zum Beispiel 6,5 Mio. diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv new file mode 100644 index 00000000000..a77f2dacaaa --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv @@ -0,0 +1,57 @@ +Jahr,Staatsangehörigkeit,Bevölkerung +1971,Schweiz,5090735 +1971,Ausland,1051054 +1973,Schweiz,5149288 +1973,Ausland,1091874 +1975,Schweiz,5212357 +1975,Ausland,1149602 +1977,Schweiz,5284253 +1977,Ausland,1178116 +1979,Schweiz,5337176 +1979,Ausland,1214283 +1981,Schweiz,5403798 +1981,Ausland,1279596 +1983,Schweiz,5480850 +1983,Ausland,1320628 +1985,Schweiz,5522363 +1985,Ausland,1372426 +1987,Schweiz,5586538 +1987,Ausland,1403167 +1989,Schweiz,5645642 +1989,Ausland,1443281 +1991,Schweiz,5731407 +1991,Ausland,1501651 +1993,Schweiz,5790434 +1993,Ausland,1519336 +1995,Schweiz,5845489 +1995,Ausland,1592226 +1997,Schweiz,5914754 +1997,Ausland,1610270 +1999,Schweiz,5974773 +1999,Ausland,1674374 +2001,Schweiz,6015759 +2001,Ausland,1710136 +2003,Schweiz,6077411 +2003,Ausland,1744682 +2005,Schweiz,6168536 +2005,Ausland,1784431 +2007,Schweiz,6216939 +2007,Ausland,1855071 +2009,Schweiz,6272127 +2009,Ausland,1887823 +2011,Schweiz,6341441 +2011,Ausland,1932134 +2013,Schweiz,6397995 +2013,Ausland,1981941 +2015,Schweiz,6473849 +2015,Ausland,2013750 +2017,Schweiz,6531505 +2017,Ausland,2047569 +2019,Schweiz,6576507 +2019,Ausland,2097893 +2021,Schweiz,6655738 +2021,Ausland,2164121 +2023,Schweiz,6719001 +2023,Ausland,2203491 +2025,Schweiz,6762891 +2025,Ausland,2231271 diff --git a/agents/evals/report.py b/agents/evals/report.py index 9078dd15bf1..2d14706df04 100644 --- a/agents/evals/report.py +++ b/agents/evals/report.py @@ -22,7 +22,7 @@ **Pass.** A run passes when it shipped a render (`ok` or `needs_attention`) that passed every deterministic gate: the host gates R1 to R3 without a padded canvas, and no ADAPTATION validator finding on the shipped code. The advisory probe gates (G3, -G5, G7, G8, G9, G10) and the reviewer's verdict do not decide a pass; the owner's blind +G5, G7, G8, G9, G10, G11, G12) and the reviewer's verdict do not decide a pass; the owner's blind judgement in the gallery does that for the sample, which holds passed renders only. Every rate counts all runs: a run that crashed in the harness did not pass. **Accept-match** compares the harness outcome (`accepted` when the status is `ok`, the diff --git a/changelog.d/agent-density-locale-gates.md b/changelog.d/agent-density-locale-gates.md new file mode 100644 index 00000000000..18686aeaaa5 --- /dev/null +++ b/changelog.d/agent-density-locale-gates.md @@ -0,0 +1,14 @@ +### Added + +- **Agent chat: gates for crowded value labels and the plot language's + number format.** G11 counts the value labels, dots and shadows the + catalogue draws once per bar or point and writes a measured VQ-03 line + when more than 25 crowd a plot, or bar labels are wider than the space + per bar (the live case: 52 labelled, dotted, shadowed bars of 26 + regions). G12 writes a VQ-06 line when a numeric tick label or annotation + uses the other decimal mark than the plot language (`5.30` and `6.0 M` + in a German plot). A server-side table of decimal and thousands marks and + large-number words per language (`de`: `5,30`, `6,5 Mio.`) goes into the + adapter's instruction, and the house-style exemption of the reviewer and + scorer no longer covers elements whose count grows with the data + (#12184). diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 956614a5c1a..3bdb656849a 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -319,10 +319,12 @@ The `remote` backend mints the ID token for `AGENT_RENDER_URL`, the renderer's s A run renders one theme, the one `PipelineArgs.theme` names, and the host gates judge exactly the themes of the job (an output for another theme is ignored, a missing one fails R1). The theme toggle renders the other theme of a finished version later, as a job of its own with the same code and `data.csv`, and adds its PNG to the version's render. Renders are strictly serial per instance, because spikes S and S2 showed one 4 GiB instance serves one sandbox at a time safely. No backend keeps a semaphore of its own: `Services.backend` wraps whichever backend `make_backend` built in `SerialRenderer` (`render/serial.py`), which renders one theme per slot under `AGENT_RENDER_CONCURRENCY` slots, 1 by default, so every backend is serial by construction, `adk web` runs included; the `remote` backend therefore sends one theme per request. The renderer holds one render slot of its own on top, because the deployed agents service and a developer's `adk web` call it independently. A freed slot goes to a waiting pipeline render before a waiting theme toggle, so a run waits for at most the render in progress; a render's timeout starts when it runs. The theme toggle records a theme that failed the host gates and renders it at most twice; a renderer that could not run is not recorded. `harness.py` (copied into the renderer image at `/opt/anyplot/harness.py`) prints its `HARNESS` start line, sets the resource limits soft and hard alike (CPU seconds and file size always; the address space and the process count when the renderer passes them, because outside a sandbox `RLIMIT_NPROC` counts every process of the user), copies the baked font cache into `MPLCONFIGDIR`, appends the installed fallback families (`FALLBACK_FAMILIES` in `harness.py`: Noto Sans, Noto Sans CJK JP, and the Noto Sans faces for Arabic, Hebrew, the nine Indic scripts, Thai, Khmer, Myanmar, Ethiopic and two symbol blocks) to every font lookup after the font the code chose, so DejaVu Sans stays the catalogue's first font and text in any of those scripts still draws, whatever `font.family` the code or seaborn's `set_theme` set; records every code point matplotlib reports as having no glyph in any font of its text, plain text and `$…$` math alike, as `missing_glyphs` in the probe; registers a `Figure.savefig` probe that writes `probe-.json`, compiles the plot file and runs it as `__main__` under the plot guard, and prints its peak memory and CPU time in a closing `HARNESS` line. -Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` with no symlinks, at most 10 MB, magic bytes, Pillow under `MAX_IMAGE_PIXELS`, non-blank with less than 98 % background, re-encode) are blocking; a render that fails either is discarded and never becomes `best`. R3 (canvas within 16 px through `core/canvas.py`, lifted from `.github/workflows/impl-review.yml`) is repair-triggering with a fallback: an R3 miss after normalisation is a defect line for the repair; if it persists after the last allowed attempt, the host pads the PNG of the rendered theme to the target canvas (never crops), the padded artifact counts as having passed the host gates, and the result ships as `needs_attention` with the residual line `canvas padded after render ()` and the VQ-05 line for that theme, never silently. A job with both themes pads each theme that missed and keeps the other theme's PNG as it rendered. The theme toggle answers a padded theme as `needs_attention` with the reason `canvas_padded`. The probe gates (G3 clipping, G5 annotation out of view, G7 overlapping text: tick labels among themselves, and since 2026-10-12 every other drawn text (axis label, title, legend text, annotation, or a text no role names) against any drawn text by more than 3 px, named by role and never by content, with a polar hint when an axis label of polar axes is involved, after the owner's polar case where the radial label `y` sat on the `270°` tick label and the reviewer called it missing; G8 row fidelity, G9 more than 500,000 plotted points, G10 characters without a glyph) are advisory because in-sandbox data can be tampered with: they can only trigger a repair or add notes, never fail a render. Since 2026-10-12 the probe also describes each polar axes (`polar`: the radial labels' angle, the visible theta span and a histogram of the plotted points by 5° of angle and by tenth of the radius range, all in the axes' own theta coordinates), and `agents/anyplot/render/polar.py` turns it into a measured target, not a gate: when the radial tick labels' ray holds plotted points and a ray every 22.5° inside the span holds at most half as many, `GateReport.repair_hints` gets one line such as "polar radial tick labels: at 67.5° they sit on 10 plotted points → ax.set_rlabel_position(270), the freest direction, 0 points there. Likely cause: ...". It never starts a repair; a repair attempt that runs for other findings carries the latest render's line first among its `hints`, and the adapter prompt says to apply it with exactly that value. The owner's case of that day (seaborn `polar-scatter`, `y = sin(x)`) had the repair guess 67.5°, onto the data; evenly spread angles give no line, but a cardioid does, because a point counts only when its distance from the ray as a share of the radius is small and it is outside the innermost tenth. The first attempt gets no such hint: nothing is rendered yet, and the angle of the user's column as the plot will draw it (degrees or radians, offset, direction) is only known from the render. A renderer limit that stopped a run (`memory`, `disk_budget`, `file_budget`, `output_rejected`) is recorded as `R1-` among the failed gates. The defect-line helpers (`DEFECT_RE`, `weakness_class`, `defect_ids`, and the `format_defect` builder) live in `core/defects.py`, the canonical stdlib-only implementation, and the PNG auto-reject checks (AR-04 blank and AR-07 format; R2 calls `check_blank` with its 98 % threshold) live next to the canvas gate in `core/canvas.py`, so the agents image needs no `automation/` or `scripts/` files. `automation/scripts/regen_gate.py` keeps a parity-tested copy of the grammar, because the workflows run it as a single-file copy with the runner's system Python, where `core` is not importable. The pipeline switches to `core.defects` in the later change that also switches the canvas gate and updates the copy sites: the `cp` step in `impl-review.yml`, the `git show` in `impl-generate.yml`, `review_retest.py materialize`, and the overlay in `review-retest.yml`. +Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` with no symlinks, at most 10 MB, magic bytes, Pillow under `MAX_IMAGE_PIXELS`, non-blank with less than 98 % background, re-encode) are blocking; a render that fails either is discarded and never becomes `best`. R3 (canvas within 16 px through `core/canvas.py`, lifted from `.github/workflows/impl-review.yml`) is repair-triggering with a fallback: an R3 miss after normalisation is a defect line for the repair; if it persists after the last allowed attempt, the host pads the PNG of the rendered theme to the target canvas (never crops), the padded artifact counts as having passed the host gates, and the result ships as `needs_attention` with the residual line `canvas padded after render ()` and the VQ-05 line for that theme, never silently. A job with both themes pads each theme that missed and keeps the other theme's PNG as it rendered. The theme toggle answers a padded theme as `needs_attention` with the reason `canvas_padded`. The probe gates (G3 clipping, G5 annotation out of view, G7 overlapping text: tick labels among themselves, and since 2026-10-12 every other drawn text (axis label, title, legend text, annotation, or a text no role names) against any drawn text by more than 3 px, named by role and never by content, with a polar hint when an axis label of polar axes is involved, after the owner's polar case where the radial label `y` sat on the `270°` tick label and the reviewer called it missing; G8 row fidelity, G9 more than 500,000 plotted points, G10 characters without a glyph, and since 2026-10-15 G11 per-mark value labels and decorations past what the marks leave room for, and G12 a decimal mark that is not the plot language's) are advisory because in-sandbox data can be tampered with: they can only trigger a repair or add notes, never fail a render. Since 2026-10-12 the probe also describes each polar axes (`polar`: the radial labels' angle, the visible theta span and a histogram of the plotted points by 5° of angle and by tenth of the radius range, all in the axes' own theta coordinates), and `agents/anyplot/render/polar.py` turns it into a measured target, not a gate: when the radial tick labels' ray holds plotted points and a ray every 22.5° inside the span holds at most half as many, `GateReport.repair_hints` gets one line such as "polar radial tick labels: at 67.5° they sit on 10 plotted points → ax.set_rlabel_position(270), the freest direction, 0 points there. Likely cause: ...". It never starts a repair; a repair attempt that runs for other findings carries the latest render's line first among its `hints`, and the adapter prompt says to apply it with exactly that value. The owner's case of that day (seaborn `polar-scatter`, `y = sin(x)`) had the repair guess 67.5°, onto the data; evenly spread angles give no line, but a cardioid does, because a point counts only when its distance from the ray as a share of the radius is small and it is outside the innermost tenth. The first attempt gets no such hint: nothing is rendered yet, and the angle of the user's column as the plot will draw it (degrees or radians, offset, direction) is only known from the render. A renderer limit that stopped a run (`memory`, `disk_budget`, `file_budget`, `output_rejected`) is recorded as `R1-` among the failed gates. The defect-line helpers (`DEFECT_RE`, `weakness_class`, `defect_ids`, and the `format_defect` builder) live in `core/defects.py`, the canonical stdlib-only implementation, and the PNG auto-reject checks (AR-04 blank and AR-07 format; R2 calls `check_blank` with its 98 % threshold) live next to the canvas gate in `core/canvas.py`, so the agents image needs no `automation/` or `scripts/` files. `automation/scripts/regen_gate.py` keeps a parity-tested copy of the grammar, because the workflows run it as a single-file copy with the runner's system Python, where `core` is not importable. The pipeline switches to `core.defects` in the later change that also switches the canvas gate and updates the copy sites: the `cp` step in `impl-review.yml`, the `git show` in `impl-generate.yml`, `review_retest.py materialize`, and the overlay in `review-retest.yml`. **Scripts and fonts.** Plot text can be in the scripts most users' data and languages bring: the renderer image installs Noto Sans, Noto Sans CJK and the Noto Sans faces of the scripts DejaVu Sans lacks (37 files, 46.8 MB, Regular and Bold where the family has both), and the harness appends them to every font lookup, so a Chinese title or a Devanagari tick label draws with the right glyphs while the code's own first font stays in charge of Latin text. Right-to-left scripts work: the pinned matplotlib lays text out through libraqm, so Arabic and Hebrew render shaped and in reading order, within one label aligned by the code's `ha`. Three limits remain, and G10 reports the first and the last: a `$…$` math string uses matplotlib's math fonts without fallback and draws `¤` for every character they lack, so non-Latin words belong outside math text; Han characters take the Japanese regional forms of the CJK face until a caller passes the plot's language (`text.language`, a follow-up with the language chip); and a character outside the installed families draws as a Last Resort placeholder box. Most pictographs are not in that last group, because Noto Sans Symbols2 or DejaVu Sans draws them as monochrome outlines (📈, 📊, ⚠, 😀), but 🔥, ✅, the flag letters, Tibetan and Mongolian have no glyph in any installed font. G10 counts those characters deterministically from matplotlib's own reports, names up to three of them with their code points (`U+1F525 🔥 fire`), tells the repair that a user's column name stays as it is, and keeps the line in the result's residual lines even when the reviewer passed the render, because a measured missing glyph is not a judgement. +**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits at a bar's end, inside a bar's segment (`bar_label(label_type="center")`) or on a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels, and neither is a text matplotlib clips away in a zoomed view), which of them are numbers, the dots on bar ends (scatter marks or `plot(..., "o")` markers) and the shadowed bars, and measures the median bar label width against the space between neighbouring labelled bars (unlabelled bars between them, such as an offset shadow series, do not shrink it). Bar ends come from the bars' data coordinates, so an inverted axis works. G11 writes a VQ-03 line (owner, 2026-10-15: the thresholds stay) when more than `MAX_VALUE_LABELS` (25) bar positions carry a label (a stacked bar's segment labels count once), when more than 25 numeric labels sit on points (named points, the point of scatter-annotated, never count), from six labels on when a bar label is wider than 90 % of the space per bar, or when more than `MAX_DECORATIONS` (25) dots or shadows are drawn. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string whose marks are swapped through a placeholder (`.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`), exact for the space-grouped languages too. A region such as `de-CH` with its apostrophe (7'184'003) is a follow-up of its own setting (owner, 2026-10-15). G12 reads the probe's `labels` field (the drawn texts by kind, from block F) and its `tick_axes` field (`harness.tick_axes`: whether each axis's tick labels are numbers, dates, categories or labels the code set itself), and writes a VQ-06 line when an annotation, or a tick label of a numeric axis, uses the other decimal mark, mathtext read as drawn: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`, `5.30 €`), a point after a whole part of 0 (`0.125`), a point after a comma, or commas grouping thousands; dates (a zero-padded `14.10` in an annotation reads as a day and month), years, version strings, titles, legend entries and tick labels the code set itself never count, and no label text reaches the line. + Later languages reuse the CI commands: `Rscript plot.R`; `julia --project=/opt/julia plot.jl` with a stacked `JULIA_DEPOT_PATH` and a pinned `JULIA_CPU_TARGET`; `node /opt/anyplot/js-render/render.mjs plot.js` with `window.ANYPLOT_DATA` injected; bokeh with `BOKEH_RESOURCES=inline`, `SE_OFFLINE=true` and a shipped chromedriver; plotly with local or disabled MathJax. ### Footer strip @@ -391,13 +393,13 @@ Built since 2026-10-12, the capture half. Every result card carries the control Testing a newer model version, the other provider's arm, a different judge model, or a prompt or validator change is one command locally or one workflow dispatch in CI, and the answer is a diff against the pinned baseline. -Built on 2026-10-10: the harness, the two fixture generators and the 146 fixture cases (the 123 of the full matrix and the 23 stress cases with their model-free `--static` stage), the reports, the diff, the review gallery, the blind two-run gallery with its scoring, the model-free catalogue eligibility sweep `agents/evals/eligibility.py` (`agents/README.md`, "Run the regression harness"), and the judge-only scope eval `agents/evals/scope.py` with its refusal recall and false-refusal rate (`agents/README.md`, "Run the scope eval"; its first run passed both gates). Not built yet: `--prompts `, the Gemini baseline, and the CI workflow. +Built on 2026-10-10: the harness, the two fixture generators and the 146 fixture cases, 150 since 2026-10-15 (the 127 of the full matrix and the 23 stress cases with their model-free `--static` stage), the reports, the diff, the review gallery, the blind two-run gallery with its scoring, the model-free catalogue eligibility sweep `agents/evals/eligibility.py` (`agents/README.md`, "Run the regression harness"), and the judge-only scope eval `agents/evals/scope.py` with its refusal recall and false-refusal rate (`agents/README.md`, "Run the scope eval"; its first run passed both gates). Not built yet: `--prompts `, the Gemini baseline, and the CI workflow. -- **Fixture cases**, each a directory with `case.json` (spec, library, bindings, expected `accepted`, `rejected` or `unknown`, tags, the perturbation and the source of the data, and optionally a session `language` that the harness chooses through the language route before the first turn; a promoted case carries its session's when that was detected or chosen), `data.csv`, and an optional `change_request.txt`. Two sources: the committed synthetic fixtures in `agents/evals/fixtures/cases//`, which are the spike-X matrix (10 specs times matplotlib and seaborn times 6 perturbation datasets, written by the seeded generator `agents/evals/make_fixtures.py`) plus three hand-written cases (one of them, since 2026-10-12, a German follow-up message that sets the session language) and the 23 stress cases of `agents/evals/make_stress.py` (tag `stress`, left out of `full`; see [Bounds](#bounds)), and the promoted feedback cases synced from the private bucket into the gitignored `agents/evals/.cases/` directory (never committed, because they hold users' data). The tag `smoke` marks the 12 cases of the smoke set. The scope set is the judge-only case file `agents/evals/scope.evalset.json` (see Evals below); the injection and code-escape sets stay as flow evals. +- **Fixture cases**, each a directory with `case.json` (spec, library, bindings, expected `accepted`, `rejected` or `unknown`, tags, the perturbation and the source of the data, and optionally a session `language` that the harness chooses through the language route before the first turn; a promoted case carries its session's when that was detected or chosen), `data.csv`, and an optional `change_request.txt`. Two sources: the committed synthetic fixtures in `agents/evals/fixtures/cases//`, which are the spike-X matrix (10 specs times matplotlib and seaborn times 6 perturbation datasets, written by the seeded generator `agents/evals/make_fixtures.py`) plus seven hand-written cases (one of them, since 2026-10-12, a German follow-up message that sets the session language; four, since 2026-10-15, choose a language and cover label density and number formats) and the 23 stress cases of `agents/evals/make_stress.py` (tag `stress`, left out of `full`; see [Bounds](#bounds)), and the promoted feedback cases synced from the private bucket into the gitignored `agents/evals/.cases/` directory (never committed, because they hold users' data). The tag `smoke` marks the 12 cases of the smoke set. The scope set is the judge-only case file `agents/evals/scope.evalset.json` (see Evals below); the injection and code-escape sets stay as flow evals. - **Harness** (`agents/evals/matrix.py`, the spike-X runner made permanent): runs every case through the real `/v1` flow in-process (httpx `ASGITransport` against `agents.main:app`, the same plugins and run queue, the models from the settings, and the render backend `--renderer` names: `remote` against a deployed renderer, `local` in Docker, or `fake` for a dry run of everything but the render) with candidate settings that are passed only to the harness: `--provider` (alone it selects that provider's default models; the settings refuse a model from the other provider), `--model`, `--judge-model`, `--location`, `--cases smoke|full|stress|`, `--static` (only the model-free static stage of `agents/evals/stress.py`: parse, eligibility, bindings, loader, prompt sizes and the stand-in renders), `--repeats N`, `--budget-usd` (stop before a case that could take the estimated list-price cost past it), and `--gcloud-token` (mint the developer's renderer token with the gcloud CLI and renew it before its one-hour expiry). Before the first case it renders each library's catalogue file once through the configured backend, without a model call, and stops with exit code 2 when that fails; during the run a pipeline that ended in `RendererUnavailable` (the `error` class on its `pipeline_result` attribution line), or three errors in a row, stop it with exit code 4, so an outage never becomes a matrix of failed runs or a baseline. Per case it records the status and reason, attempts, gate failures by gate, validator rejections, edit-apply failures, the reviewer verdict, LLM calls, `model_version`, tokens (prompt, candidates, thoughts, cached, cache writes, the judge's input and output), cost at list price, the time to the first event, the end-to-end time and the render time, and, since report schema 2, the stage that stopped each attempt and the run (for example `adapter_truncated`, `edit_apply` or `reviewer_defects`, never a bare `validation`), the shipped and the last reviewed attempt, each attempt's adapter outcome, finish reason and plan shape, the edit-failure kinds, every model call's finish reason and tokens, the answers that failed their schema, by agent kind and outcome (`answer_outcomes`: `repaired` or `refused`) and by broken rule (`answer_rules`), the edits each tolerant matching tier applied (`edit_tolerant`), and the module labels of banned imports (`banned_imports`). It reads them from the stream and from the content-free attribution lines that the pipeline, the Budget plugin and the judges write per request id. It writes `/-.json`, a Markdown summary, and a review gallery of 30 renders sampled at random from the runs that passed, with an accept and reject pair each and a JSON export, and prints a diff table against `agents/evals/baselines/.json` (or `--baseline`) over the cases both reports hold: pass rate, accept-match rate (cases whose outcome matches the case's expected outcome), cost per successful plot, p50 and p95 latency, and every case that flipped. A run passes when it ships a render that passed every deterministic gate (no padded canvas, no ADAPTATION finding); every rate counts harness errors as runs that did not pass. The owner's judgement is the acceptance measure: to compare two arms, `python -m agents.evals.report blind` merges both reports into one gallery of the same cases with the arm hidden in a separate key file, and `report score` gives the acceptance per arm. The exit code is 1 when the pass rate on the shared cases drops below the baseline minus a tolerance (5 percentage points by default) or a run crashed in the harness, 2 for a setup error, 3 when the budget stopped the run, and 4 for an outage. - **Baselines.** `agents/evals/baselines/claude-haiku-5-5.json` is committed with the default model: the thinking run of 2026-10-11 on main at `e2e25ef61` (122 cases with one run each, root thinking 1,024 and adapter thinking 2,048, through the deployed renderer; 122 passed, 62 `ok`), which replaced the thinking-off rerun of spike X of 2026-10-10 (244 runs). `gemini-3.8-flash.json` follows with the Gemini arm, so the price and quality comparison is a diff between two reports. `--save-baseline` refuses a run that stopped early, holds promoted cases, or has a run that ended in an error. A model bump is a PR that changes the `AGENT_MODEL` default and adds the new baseline file; a unit test asserts that a baseline exists for the pinned model once any baseline is committed. The same mechanism covers judge-model bumps and ADK upgrades, because the ADK version is part of the report stamp. - **CI.** `.github/workflows/agents-eval.yml` has `workflow_dispatch` inputs `provider`, `model`, `judge_model`, `location`, `cases` (smoke or full) and `repeats`, and a nightly `smoke` run on the pinned model. It authenticates through Workload Identity Federation, sets `GOOGLE_CLOUD_LOCATION=eu`, uploads the report as an artifact and writes the diff table into the job summary. It mirrors the candidate-versus-baseline pattern of `review-retest.yml` (see [Review retest](../workflows/review-retest.md)). A lifecycle probe in the nightly run calls `models.get` on the pinned model and fails loudly on a 404 or a retirement notice, so a forced bump of the pinned model, or of the Gemini arm's short-term 3.8 Flash tier, is noticed before users are. -- **Cost.** A full run is 123 cases times at most 2 attempts. Measured on 2026-10-10 at list price for the full matrix: $0.52 on the Claude Haiku 5.5 arm ($0.0051 per passed plot, median $0.0041, 4.77 model calls per run) and $15.89 on the Gemini 3.8 Flash arm ($0.169 per passed plot, median $0.124, 4.80 model calls per run, $3.99 of it on failed runs); `smoke` (12 cases) is about a tenth of a full run. Both fit inside the spend cap. +- **Cost.** A full run is 127 cases (123 until 2026-10-15) times at most 2 attempts. Measured on 2026-10-10 at list price for the full matrix: $0.52 on the Claude Haiku 5.5 arm ($0.0051 per passed plot, median $0.0041, 4.77 model calls per run) and $15.89 on the Gemini 3.8 Flash arm ($0.169 per passed plot, median $0.124, 4.80 model calls per run, $3.99 of it on failed runs); `smoke` (12 cases) is about a tenth of a full run. Both fit inside the spend cap. ## Serving and infrastructure diff --git a/tests/unit/agents/evals/test_cases.py b/tests/unit/agents/evals/test_cases.py index 1d7595cbb39..924ce49e868 100644 --- a/tests/unit/agents/evals/test_cases.py +++ b/tests/unit/agents/evals/test_cases.py @@ -30,19 +30,27 @@ def test_every_committed_case_loads() -> None: cases = load_cases(promoted_dir=Path("/nonexistent")) stress = [case for case in cases if "stress" in case.tags] - assert len(cases) - len(stress) == 123 # 120 generated plus the three hand-written cases + assert len(cases) - len(stress) == 127 # 120 generated plus the seven hand-written cases assert len(stress) == 23 and all(case.case_id.startswith("stress-") for case in stress) assert [case.case_id for case in cases] == sorted(case.case_id for case in cases) assert {case.origin for case in cases} == {"fixtures"} hand_written = {case.case_id: case for case in cases if case.perturbation is None and "stress" not in case.tags} + chosen = { + "bar-grouped-matplotlib-many-categories-de": "de", + "bar-basic-seaborn-decimals-de": "de", + "bar-basic-seaborn-decimals-en": "en", + "line-multi-matplotlib-millions-de": "de", + } assert set(hand_written) == { "scatter-basic-matplotlib", "bar-grouped-seaborn", "scatter-basic-matplotlib-german-message", + *chosen, } assert hand_written["bar-grouped-seaborn"].locale == "de" - # No committed case chooses a language: the German case tests detection by its first typed message. - assert all(case.language is None for case in cases) + # Only the block H cases (density and number formats) choose a language; the German message case + # tests detection by its first typed message. + assert {case.case_id: case.language for case in cases if case.language is not None} == chosen def test_the_german_message_case_sets_the_language_by_its_first_typed_message() -> None: @@ -66,7 +74,7 @@ def test_selection_by_smoke_full_and_globs() -> None: smoke = select_cases(cases, "smoke") assert len(smoke) == 12 and all("smoke" in case.tags for case in smoke) full = select_cases(cases, "full") - assert len(full) == 123 and full == [case for case in cases if "stress" not in case.tags] + assert len(full) == 127 and full == [case for case in cases if "stress" not in case.tags] stress = select_cases(cases, "stress") assert len(stress) == 23 and all("stress" in case.tags for case in stress) assert len(select_cases(cases, "stress-compute-*")) == 3 diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py new file mode 100644 index 00000000000..0ed94c7be05 --- /dev/null +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -0,0 +1,594 @@ +"""G11 (per-mark labels and decorations past what the marks leave room for) and G12 (the plot language's decimal mark). + +The fixtures generalise two findings of the live tests of 2026-10-14/15 (block H): + +- Q3: bar-grouped on 26 cantons times two series kept the catalogue's per-bar value labels, + dots on the bar tops and drop shadows, styled for its 12 example bars, on all 52 bars; +- Q4: German sessions showed "6.0 M" on an axis (the user asked for "6,5 Mio.") and seaborn + bar labels "5.30" instead of "5,30". + +The density report runs in-process on real matplotlib figures (the harness's own code); the +label lists stand for the probe's `labels` field (`{"kind", "text"}` per drawn text). +""" + +import json +from typing import Any + +import matplotlib.patheffects as patheffects +import matplotlib.pyplot as plt +import numpy as np +import pytest +from matplotlib.figure import Figure +from matplotlib.ticker import FuncFormatter + +from agents.anyplot.language import NUMBER_FORMATS, number_format, number_format_table +from agents.anyplot.policy import adapter_instruction, adapter_rubric +from agents.anyplot.render import harness +from agents.anyplot.render.backends.fake import fixture_png +from agents.anyplot.render.contract import RenderResult, ThemeOutput +from agents.anyplot.render.gates import ( + MAX_VALUE_LABELS, + _density_lines, + _number_format_lines, + evaluate, + foreign_decimal, +) +from core.defects import DEFECT_RE + + +GERMAN = NUMBER_FORMATS["de"] +ENGLISH = NUMBER_FORMATS["en"] + + +def gate_lines(probe: dict[str, Any], plot_language: str | None = None) -> list[str]: + output = ThemeOutput("light", 0, png=fixture_png("light", (3200, 1800)), probe=probe) + report = evaluate( + RenderResult("p", {"light": output}), + themes=("light",), + library="matplotlib", + rows=100, + plot_language=plot_language, + ) + return report.advisory + + +def grouped_bars(categories: int, *, labels: bool = True, dots: bool = True, shadows: bool = True) -> plt.Figure: + """The catalogue's bar-grouped style (value labels, dots on the bar tops, shadows) on `categories` x 2 bars.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + x = np.arange(categories) + values = np.random.default_rng(7).integers(500, 90_000, size=(2, categories)) + for offset, series in zip((-0.2, 0.2), values, strict=True): + container = ax.bar(x + offset, series, 0.4) + for rect in container: + if shadows: + rect.set_path_effects([patheffects.withSimplePatchShadow(offset=(4, -4))]) + top = rect.get_height() + centre = rect.get_x() + rect.get_width() / 2 + if dots: + ax.scatter(centre, top, s=34) + if labels: + ax.annotate(f"{int(top)}", xy=(centre, top), xytext=(0, 6), textcoords="offset points", ha="center") + ax.set_xticks(x, [f"Region {index}" for index in range(categories)], rotation=90) + return fig + + +@pytest.fixture(autouse=True) +def _close_figures() -> Any: + yield + plt.close("all") + + +class TestDensityReport: + def test_the_live_case_52_bars_with_labels_dots_and_shadows(self) -> None: + fig = grouped_bars(26) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 52 and axes["labelled"] == 52 + assert axes["decorated"] == 104 # 52 dots on the bar tops and 52 shadowed bars + assert axes["marks"] == 52 # the dots are decorations, not data marks + assert axes["label_px"] > axes["slot_px"] + [line] = _density_lines({"density": [axes]}) + assert line[0] == "G11" + + def test_the_catalogue_scale_is_clean(self) -> None: + """The catalogue's own 4 categories x 3 series with 3-digit labels: what the style was made for.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + x = np.arange(4) + for index, series in enumerate(([245, 312, 287, 425], [178, 195, 285, 310], [125, 210, 195, 165])): + container = ax.bar(x + (index - 1) * 0.25, series, 0.25) + ax.bar_label(container, fmt="%d", fontsize=9) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 12 and axes["labelled"] == 12 and axes["decorated"] == 0 + assert _density_lines({"density": [axes]}) == [] + + def test_heatmap_cell_annotations_are_no_value_labels(self) -> None: + """Cell text sits on no bar or point: the rubric allows it (quality-criteria, cell annotations).""" + fig, ax = plt.subplots(figsize=(6, 6), dpi=400) + ax.imshow(np.ones((20, 20))) + for row in range(20): + for column in range(20): + ax.text(column, row, "1", ha="center", va="center") + fig.canvas.draw() + + assert harness.density_report(fig, 400) == [] + + def test_a_label_on_every_point_of_a_long_line(self) -> None: + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + values = np.arange(60) ** 1.1 + ax.plot(np.arange(60), values, marker="o") + for index, value in enumerate(values): + ax.text(index, value + 1, f"{value:.0f}", ha="center") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 0 and axes["labelled"] == 60 + [(gate, line)] = _density_lines({"density": [axes]}) + assert "60 marks carry 60 value labels" in line + + def test_one_callout_in_axes_coordinates_is_no_value_label(self) -> None: + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.bar(range(40), range(1, 41)) + ax.annotate("Peak", xy=(0.5, 0.9), xycoords="axes fraction") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 0 and _density_lines({"density": [axes]}) == [] + + def test_named_points_are_no_value_labels_review_d02(self) -> None: + """Fable g11/d02 and perf/p04: 30 and 40 named scatter points (scatter-annotated asks for 10 to 50).""" + for count in (30, 40): + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + rng = np.random.default_rng(9) + x, y = rng.random(count) * 100, rng.random(count) * 100 + ax.scatter(x, y, s=80) + for index, (xi, yi) in enumerate(zip(x, y, strict=True)): + ax.text(xi + 1, yi + 1, f"Firm {index}", fontsize=9) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == count and axes["numeric_labels"] == 0 + assert _density_lines({"density": [axes]}) == [] + + def test_offset_shadow_bars_leave_the_space_per_label_review_d01(self) -> None: + """Fable g11/d01: 12 labelled bars over 12 offset shadow bars once measured 8 px per bar.""" + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + x = np.arange(12) + y = np.random.default_rng(1).integers(50, 500, 12) + ax.bar(x + 0.04, y - 4, color="#00000033", zorder=1) + container = ax.bar(x, y, zorder=2) + ax.bar_label(container, fmt="%d") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == 12 and axes["slot_px"] > 150 + assert _density_lines({"density": [axes]}) == [] + + def test_labelling_only_the_extremes_of_52_bars_is_clean(self) -> None: + """Gemini on #12184: the fix G11 asks for must pass it.""" + fig = grouped_bars(26, labels=False, dots=False, shadows=False) + ax = fig.axes[0] + rects = [rect for container in ax.containers for rect in container] + for rect in (max(rects, key=lambda r: r.get_height()), min(rects, key=lambda r: r.get_height())): + ax.annotate(f"{int(rect.get_height())}", xy=(rect.get_x() + rect.get_width() / 2, rect.get_height())) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 2 and _density_lines({"density": [axes]}) == [] + + def test_stacked_centre_labels_count_once_per_bar_review_p05(self) -> None: + """Fable perf/p05 and g11/a05: `bar_label(label_type="center")` labels were not counted at all.""" + for positions, fires in ((8, False), (40, True)): + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + x = np.arange(positions) + rng = np.random.default_rng(2) + bottom = np.zeros(positions) + for _ in range(4): + values = rng.integers(20, 100, positions) + container = ax.bar(x, values, bottom=bottom) + ax.bar_label(container, label_type="center", fmt="%d", fontsize=6) + bottom += values + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == 4 * positions and axes["labelled_bars"] == positions + assert bool(_density_lines({"density": [axes]})) is fires + + def test_an_inverted_value_axis_keeps_the_bar_ends(self) -> None: + """Gemini on #12184: the end of a bar on an inverted axis is its lower edge on the canvas.""" + fig = grouped_bars(26, labels=True, dots=True, shadows=False) + fig.axes[0].invert_yaxis() + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 52 and axes["decorated"] == 52 + + def test_dots_drawn_with_plot_markers_are_decorations(self) -> None: + """Gemini on #12184: `ax.plot(x, top, "o")` per bar, a Line2D of one vertex.""" + fig = grouped_bars(26, labels=False, dots=False, shadows=False) + ax = fig.axes[0] + for rect in [rect for container in ax.containers for rect in container]: + ax.plot(rect.get_x() + rect.get_width() / 2, rect.get_height(), "o") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["decorated"] == 52 and axes["marks"] == 52 + + def test_a_clipped_text_outside_a_zoomed_view_is_not_counted(self) -> None: + """Gemini on #12184: `ax.text(..., clip_on=True)` outside the view is never drawn.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.bar(range(50), range(1, 51)) + for index in range(50): + ax.text(index, index + 1.5, str(index + 1), ha="center", clip_on=True) + ax.set_xlim(-0.5, 4.5) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 5 + + def test_a_missing_bar_value_keeps_the_report_valid_json(self) -> None: + """Copilot on #12184: a NaN bar (a group missing after a pivot) would void the probe's JSON.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(4), [3.0, float("nan"), 2.0, 1.0]) + ax.bar_label(container) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 3 + json.dumps(axes, allow_nan=False) + + def test_bars_and_labels_outside_a_zoomed_view_are_not_counted(self) -> None: + """Copilot on #12184: 50 labelled bars zoomed to five draw five labels.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(50), range(1, 51)) + ax.bar_label(container) + ax.set_xlim(-0.5, 4.5) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 5 and axes["labelled"] == 5 + + def test_bars_zoomed_on_their_value_axis_still_count(self) -> None: + """Copilot round 2 on #12184: 30 bars ending at 1000 under `ylim(900, 1100)` still draw their ends.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(30), [1000] * 30) + ax.bar_label(container) + ax.set_ylim(900, 1100) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 30 and axes["labelled"] == 30 + assert _density_lines({"density": [axes]}) + + def test_many_scatter_marks_are_matched_in_bounded_chunks(self) -> None: + """Copilot on #12184: the pairwise arrays stay bounded however many marks the plot draws.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(30), range(1, 31)) + rng = np.random.default_rng(3) + ax.scatter(rng.uniform(0, 29, 9_000), rng.uniform(0, 30, 9_000), s=1) + ax.scatter([rect.get_x() + rect.get_width() / 2 for rect in container], range(1, 31), s=30) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["decorated"] >= 30 and axes["marks"] < 9_030 + + def test_the_probe_writes_the_density_field(self, tmp_path: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Figure, "savefig", Figure.savefig) # restored after the test + harness.install_probe("light") + fig = grouped_bars(26) + fig.savefig("plot-light.png", dpi=100) + + probe = json.loads((tmp_path / "probe-light.json").read_text()) + assert probe["density"][0]["bars"] == 52 + assert probe["tick_axes"] == [{"x": "fixed", "y": "number"}] + + +class TestDensityGate: + def test_the_line_is_a_measured_vq03_defect_line(self) -> None: + probe = { + "density": [{"bars": 52, "marks": 52, "labelled": 52, "decorated": 104, "slot_px": 35.0, "label_px": 160.0}] + } + + [line] = gate_lines(probe) + + assert DEFECT_RE.match(line), line + assert line.startswith("VQ-03 (light): 52 bars carry 52 value labels (160 px wide on 35 px per bar)") + assert "104 dots or shadows" in line and f"at most {MAX_VALUE_LABELS} labels" in line + assert "Likely cause:" in line + + @pytest.mark.parametrize( + "entry", + [ + {"bars": "52", "marks": 52, "labelled": 52}, + {"bars": True, "marks": 52, "labelled": 99}, + {"bars": 52, "marks": 52, "labelled": 10, "slot_px": float("nan"), "label_px": 1e9}, + "not an entry", + ], + ) + def test_a_tampered_entry_yields_no_line(self, entry: Any) -> None: + assert gate_lines({"density": [entry]}) == [] + + def test_a_malformed_density_never_hides_another_gate(self) -> None: + lines = gate_lines({"density": "broken", "tick_overlaps": 3}) + + assert any(line.startswith("VQ-02") for line in lines) + + def test_a_few_cramped_labels_are_callouts(self) -> None: + entry = {"bars": 4, "marks": 4, "labelled": 3, "decorated": 0, "slot_px": 20.0, "label_px": 120.0} + + assert _density_lines({"density": [entry]}) == [] + + +class TestForeignDecimal: + @pytest.mark.parametrize( + "text", + [ + "5.30", + "6.0 M", + "0.25", + "−1.5", + "1,234.5", + "12.5 %", + "$ 3.75", + "1,234,567", + "5.30 €", # review: a trailing currency, the German order + "-$5.30", + "0.125", # review: a leading 0 is never a thousands group + "−0.125", + ], + ) + def test_english_numbers_in_a_german_plot(self, text: str) -> None: + assert foreign_decimal(text, GERMAN) + + def test_a_zero_padded_day_and_month_is_a_date(self) -> None: + """Review of #12184: `14.10` in an annotation is a `%d.%m` date; `12.50` stays a decimal.""" + assert not foreign_decimal("14.10", GERMAN) + assert foreign_decimal("12.50", GERMAN) + assert not foreign_decimal("5,30 €", GERMAN) + + @pytest.mark.parametrize( + "text", + [ + "5,30", + "6,5 Mio.", + "1.234", # a thousands mark + "7.184.003", + "1.234,5", + "2024", + "14.10.", # a date, never a number + "2024-10-14", + "Q3", + "3.11.1 Version", + "1e6", + "", + ], + ) + def test_german_numbers_and_non_numbers_pass(self, text: str) -> None: + assert not foreign_decimal(text, GERMAN) + + @pytest.mark.parametrize("text", ["5,30", "6,5 M", "1.234,5"]) + def test_german_numbers_in_an_english_plot(self, text: str) -> None: + assert foreign_decimal(text, ENGLISH) + + @pytest.mark.parametrize("text", ["5.30", "6.5 M", "1,234", "1,234,567", "1,234.5", "7.184.003", "2024"]) + def test_english_numbers_pass_in_english(self, text: str) -> None: + assert not foreign_decimal(text, ENGLISH) + + +def probe_of(fig: plt.Figure, extra: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """The probe's `labels` (as `harness.drawn_labels` writes them, axes index included) and `tick_axes`.""" + fig.canvas.draw() + labels: list[dict[str, Any]] = [] + for index, ax in enumerate(fig.get_axes()): + labels += [{"kind": "annotation", "text": text.get_text()} for text in ax.texts] + for name in ("x", "y"): + ticks = getattr(ax, f"get_{name}ticklabels")() + labels += [ + {"kind": f"{name} tick label", "text": t.get_text(), "axes": index} for t in ticks if t.get_text() + ] + return {"labels": labels + (extra or []), "tick_axes": harness.tick_axes(fig)} + + +def seaborn_style_bar_labels(fmt_label: Any, language: str) -> dict[str, Any]: + """The live seaborn case: bar labels and y tick labels, and a title that names a version.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(["A", "B", "C", "D"], [5.3, 4.25, 3.1, 2.05]) + ax.bar_label(container, labels=[fmt_label(value) for value in container.datavalues]) + ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: fmt_label(value))) + return probe_of(fig, [{"kind": "title", "text": f"Version 2.5 ({language})"}]) + + +class TestNumberFormatGate: + def test_the_live_german_case_is_a_vq06_line(self) -> None: + probe = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "de") + + [line] = gate_lines(probe, plot_language="de") + + assert DEFECT_RE.match(line), line + assert line.startswith("VQ-06 (light): ") + assert "use a decimal point; German writes a decimal comma" in line + assert "Mio." in line and "annotations and tick labels" in line + assert "5.30" not in line # no label text reaches the line + + def test_the_fixed_german_case_is_clean(self) -> None: + german = str.maketrans(",.", ".,") + probe = seaborn_style_bar_labels(lambda value: f"{value:,.2f}".translate(german), "de") + + assert gate_lines(probe, plot_language="Deutsch") == [] + + def test_the_english_control_is_clean_and_german_marks_in_english_are_named(self) -> None: + english = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "en") + german = seaborn_style_bar_labels(lambda value: f"{value:.2f}".replace(".", ","), "en") + + assert gate_lines(english, plot_language="en") == [] + [line] = gate_lines(german, plot_language="en") + assert "English writes a decimal point" in line + + def test_date_ticks_and_category_ticks_are_never_checked(self) -> None: + """Copilot on #12184: a German `%d.%m` date (`14.10`) and a category the data names (`3.11`).""" + import matplotlib.dates as mdates + import pandas as pd + + fig, (dates, categories) = plt.subplots(1, 2, figsize=(8, 4.5), dpi=400) + days = pd.date_range("2026-10-01", periods=20, freq="D") + dates.plot(days, range(20)) + dates.xaxis.set_major_formatter(mdates.DateFormatter("%d.%m")) + dates.set_yticks([0, 10, 20]) + categories.bar(["3.11", "3.12", "3.13"], [1, 2, 3]) + categories.set_yticks([0, 1, 2, 3]) + probe = probe_of(fig) + + assert {"date", "category"} <= {entry["x"] for entry in probe["tick_axes"]} + assert any(label["text"] == "3.11" for label in probe["labels"]) + assert gate_lines(probe, plot_language="de") == [] + + @pytest.mark.parametrize(("fmt", "fires"), [("{:.2f}", True), ("{:,.2f}", False)]) + def test_the_real_probe_end_to_end( + self, fmt: str, fires: bool, tmp_path: Any, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The harness's own probe file (block F's `labels`, this gate's `tick_axes`) through `evaluate`.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Figure, "savefig", Figure.savefig) # restored after the test + harness.install_probe("light") + german = str.maketrans(",.", ".,") + + def label(value: float) -> str: + text = fmt.format(value) + return text if fires else text.translate(german) + + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(["Altbrunn", "Seefeld", "Hochmatt"], [5.3, 4.85, 6.12]) + ax.bar_label(container, labels=[label(value) for value in container.datavalues]) + ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: label(value))) + fig.savefig("plot-light.png", dpi=100) + probe = json.loads((tmp_path / "probe-light.json").read_text()) + + lines = [line for line in gate_lines(probe, plot_language="de") if line.startswith("VQ-06")] + + assert probe["labels"] and probe["tick_axes"] == [{"x": "category", "y": "number"}] + assert bool(lines) is fires + + def test_preconverted_date_coordinates_are_dates(self) -> None: + """Copilot round 2 on #12184: `date2num` coordinates under a DateFormatter leave the converter unset.""" + import datetime + + import matplotlib.dates as mdates + + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + start = datetime.datetime(2026, 10, 1) + days = [mdates.date2num(start + datetime.timedelta(days=offset)) for offset in range(20)] + ax.plot(days, range(20)) + ax.xaxis.set_major_formatter(mdates.DateFormatter("%d.%m")) + ax.set_yticks([0, 10, 20]) + probe = probe_of(fig) + + assert probe["tick_axes"] == [{"x": "date", "y": "number"}] + assert gate_lines(probe, plot_language="de") == [] + + def test_mathtext_reads_as_drawn(self) -> None: + """Copilot on #12184: `$\\mathdefault{5.30}$` shows a decimal point like `5.30`.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.plot([0, 1, 2], [0, 1, 2]) + ax.set_xticks([0, 1, 2]) + ax.set_yticks([0.25, 1.5], [r"$\mathdefault{0.25}$", r"$1.50$"]) + ax.text(1, 1, r"$5{,}30$") + probe = probe_of(fig) + assert probe["tick_axes"] == [{"x": "number", "y": "fixed"}] + probe["tick_axes"][0]["y"] = "number" # as a ScalarFormatter with mathtext would write them + + [line] = gate_lines(probe, plot_language="de") + + # x: 0, 1, 2 (no marks); y: the two mathtext labels; the annotation `$5{,}30$` reads 5,30. + assert line.startswith("VQ-06 (light): 2 of 6 numeric tick labels use a decimal point") + + def test_without_tick_axes_only_annotations_count(self) -> None: + labels = [{"kind": "y tick label", "text": "5.30", "axes": 0}, {"kind": "annotation", "text": "4.85"}] + + [line] = gate_lines({"labels": labels}, plot_language="de") + + assert line.startswith("VQ-06 (light): 1 of 1 numeric annotations use a decimal point") + + def test_a_language_without_a_number_format_and_no_language_have_no_gate(self) -> None: + probe = {"labels": [{"kind": "annotation", "text": "5.30"}]} + + assert _number_format_lines(probe, number_format("Swedish"), "Swedish") == [] + assert gate_lines(probe) == [] + + def test_titles_and_legend_entries_are_never_checked(self) -> None: + labels = [{"kind": "title", "text": "2.5"}, {"kind": "legend text", "text": "1.5"}] + + assert gate_lines({"labels": labels, "tick_axes": []}, plot_language="de") == [] + + def test_colorbar_and_3d_tick_labels_count(self) -> None: + labels = [ + {"kind": "colorbar tick label", "text": "0.25", "axes": 1}, + {"kind": "z tick label", "text": "1.5", "axes": 0}, + ] + ticks = [{"x": "number", "y": "number", "z": "number"}, {"x": "none", "y": "number"}] + + [line] = gate_lines({"labels": labels, "tick_axes": ticks}, plot_language="de") + + assert line.startswith("VQ-06 (light): 2 of 2 numeric tick labels use a decimal point") + + @pytest.mark.parametrize( + "probe", + [ + {"labels": "broken"}, + {"labels": [None, {"kind": "annotation", "text": 5.3}]}, + {"labels": [{"text": "5.30"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": True}], "tick_axes": [{"x": "number"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": 3}], "tick_axes": [{"x": "number"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": 0}], "tick_axes": "broken"}, + ], + ) + def test_a_tampered_labels_field_yields_no_line(self, probe: Any) -> None: + assert gate_lines(probe, plot_language="de") == [] + + +class TestNumberFormatTable: + def test_every_named_language_has_marks_and_words(self) -> None: + for code, spec in NUMBER_FORMATS.items(): + assert spec.decimal in ".," and spec.decimal != spec.thousands, code + assert spec.suffixes and all(word for _, word in spec.suffixes), code + + def test_names_and_codes_share_a_format(self) -> None: + assert number_format("German") is number_format("de") is number_format("Deutsch") is GERMAN + assert number_format("Klingon") is None + + def test_the_adapter_reads_the_table_and_the_rules(self) -> None: + instruction = adapter_instruction("matplotlib") + + assert number_format_table() in instruction + assert "| `de` (German) | `,` | `.` | 10^3: Tsd., 10^6: Mio., 10^9: Mrd. |" in instruction + assert "**Numbers.**" in instruction and "**Per-mark labels and decorations.**" in instruction + + def test_the_adapter_pattern_writes_every_table_language_and_passes_g12(self) -> None: + """Gemini on #12184: the old hint (`maketrans(",.", ".,")`, then the thousands mark for "the first `.`") was + ambiguous for space-grouped languages; the placeholder pattern is exact for every row.""" + assert '.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)' in adapter_instruction("matplotlib") + for code, spec in NUMBER_FORMATS.items(): + text = f"{1234567.25:,.2f}".replace(",", "_").replace(".", spec.decimal).replace("_", spec.thousands) + + assert text == f"1{spec.thousands}234{spec.thousands}567{spec.decimal}25", code + assert not foreign_decimal(f"{0.5:,.2f}".replace(".", spec.decimal), spec), code + + def test_the_adapter_reads_the_narrowed_house_style_rule(self) -> None: + assert "not those drawn once per bar or point" in adapter_rubric()