From 6873455c9472b44886737aec533be74026db9dbd Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:35:04 +0200 Subject: [PATCH 1/8] feat(agents): gate per-mark label density and the plot language's number format Two findings of the live tests of 2026-10-14/15 become measured gates: - G11 (VQ-03): the harness's density_report counts per axes the bars and points, the data-anchored value labels on them, dots on bar ends and shadowed bars, and the space a bar label has. More than 25 labels, bar labels wider than 90 % of the space per bar, or more than 25 dots or shadows give a measured line. The scorer's, reviewer's and adapter's house-style exemption no longer covers elements whose count grows with the data. - G12 (VQ-06): a numeric tick label or annotation whose decimal mark is not the plot language's (5.30 and 6.0 M in a German plot), read from the probe's `labels` field. language.NUMBER_FORMATS is the server's table of decimal and thousands marks and large-number words, carried in the adapter's static instruction with the rule to format through a FuncFormatter or an f-string (locale stays banned). Four hand-written eval cases generalise the live cases (de and an en control). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/README.md | 4 +- agents/anyplot/language.py | 48 +++ agents/anyplot/pipeline.py | 8 +- agents/anyplot/policy.py | 3 + agents/anyplot/prompts/adapter.md | 2 + agents/anyplot/prompts/reviewer.md | 2 +- agents/anyplot/prompts/scorer.md | 2 +- agents/anyplot/render/gates.py | 193 ++++++++++- agents/anyplot/render/harness.py | 164 +++++++++- agents/anyplot/settings.py | 2 +- agents/evals/cases.py | 2 +- agents/evals/fixtures/README.md | 8 +- .../bar-basic-seaborn-decimals-de/case.json | 13 + .../bar-basic-seaborn-decimals-de/data.csv | 7 + .../bar-basic-seaborn-decimals-en/case.json | 13 + .../bar-basic-seaborn-decimals-en/data.csv | 7 + .../case.json | 14 + .../data.csv | 53 ++++ .../case.json | 14 + .../change_request.txt | 1 + .../data.csv | 57 ++++ agents/evals/report.py | 2 +- docs/concepts/agent-network.md | 10 +- tests/unit/agents/evals/test_cases.py | 16 +- .../runtime/test_density_locale_gates.py | 299 ++++++++++++++++++ 25 files changed, 911 insertions(+), 33 deletions(-) create mode 100644 agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json create mode 100644 agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv create mode 100644 agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json create mode 100644 agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv create mode 100644 agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json create mode 100644 agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv create mode 100644 agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json create mode 100644 agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt create mode 100644 agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv create mode 100644 tests/unit/agents/runtime/test_density_locale_gates.py diff --git a/agents/README.md b/agents/README.md index 44e6ab47dad..75c2aaca091 100644 --- a/agents/README.md +++ b/agents/README.md @@ -30,7 +30,7 @@ The model is **Claude Haiku 5.5 on Vertex AI** (`claude-haiku-5-5`) by default. | `anyplot/data/generate.py` | The formula source behind `generate_data`: the `Generate` spec (one or two variable ranges, one to ten series), the whitelist evaluator over Python's `ast` (NumPy float64, never `eval`), the canonical CSV it writes, and `looks_like_formula`, the dataset route's `422 formula` guard | | `anyplot/plugins/` | `ScopeGuardPlugin`, `BudgetPlugin`, `ToolSafetyPlugin` and the request ledger they share | | `anyplot/output_guard.py` | The output sanitiser of every root message and plot-event line (links, HTML and long code blocks out, at most 1,200 characters) and the reply cap the ScopeGuard runs on the root's final reply before it is stored: a reply over 1,200 characters becomes the fixed refusal, the same in every language, or is cut at 1,200 characters when the turn ran the plot pipeline | -| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8/G9/G10), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, `serial.py`, the one render semaphore in front of every backend, and `watermark.py`, the footer strip the artifact route and the feedback bundle add to the raw render | +| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8/G9/G10/G11/G12), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, `serial.py`, the one render semaphore in front of every backend, and `watermark.py`, the footer strip the artifact route and the feedback bundle add to the raw render | | `anyplot/render/fonts/` | JetBrains Mono 2.304 Regular and Bold for the footer strip, unmodified, with `OFL.txt` (SIL Open Font License 1.1) and a notice with their SHA-256 values ([fonts README](anyplot/render/fonts/README.md)) | | `renderer/` | The `anyplot-renderer` service: `main.py` (`POST /render`, `GET /status`, one render slot), `executor.py` (one `sandbox do` per theme with the kill path, the byte watchdog and the launcher retry), `auth.py` (the caller check), `settings.py` (`RENDERER_*`), `wire.py` (the JSON contract the `remote` backend shares), `Dockerfile` and `cloudbuild.yaml`. No ADK import and no `__init__.py`, so `adk web agents` does not list it as an agent | | `anyplot/opening.py`, `session_state.py`, `services.py`, `briefs.py` | Opening a session and taking in a dataset, the server-set session state, the process-wide stores, and the spec and dataset briefs | @@ -403,7 +403,7 @@ Before the first case the harness renders the catalogue file of the first case o --renderer remote --render-url https://RENDERER_URL --gcloud-token ``` -- The full matrix is the 123 cases that are not stress cases. Cap the spend with `--budget-usd`: +- The full matrix is the 127 cases that are not stress cases. Cap the spend with `--budget-usd`: ```bash uv run --extra agents python -m agents.evals.matrix --cases full --budget-usd 4 \ diff --git a/agents/anyplot/language.py b/agents/anyplot/language.py index 28ec5f536dd..ab16f26c110 100644 --- a/agents/anyplot/language.py +++ b/agents/anyplot/language.py @@ -18,6 +18,7 @@ """ import re +from dataclasses import dataclass from typing import Literal @@ -74,6 +75,53 @@ """The English name of each code in `_NAMES`: server text, so a prompt may name the language outside a fence.""" +@dataclass(frozen=True) +class NumberFormat: + """How the text in a plot writes numbers in one language: its decimal and thousands marks and its short words + for large numbers, each as `(power of ten, word)`, smallest first.""" + + decimal: str + thousands: str + suffixes: tuple[tuple[int, str], ...] + + +NUMBER_FORMATS: dict[str, NumberFormat] = { + "en": NumberFormat(".", ",", ((3, "k"), (6, "M"), (9, "B"))), + "de": NumberFormat(",", ".", ((3, "Tsd."), (6, "Mio."), (9, "Mrd."))), + "fr": NumberFormat(",", " ", ((3, "k"), (6, "M"), (9, "Md"))), + "es": NumberFormat(",", ".", ((3, "mil"), (6, "M"), (9, "mil M"))), + "it": NumberFormat(",", ".", ((3, "mila"), (6, "Mln"), (9, "Mld"))), + "pt": NumberFormat(",", ".", ((3, "mil"), (6, "mi"), (9, "bi"))), + "nl": NumberFormat(",", ".", ((3, "k"), (6, "mln"), (9, "mld"))), + "pl": NumberFormat(",", " ", ((3, "tys."), (6, "mln"), (9, "mld"))), + "ru": NumberFormat(",", " ", ((3, "тыс."), (6, "млн"), (9, "млрд"))), + "vi": NumberFormat(",", ".", ((3, "N"), (6, "Tr"), (9, "Tỷ"))), + "zh": NumberFormat(".", ",", ((4, "万"), (8, "亿"))), + "ja": NumberFormat(".", ",", ((4, "万"), (8, "億"))), +} +"""The number format of each language in `ENGLISH_NAMES`, the server's own table (Q4 of the live test of 2026-10-14: +a German plot showed "6.0 M" and "5.30"). `locale` is a banned import in plot code, so the adapter formats with +`FuncFormatter` or an f-string and these marks (`number_format_table`), and gate G12 measures the decimal mark. +Chinese and Japanese group large numbers by ten thousand (万) and a hundred million (亿, 億). A language +outside the table has no number rule and no G12.""" + + +def number_format(language: str) -> NumberFormat | None: + """The number format of a session language (`de`, `Deutsch` and `German` share one), or None outside the table.""" + return NUMBER_FORMATS.get(canonical(language)) + + +def number_format_table() -> str: + """The table as Markdown rows for the adapter's static instruction: code, language, marks and words.""" + names = {" ": "space", " ": "narrow no-break space", " ": "no-break space"} + rows = ["| Plot language | Decimal mark | Thousands mark | Large numbers |", "|---|---|---|---|"] + for code, spec in NUMBER_FORMATS.items(): + words = ", ".join(f"10^{power}: {word}" for power, word in spec.suffixes) + thousands = names.get(spec.thousands, f"`{spec.thousands}`") + rows.append(f"| `{code}` ({ENGLISH_NAMES[code]}) | `{spec.decimal}` | {thousands} | {words} |") + return "\n".join(rows) + + def normalise(value: str) -> str: """The stored form of a language. diff --git a/agents/anyplot/pipeline.py b/agents/anyplot/pipeline.py index 6fb66bf263a..29dd9bf5761 100644 --- a/agents/anyplot/pipeline.py +++ b/agents/anyplot/pipeline.py @@ -352,7 +352,8 @@ class Candidate: adaptation_rules: list[str] = field(default_factory=list) """The ADAPTATION validator rule ids behind `adaptation_lines`, for the attribution log.""" advisory_gates: list[str] = field(default_factory=list) - """The advisory probe gate ids (`G3`, `G5`, `G7`, `G8`, `G9`, `G10`) behind `advisory_lines`, for the attribution log.""" + """The advisory probe gate ids (`G3`, `G5`, `G7`, `G8`, `G9`, `G10`, `G11`, `G12`) behind `advisory_lines`, for the + attribution log.""" review_score: int | None = None """The catalogue review's score of this render, when it had one (`AGENT_REVIEWER_MODE=catalogue`).""" review_lines: list[str] = field(default_factory=list) @@ -825,7 +826,8 @@ def finish(run: Run) -> PlotResult: if shipped.canvas_line: residual.append(shipped.canvas_line) # The measured findings are facts of the shipped render, reviewed or not: the ADAPTATION findings and - # every advisory gate line (clipping G3, annotations outside G5, overlap G7, G8, G9, missing glyphs G10). + # every advisory gate line (clipping G3, annotations outside G5, overlap G7, G8, G9, missing glyphs G10, + # crowded per-mark labels G11, the plot language's decimal mark G12). # No review score hides them (until 2026-10-14 only G10 stayed above the ship bar). residual += shipped.adaptation_lines residual += shipped.advisory_lines @@ -1292,7 +1294,7 @@ async def _attempts( started = time.monotonic() rendered = await services.backend.render(job) render_s = time.monotonic() - started - report = evaluate(rendered, themes=job.themes, library=view.library, rows=rows) + report = evaluate(rendered, themes=job.themes, library=view.library, rows=rows, plot_language=run.language) if report.passed_host_gates: run.repair_hints = list(report.repair_hints) attribution( diff --git a/agents/anyplot/policy.py b/agents/anyplot/policy.py index 48cc8221455..fcbf592f835 100644 --- a/agents/anyplot/policy.py +++ b/agents/anyplot/policy.py @@ -40,6 +40,8 @@ import yaml +from .language import number_format_table + PACKAGE_DIR = Path(__file__).resolve().parent PROMPTS_DIR = PACKAGE_DIR / "prompts" @@ -263,6 +265,7 @@ def adapter_instruction(library: str) -> str: # The rubric holds the VQ-03 marker-size and alpha table (`vq03_table`), so it is not repeated. return _join( own_prompt("adapter.md"), + "## Number formats by plot language\n\n" + number_format_table(), highlights, "## Excerpt: which weaknesses to fix\n\n" + weaknesses, "## Excerpt: visual-sizing fixes\n\n" + sizing, diff --git a/agents/anyplot/prompts/adapter.md b/agents/anyplot/prompts/adapter.md index 2dea33fb821..30483df5a5c 100644 --- a/agents/anyplot/prompts/adapter.md +++ b/agents/anyplot/prompts/adapter.md @@ -65,6 +65,8 @@ These rules replace the catalogue's own rules where they differ: - **Keep the plot.** Same plot type, same variant, same encodings and the same style. Change only what the user's data, the hints, the feedback and the change request need. Add no encodings, reference lines, highlights or callouts that nobody asked for. - **Callouts and statistics.** A label or callout that states a number must compute it from `df` in an f-string (`f"r = {df[x].corr(df[y]):.2f}"`, written with the column variables of the code). A callout target comes from a rule over `df` (the maximum, the latest row), and a conditional claim is drawn only when `df` meets it. Remove a callout that cannot be derived. - **Density.** Adapt marker sizes, line widths and alpha to the user's row count (see the VQ-03 table below). +- **Per-mark labels and decorations.** "The same style" covers elements whose count stays fixed (a title, a legend, one callout), not elements the catalogue draws once per bar or point of its few example marks: value labels, dots on bar ends, shadows. Their count grows with the user's data, so keep them only while each label fits the space of its mark and the plot stays legible, which rarely holds past about 25 marks; with more marks, drop them, or label only the extremes (the largest and the smallest) when the plot needs numbers. The server measures this (a `VQ-03` gate line counts the labels, the decorations and the pixels each label has). +- **Numbers.** Write every number in the plot text (tick labels, value labels, annotations, a number in the title) with the plot language's decimal and thousands marks and its words for large numbers, from the table "Number formats by plot language" below: `de` writes `5,30` and `6,5 Mio.`, `en` writes `5.30` and `6.5 M`. Python's `locale` is not available: format with an f-string and swap the marks (`f"{v:,.2f}".translate(str.maketrans(",.", ".,"))` for `,` decimals and `.` thousands; the table's thousands mark goes in place of the first `.`), through a `matplotlib.ticker.FuncFormatter` for the axes and the same function for value labels (`ax.bar_label(container, labels=[...])`, never `fmt="%.2f"`, which always writes a point). Years, dates, identifiers and the cells of the user's data stay as they are. A plot language the table does not list keeps the catalogue's formats. The server checks the decimal mark of every numeric tick label and annotation (a `VQ-06` gate line). - **Dates.** Use `matplotlib.dates.AutoDateLocator` with `ConciseDateFormatter` instead of a fixed locator, unless the change request asks for a specific one. - **Safe code.** Use only the imports the code already has, plus, when needed, `numpy`, `pandas`, the plotting library, scipy's computing modules (`scipy.stats`, `scipy.interpolate`, `scipy.signal`, `scipy.optimize`, `scipy.fft`, `scipy.linalg`, ...) and these standard modules: `math`, `cmath`, `statistics`, `decimal`, `fractions`, `numbers`, `datetime`, `calendar`, `zoneinfo`, `collections`, `itertools`, `functools`, `operator`, `bisect`, `heapq`, `typing`, `warnings`, `textwrap`, `colorsys` and `re`. Never import `io`, `sys`, `pathlib`, `random`, `locale` or `string`. No file or network access, no `os` beyond the THEME line, no `.format(...)` (use f-strings), no `eval`, `exec`, `getattr`, `operator.attrgetter` or other reflection, no names with double underscores or private names of a module (`module._name`), no `class` statements, and no string literal longer than 200 characters. diff --git a/agents/anyplot/prompts/reviewer.md b/agents/anyplot/prompts/reviewer.md index 427de130e45..490a582d642 100644 --- a/agents/anyplot/prompts/reviewer.md +++ b/agents/anyplot/prompts/reviewer.md @@ -39,7 +39,7 @@ Do not judge anything else: not the catalogue title format, not the realism of t These are never defects: -- The house style: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had, such as a shadow, an outline, a grid or a muted reference line. Data claims are not styling: a callout at an example value is still DQ-03. +- The house style: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had, such as a shadow, an outline, a grid or a muted reference line. Data claims are not styling: a callout at an example value is still DQ-03. Nor is an element drawn once per bar or point: value labels, dots on bar ends or shadows that the catalogue drew on a few example marks and the user's data repeats on many are judged like any other element (VQ-03 when they crowd the marks). - Column names, units and category labels written the way the user's data writes them, in every plot language; a text whose language you cannot judge. - A gate note on its own (see "The request"). - Something that could be better but fails no criterion above. diff --git a/agents/anyplot/prompts/scorer.md b/agents/anyplot/prompts/scorer.md index 44953f87f99..f615cb6252a 100644 --- a/agents/anyplot/prompts/scorer.md +++ b/agents/anyplot/prompts/scorer.md @@ -22,7 +22,7 @@ The rubric below was written for catalogue implementations that generate their o 3. **No catalogue title.** The mandated catalogue title (`{spec-id} · {language} · {library} · anyplot.ai`) does not apply: a title that names the user's data is correct, and the 5d note on the long mandated title is about that format only. SC-04 asks whether the title and the legend describe the user's data. The adapter keeps the user's column names on purpose: an axis title, legend title or label that is a bound column name (the bindings block lists them) or a category from the data, exactly as the data writes it, is correct and never a VQ-06 or SC-04 deduction, even when it is terse, abbreviated, has no unit or is in another language than the plot language. Units and category labels keep the spelling of the user's data too. Only a text the code added, a sentence or phrase rather than a name from the data, that is plainly in another language than the plot language is a VQ-06 defect. 4. **One theme.** Apply the theme-readability checklist (5c) only for the attached render's theme, and name that theme, or `code`, in every defect line instead of `both`. -The house style is never a defect: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had. +The house style is never a defect: a size, color, font or spacing the style guide prescribes, and a styling element the catalogue plot already had. This covers elements whose count stays fixed, not those drawn once per bar or point: value labels, dots on bar ends or shadows that the catalogue drew on its few example marks and the user's data repeats on many are judged like any other element (VQ-03 when they crowd the marks, VQ-02 when they overlap). ## Your answer diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 9fb0df93a0b..3eb59af1119 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -1,4 +1,4 @@ -"""Host gates on a render: R1-R3 decide, G3/G5/G7/G8/G9/G10 advise. +"""Host gates on a render: R1-R3 decide, G3/G5/G7/G8/G9/G10/G11/G12 advise. | Gate | Checks | Effect | |---|---|---| @@ -11,6 +11,8 @@ | G8 | probe: more point marks than data rows (fabricated data) | advisory: a DQ-03 line | | G9 | probe: scatter marks plus line vertices over `MAX_PLOTTED_POINTS` (computed mass data) | advisory: a DQ-03 line | | G10 | probe: characters without a glyph in any font of their text, plain text and mathtext (the harness records matplotlib's own reports) | advisory: a VQ-01 line | +| G11 | probe (`density`): value labels at a data coordinate on a bar's end or point, dots on bar ends and shadowed bars, per axes: more than `MAX_VALUE_LABELS` labels, bar labels wider than `LABEL_SLOT_FILL` of the space per bar, or more than `MAX_DECORATIONS` decorations | advisory: a VQ-03 line | +| G12 | probe (`labels`): numeric tick labels and annotations whose decimal mark is not the plot language's (`language.NUMBER_FORMATS`; `foreign_decimal`) | advisory: a VQ-06 line | A pipeline run renders one theme (`PipelineArgs.theme`) and the theme toggle renders the other one later, so `evaluate` judges exactly the themes of the job it is given: @@ -36,6 +38,7 @@ from core.canvas import check_canvas +from ..language import NumberFormat, english_name, number_format from ..schemas import MAX_LINE_CHARS from .contract import THEMES, RenderResult, Theme from .png import PngRejected, harden, pad_to @@ -71,6 +74,15 @@ of 50 columns, so the owner raised it to 500,000 (2026-10-14): a dense plot of the user's own data, such as 20,000 rows of 24 series (480,000 points), passes, while the stress cases' 610,000 and 1,000,000 points still trip it.""" +MAX_VALUE_LABELS = 25 +"""G11: value labels (texts at a data coordinate on a bar's end or a point) one axes may carry. Q3 of the live test of +2026-10-14: the catalogue's bar-grouped labels, dots and shadows, meant for its 12 bars, on 52 bars of 26 cantons.""" +LABEL_SLOT_FILL = 0.9 +"""G11: a bar label wider than this share of the space between neighbouring bar centres crowds its neighbours.""" +MIN_CRAMPED_LABELS = 6 # fewer labels than this are a few callouts, never crowded +MAX_DECORATIONS = 25 # G11: dots on bar ends and shadowed bars one axes may carry +MAX_DENSITY_AXES = 8 # `harness.DENSITY_AXES_LIMIT`: more entries are a tampered probe, read no further +MAX_CANVAS_PX = 10_000.0 # a pixel size past any canvas the gates accept is a tampered probe G10_SAMPLE = 3 # code points the G10 line names G10_NAME_CHARS = 15 # name text per label: a label is then at most 26 characters and the line fits MAX_NOTE_CHARS MAX_CODEPOINT = 0x10FFFF @@ -149,7 +161,7 @@ class GateReport: failed_gates: list[str] = field(default_factory=list) """The ids of the gates that failed or reported, one entry per theme and finding: `R1`, `R1-timeout`, `R1-` for a renderer limit (`R1-memory`, `R1-disk_budget`, ...), `R2`, - `R3`, then the probe gates `G3`, `G5`, `G7`, `G8`, `G9`, `G10` in the order the probe checks run + `R3`, then the probe gates `G3`, `G5`, `G7`, `G8`, `G9`, `G10`, `G11`, `G12` in the order the probe checks run (not sorted). Content-free, for the attribution log.""" @property @@ -256,11 +268,18 @@ def data_rows(data_csv: str) -> int: return max(0, len([line for line in data_csv.split("\n") if line]) - 1) -def evaluate(result: RenderResult, *, themes: Sequence[Theme], library: str, rows: int) -> GateReport: - """Run R1-R3 and the advisory gates over every theme of the job; a theme without an output fails R1.""" +def evaluate( + result: RenderResult, *, themes: Sequence[Theme], library: str, rows: int, plot_language: str | None = None +) -> GateReport: + """Run R1-R3 and the advisory gates over every theme of the job; a theme without an output fails R1. + + `plot_language` is the session language the plot text is written in; G12 checks the + decimal mark of its numbers when the language has a number format (`language.NUMBER_FORMATS`). + """ wanted = [theme for theme in THEMES if theme in themes] if not wanted: raise ValueError("a render job names at least one theme") + numbers = number_format(plot_language) if plot_language else None report = GateReport(passed_host_gates=True, canvas_ok=True) advisory: dict[str, list[Theme]] = {} missed: list[Theme] = [] @@ -307,7 +326,7 @@ def evaluate(result: RenderResult, *, themes: Sequence[Theme], library: str, row if verdict.defect_line and verdict.defect_line not in report.canvas_defects: report.canvas_defects.append(_line(verdict.defect_line)) report.padded_pngs[theme] = pad_to(hardened.data, verdict.target) - for gate, line in _probe_lines(output.probe, rows): + for gate, line in _probe_lines(output.probe, rows, numbers, plot_language): report.failed_gates.append(gate) advisory.setdefault(line, []).append(theme) # Both themes draw the same geometry, so the hint comes from the first theme that has one. @@ -424,12 +443,166 @@ def _text_kind(value: Any) -> str: return value if isinstance(value, str) and value in TEXT_KINDS else "text" -def _probe_lines(probe: dict[str, Any] | None, rows: int) -> list[tuple[str, str]]: - """Advisory `(gate, line)` pairs from one theme's probe; a malformed probe yields none, never an exception.""" - try: - return _probe_lines_unchecked(probe, rows) - except (ValueError, TypeError, OverflowError, AttributeError): +def _probe_lines( + probe: dict[str, Any] | None, rows: int, numbers: NumberFormat | None = None, plot_language: str | None = None +) -> list[tuple[str, str]]: + """Advisory `(gate, line)` pairs from one theme's probe; a malformed probe yields none, never an exception. + + Each gate is guarded on its own, so a malformed field of one gate never hides another's line. + """ + lines: list[tuple[str, str]] = [] + checks = ( + lambda: _probe_lines_unchecked(probe, rows), + lambda: _density_lines(probe), + lambda: _number_format_lines(probe, numbers, plot_language), + ) + for check in checks: + try: + lines += check() + except (ValueError, TypeError, OverflowError, AttributeError): + continue + return lines + + +def _count(value: Any) -> int | None: + """A non-negative int of the probe, or None (a bool, a float or a negative number is a tampered probe).""" + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _pixels(value: Any) -> float | None: + """A positive finite pixel size of the probe, or None.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + number = float(value) + return number if math.isfinite(number) and 0 < number <= MAX_CANVAS_PX else None + + +def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: + """The G11 line: per-mark value labels or decorations past what the plot's marks leave room for. + + The probe's `density` field (`harness.density_report`) describes each axes; the line + names the worst one. Labels crowd when there are more than `MAX_VALUE_LABELS` of them, + or when a bar label is wider than `LABEL_SLOT_FILL` of the space between bar centres + (from `MIN_CRAMPED_LABELS` labels on); decorations (dots on bar ends, shadowed bars) + when there are more than `MAX_DECORATIONS`. Only counts and pixels reach the line. + """ + entries = probe.get("density") if isinstance(probe, dict) else None + if not isinstance(entries, list): return [] + worst: tuple[int, str] | None = None + for entry in entries[:MAX_DENSITY_AXES]: + if not isinstance(entry, dict): + continue + bars, marks = _count(entry.get("bars")), _count(entry.get("marks")) + labelled, decorated = _count(entry.get("labelled")) or 0, _count(entry.get("decorated")) or 0 + if bars is None or marks is None: + continue + slot, label = _pixels(entry.get("slot_px")), _pixels(entry.get("label_px")) + cramped = ( + bars > 0 + and labelled >= MIN_CRAMPED_LABELS + and slot is not None + and label is not None + and label > slot * LABEL_SLOT_FILL + ) + many_labels = labelled > MAX_VALUE_LABELS + many_decorations = decorated > MAX_DECORATIONS + if not (cramped or many_labels or many_decorations): + continue + observed: list[str] = [] + if cramped or many_labels: + room = f" ({round(label)} px wide on {round(slot)} px per bar)" if cramped and label and slot else "" + observed.append(f"{labelled} value labels{room}") + if many_decorations: + observed.append(f"{decorated} dots or shadows") + marks_text = f"{bars} bars" if bars else f"{marks} marks" + line = ( + f"VQ-03 (THEME): {marks_text} carry {' and '.join(observed)} → at most {MAX_VALUE_LABELS} labels, each " + "fitting its mark, and no per-mark dots or shadows past that: drop them or label only the extremes. " + "Likely cause: the catalogue's per-mark labels, dots or shadows repeated on every mark of the data." + ) + severity = labelled + decorated + if worst is None or severity > worst[0]: + worst = (severity, line) + return [] if worst is None else [("G11", worst[1])] + + +_NUMBER_LABEL = re.compile( + r"^(?:[$€£¥]\s?)?[+\-−]?(?P\d[\d.,    ']*\d|\d)" + r"(?:\s?(?:%|‰|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?$" +) +"""A label that is one number: an optional currency sign and sign, digits with marks, an optional unit or word +(`%`, `Mio.`, `k`, `mil M`). Dates (`14.10.`, `2024-10-14`), ranges, times and words with digits never match.""" +NUMBER_LABEL_KINDS = frozenset({"tick label", "annotation", "text"}) +"""The label kinds whose numbers G12 checks; titles, axis titles and legend entries may name a version or a code.""" + + +def foreign_decimal(text: str, numbers: NumberFormat) -> bool: + """Whether a numeric label writes its decimals with the other mark than the plot language's. + + Only a label that is one number counts (`_NUMBER_LABEL`). With a decimal comma, a single + point followed by other than three digits (`5.30`, `6.0 M`) is foreign, and so is a point + after a comma (`1,234.5`) or more than one comma (`1,234,567`); a single point with three + digits (`1.234`) is a thousands mark and passes. A decimal point mirrors this, except that + several points (`7.184.003`) pass, because a version-like label writes them too. + """ + match = _NUMBER_LABEL.match(text.strip()) + if match is None: + return False + body = match.group("body") + marks = [char for char in body if char in ".,"] + if not marks: + return False + other = "." if numbers.decimal == "," else "," + own = numbers.decimal + if own in marks and other in marks: + return marks[-1] == other # the decimal mark comes last + if other not in marks: + return own == "," and marks.count(own) > 1 # `1,234,567` groups with commas: English + if marks.count(other) > 1: + return False # `7.184.003` or `1,234,567` group thousands the other way: ambiguous, never named + decimals = body.rsplit(other, 1)[1] + return len(decimals) != 3 + + +def _number_format_lines( + probe: dict[str, Any] | None, numbers: NumberFormat | None, plot_language: str | None +) -> list[tuple[str, str]]: + """The G12 line: numeric tick labels or annotations whose decimal mark is not the plot language's. + + Reads the probe's `labels` field (`{"kind", "text"}` per drawn text). No label text + reaches the line: a tick label can be a cell of the user's data. The language is named + by its English name from the server's table, never by the session's free-text value. + """ + labels = probe.get("labels") if isinstance(probe, dict) else None + if numbers is None or not isinstance(labels, list): + return [] + numeric = foreign = 0 + kinds: set[str] = set() + for label in labels: + if not isinstance(label, dict) or label.get("kind") not in NUMBER_LABEL_KINDS: + continue + text = label.get("text") + if not isinstance(text, str) or _NUMBER_LABEL.match(text.strip()) is None: + continue + numeric += 1 + if foreign_decimal(text, numbers): + foreign += 1 + kinds.add("tick labels" if label["kind"] == "tick label" else "annotations") + if not foreign: + return [] + name = english_name(plot_language or "") or "the plot language" + own, other = ("comma", "point") if numbers.decimal == "," else ("point", "comma") + thousands = {".": "a point", ",": "a comma"}.get(numbers.thousands, "a space") + words = ", ".join(word for _, word in numbers.suffixes) + return [ + ( + "G12", + f"VQ-06 (THEME): {foreign} of {numeric} numeric {' and '.join(sorted(kinds))} use a decimal {other}; " + f"{name} writes a decimal {own} → decimal {own}, {thousands} for thousands, {words} for large numbers. " + 'Likely cause: a default formatter or fmt="%.2f" instead of a FuncFormatter or f-string with these marks.', + ) + ] def _probe_lines_unchecked(probe: dict[str, Any] | None, rows: int) -> list[tuple[str, str]]: diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 4b8cea0e418..0a02f3b9a79 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -104,7 +104,9 @@ The probe feeds the advisory gates G3 (clipped text), G5 (annotation outside its axes), G7 (overlapping text: tick labels among themselves, any other text with any), G8 (more point marks than data rows), G9 (more plotted points than any dataset within the parser's caps can hold) and G10 -(characters without a glyph in any font of the text). It is +(characters without a glyph in any font of the text); the `density` field each save adds after the +probe (`density_report`: per axes, the bars and points and the value labels and decorations on them) +feeds G11. It is advisory because the code under test could tamper with it; the host never fails a render on it. The same holds for the `HARNESS` lines: the code under test shares stdout, so the host reads them as hints (a retry decision, a memory figure), never @@ -174,6 +176,11 @@ POLAR_BIN_DEG = 5 # width of one angle bin of the plotted points in the probe's `polar` field POLAR_RINGS = 10 # radius bins: tenths of the visible radius range, centre to edge POLAR_AXES_LIMIT = 4 # polar axes the probe describes per figure +DENSITY_AXES_LIMIT = 8 # axes the probe's `density` field describes per figure +DENSITY_LABEL_LIMIT = 500 # data-anchored texts per axes matched against the marks +DENSITY_MARK_LIMIT = 2_000 # bar ends and points per axes a text is matched against +DENSITY_BAR_SLACK = 0.12 # a label's anchor sits within this share of the axes' value extent past its bar's end +DENSITY_POINT_SLACK = (0.02, 0.08) # ... and within these shares of the axes' width and height of its point GLYPH_SAMPLE_LIMIT = 16 # code points the probe lists; the host shows a few of them MISSING_GLYPHS: set[int] = set() # code points reported by matplotlib's text layout, this process MISSING_MATH_GLYPHS: set[int] = set() # code points reported by mathtext (no fallback inside `$…$`) @@ -1194,6 +1201,155 @@ def polar_report(figure): return report +def _bar_ends(axes): + """Every visible bar of the axes as `(centre, end, width)` in display pixels along its orientation, plus a flag + whether it draws a path effect (a shadow), and the orientation of the first bar container. + + For a vertical bar the centre is its x centre, the end the y of the side away from the + baseline and the width its pixel width; a horizontal bar swaps the axes. + """ + bars = [] + orientation = None + for container in axes.containers: + if type(container).__name__ != "BarContainer": + continue + horizontal = getattr(container, "orientation", "vertical") == "horizontal" + orientation = orientation or ("horizontal" if horizontal else "vertical") + for patch in container.patches: + if not patch.get_visible(): + continue + box = patch.get_window_extent() + negative = (patch.get_width() if horizontal else patch.get_height()) < 0 + if horizontal: + bar = ((box.y0 + box.y1) / 2, box.x0 if negative else box.x1, box.y1 - box.y0) + else: + bar = ((box.x0 + box.x1) / 2, box.y0 if negative else box.y1, box.x1 - box.x0) + bars.append((*bar, bool(patch.get_path_effects()))) + return bars, orientation + + +def _point_marks(axes): + """The display positions of the axes' visible scatter marks and of the vertices of its lines of three or more.""" + import numpy as np + + scatter, vertices = [], [] + for collection in axes.collections: + if type(collection).__name__ != "PathCollection" or not collection.get_visible(): + continue + offsets = np.asarray(collection.get_offsets(), dtype=float).reshape(-1, 2) + scatter.append(collection.get_offset_transform().transform(offsets)) + for line in axes.get_lines(): + xy = np.asarray(line.get_xydata(), dtype=float).reshape(-1, 2) + if line.get_visible() and len(xy) >= 3: # two-vertex lines are rules, brackets and reference lines + vertices.append(line.get_transform().transform(xy)) + empty = np.empty((0, 2)) + return (np.concatenate(scatter) if scatter else empty), (np.concatenate(vertices) if vertices else empty) + + +def _label_anchors(axes): + """The visible texts of the axes placed at a data coordinate, with their anchors in display pixels. + + An annotation counts when its `xycoords` is "data" (`ax.annotate`, `ax.bar_label`), any + other text when it uses the data transform (`ax.text` at a data position). A callout in + axes coordinates is no value label. + """ + found = [] + for text in axes.texts: + if not text.get_visible() or not text.get_text().strip(): + continue + if type(text).__name__ == "Annotation": + if text.xycoords != "data": + continue + anchor = axes.transData.transform(text.xy) + elif text.get_transform() == axes.transData: + anchor = axes.transData.transform(text.get_position()) + else: + continue + found.append((text, float(anchor[0]), float(anchor[1]))) + if len(found) >= DENSITY_LABEL_LIMIT: + break + return found + + +def density_report(figure, dpi): + """Per axes (at most `DENSITY_AXES_LIMIT`, only those with bars or point marks): how many marks it draws and how + many of them carry a value label or a decoration, with the space a label has. + + `bars` counts the visible bars, `marks` bars plus scatter marks plus line vertices + (scatter marks on a bar's end count as decorations instead), `labelled` the texts at a + data coordinate whose anchor sits on a mark (a bar's end or a point; `_label_anchors`), + `decorated` the scatter marks on a bar's end plus the bars that draw a path effect such as + a shadow, `slot_px` the median distance between neighbouring bar centres (the space one + bar's label has along the category axis) and `label_px` the median extent of the bar + labels along that axis. Pixels are the saved image's. The host turns this into G11. + """ + import numpy as np + + scale = (dpi or figure.dpi) / figure.dpi + renderer = figure.canvas.get_renderer() + report = [] + for axes in figure.get_axes(): + if len(report) >= DENSITY_AXES_LIMIT: + break + try: + bars, orientation = _bar_ends(axes) + scatter, vertices = _point_marks(axes) + box = axes.get_window_extent(renderer) + except Exception: + continue + if not bars and not len(scatter) and not len(vertices): + continue + horizontal = orientation == "horizontal" + width, height = float(box.width), float(box.height) + value_extent = width if horizontal else height + bar_array = np.asarray([bar[:3] for bar in bars[:DENSITY_MARK_LIMIT]], dtype=float).reshape(-1, 3) + # A scatter mark on a bar's end is a decoration (the catalogue's dots on its bar tops), not a data mark. + on_bar = np.zeros(len(scatter), dtype=bool) + if len(bar_array) and len(scatter): + along, across = (scatter[:, 1], scatter[:, 0]) if horizontal else (scatter[:, 0], scatter[:, 1]) + near_centre = np.abs(along[:, None] - bar_array[None, :, 0]) <= bar_array[None, :, 2] / 2 + 1 + near_end = np.abs(across[:, None] - bar_array[None, :, 1]) <= 0.03 * value_extent + on_bar = (near_centre & near_end).any(axis=1) + points = np.concatenate([scatter[~on_bar], vertices])[:DENSITY_MARK_LIMIT] + labelled, label_sizes = 0, [] + for text, x, y in _label_anchors(axes): + along, across = (y, x) if horizontal else (x, y) + on_a_bar = len(bar_array) and bool( + ( + (np.abs(along - bar_array[:, 0]) <= bar_array[:, 2] / 2 + 1) + & (np.abs(across - bar_array[:, 1]) <= DENSITY_BAR_SLACK * value_extent) + ).any() + ) + on_a_point = len(points) and bool( + ( + (np.abs(x - points[:, 0]) <= DENSITY_POINT_SLACK[0] * width + 2) + & (np.abs(y - points[:, 1]) <= DENSITY_POINT_SLACK[1] * height + 2) + ).any() + ) + if not (on_a_bar or on_a_point): + continue + labelled += 1 + if on_a_bar: + try: + extent = text.get_window_extent(renderer) + except Exception: + continue + label_sizes.append(float(extent.height if horizontal else extent.width)) + centres = np.unique(np.round(bar_array[:, 0], 1)) if len(bar_array) else np.empty(0) + slot = float(np.median(np.diff(centres))) if len(centres) > 1 else None + report.append( + { + "bars": len(bars), + "marks": len(bars) + int((~on_bar).sum()) + len(vertices), + "labelled": labelled, + "decorated": int(on_bar.sum()) + sum(1 for bar in bars if bar[3]), + "slot_px": None if slot is None else round(slot * scale, 1), + "label_px": round(float(np.median(label_sizes)) * scale, 1) if label_sizes else None, + } + ) + return report + + def install_probe(theme): try: from matplotlib.figure import Figure @@ -1214,6 +1370,12 @@ def savefig(self, *args, **kwargs): polar = [] if polar: report["polar"] = polar + try: + density = density_report(self, kwargs.get("dpi")) + except Exception: # nor does the density report + density = [] + if density: + report["density"] = density try: text = json.dumps(report, allow_nan=False) # NaN or Infinity (a text at an infinite coordinate) except (TypeError, ValueError) as exc: diff --git a/agents/anyplot/settings.py b/agents/anyplot/settings.py index ee89e215eb5..2f762810ba9 100644 --- a/agents/anyplot/settings.py +++ b/agents/anyplot/settings.py @@ -215,7 +215,7 @@ class AgentSettings(BaseSettings): gate_repair: Literal["review", "first"] = "review" """When the measured findings of a render that passed the host gates start a repair (`AGENT_GATE_REPAIR`): - the advisory gate lines (G3, G5, G7, G8, G9, G10) and the ADAPTATION validator's soft findings. + the advisory gate lines (G3, G5, G7, G8, G9, G10, G11, G12) and the ADAPTATION validator's soft findings. `review`: they go to the round's self-review as findings, then to the review as gate notes, and a round whose render still has them after its review goes to the repair with them and the review's diff --git a/agents/evals/cases.py b/agents/evals/cases.py index e464aa4d935..f71fad70e4f 100644 --- a/agents/evals/cases.py +++ b/agents/evals/cases.py @@ -16,7 +16,7 @@ `select_cases` takes the harness's `--cases` value: `smoke` (the cases tagged `smoke`: one generated case per spec plus the hand-written `scatter-basic-matplotlib` and `bar-grouped-seaborn`), `full` (every case except the stress cases: the 120 generated -and the three hand-written ones, 123 since 2026-10-12), `stress` (the +and the seven hand-written ones, 127 since 2026-10-15), `stress` (the cases tagged `stress`, from `make_stress.py`), or comma-separated shell globs over case ids (`*-seaborn-*`, `pie-basic-*`, `stress-compute-*`). The catalogue snapshot comes from `snapshot.json` or, without it, from `plots//` in this diff --git a/agents/evals/fixtures/README.md b/agents/evals/fixtures/README.md index a79dab2b433..419f8c07293 100644 --- a/agents/evals/fixtures/README.md +++ b/agents/evals/fixtures/README.md @@ -12,15 +12,15 @@ Each directory under `cases/` is one case of the regression harness (`agents/eva A stress case's `case.json` also carries `bound_expected` (the gate, limit or error that must bound it), `needs_model`, `static` and `render` (what the static stage must measure), and for an injection case `marker`; `agents/evals/make_stress.py` documents each field. -There are 146 cases: +There are 150 cases: - **120 generated cases** `--`: 10 specs times matplotlib and seaborn times 6 perturbations (`renamed`, `x10`, `n12`, `n5000`, `date`, `decimal-comma`), written by `agents/evals/make_fixtures.py`. Never edit them by hand: change the generator and rerun `uv run --extra agents python -m agents.evals.make_fixtures`. Its docstring lists the specs, why each was chosen, and what each perturbation does. -- **3 hand-written cases**: `scatter-basic-matplotlib` and `bar-grouped-seaborn`, which the runtime tests also use, and `scatter-basic-matplotlib-german-message` (since 2026-10-12): German column names and a German follow-up message, `Mach die Punkte etwas größer.`, with no word about the language. The first plot comes from `create_plot`, which the scope judge never reads, so it is English; the follow-up is the session's first typed message and sets the session language to `de` (`detected`), so the second version must carry German plot text while the column names keep their spelling. The case names no `language` on purpose: it tests detection, not a choice. Each run's record holds the last `done` event's `language`. -- **23 stress cases** `stress-`, written by `agents/evals/make_stress.py` (rerun `uv run --extra agents python -m agents.evals.make_stress`): data at and one past the parser's caps, many categories, degenerate data, prompt injection, and requests that make the code compute mass data. They carry the tag `stress`; `--cases stress` selects them and `--cases full` leaves them out, so a full run keeps its 123 cases. +- **7 hand-written cases**: `scatter-basic-matplotlib` and `bar-grouped-seaborn`, which the runtime tests also use, and `scatter-basic-matplotlib-german-message` (since 2026-10-12): German column names and a German follow-up message, `Mach die Punkte etwas größer.`, with no word about the language. The first plot comes from `create_plot`, which the scope judge never reads, so it is English; the follow-up is the session's first typed message and sets the session language to `de` (`detected`), so the second version must carry German plot text while the column names keep their spelling. The case names no `language` on purpose: it tests detection, not a choice. Each run's record holds the last `done` event's `language`. Four more (since 2026-10-15, tags `density` or `number-format`) generalise the live tests of 2026-10-14/15 and choose their `language`: `bar-grouped-matplotlib-many-categories-de` (26 invented regions times two series, where the catalogue's per-bar labels, dots and shadows crowded 52 bars; gate G11), `bar-basic-seaborn-decimals-de` and its English control `bar-basic-seaborn-decimals-en` (bar values with two decimals, which seaborn's `bar_label` wrote as `5.30` in a German plot; gate G12), and `line-multi-matplotlib-millions-de` (two population series in the millions with the follow-up `Zeig die y-Achse in Millionen, zum Beispiel 6,5 Mio.`, after an axis that read `6.0 M`; gate G12). +- **23 stress cases** `stress-`, written by `agents/evals/make_stress.py` (rerun `uv run --extra agents python -m agents.evals.make_stress`): data at and one past the parser's caps, many categories, degenerate data, prompt injection, and requests that make the code compute mass data. They carry the tag `stress`; `--cases stress` selects them and `--cases full` leaves them out, so a full run keeps its 127 cases. The tag `smoke` marks the 12 cases of `--cases smoke`: one generated case per spec, cycling through the perturbations and both libraries, plus `scatter-basic-matplotlib` and `bar-grouped-seaborn`. -43 cases open their session with `locale` `de`: the 40 generated `renamed` and `decimal-comma` cases, `bar-grouped-seaborn` (tag `german-locale`) and the stress cases `cap-columns` and `mixed-decimals`. Since 2026-10-12 the locale sets no language, and their only turn is `create_plot`, so they reply and title their plots in English like every other case; their purpose is the data perturbation. `scatter-basic-matplotlib-german-message` is the case that exercises German replies and German plot text. +46 cases open their session with `locale` `de`: the 40 generated `renamed` and `decimal-comma` cases, `bar-grouped-seaborn` (tag `german-locale`), the three German block H cases (which also choose `language` `de`) and the stress cases `cap-columns` and `mixed-decimals`. Since 2026-10-12 the locale sets no language, and their only turn is `create_plot`, so they reply and title their plots in English like every other case; their purpose is the data perturbation. `scatter-basic-matplotlib-german-message` is the case that exercises German replies and German plot text. ## Stress cases: what bounds them diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json new file mode 100644 index 00000000000..399ac4b1bd1 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/case.json @@ -0,0 +1,13 @@ +{ + "spec_id": "bar-basic", + "library": "seaborn", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "category", "column": "Stadt"}, + {"role": "value", "column": "CO2 pro Kopf (t)"} + ], + "expected": "accepted", + "tags": ["categorical", "decimals", "de", "language", "number-format"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-15 (seaborn bar labels 5.30 in a German session)" +} diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv new file mode 100644 index 00000000000..cbbec094630 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-de/data.csv @@ -0,0 +1,7 @@ +Stadt,CO2 pro Kopf (t) +Altbrunn,5.30 +Seefeld,4.85 +Hochmatt,6.12 +Talwil,3.95 +Rosshalden,7.41 +Birkenau,4.27 diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json new file mode 100644 index 00000000000..43a023a83b7 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/case.json @@ -0,0 +1,13 @@ +{ + "spec_id": "bar-basic", + "library": "seaborn", + "locale": "en", + "language": "en", + "bindings": [ + {"role": "category", "column": "City"}, + {"role": "value", "column": "CO2 per capita (t)"} + ], + "expected": "accepted", + "tags": ["categorical", "decimals", "en", "language", "number-format"], + "source": "hand-written synthetic data, the English control of bar-basic-seaborn-decimals-de" +} diff --git a/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv new file mode 100644 index 00000000000..b78854e33a5 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-basic-seaborn-decimals-en/data.csv @@ -0,0 +1,7 @@ +City,CO2 per capita (t) +Altbrunn,5.30 +Seefeld,4.85 +Hochmatt,6.12 +Talwil,3.95 +Rosshalden,7.41 +Birkenau,4.27 diff --git a/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json new file mode 100644 index 00000000000..5e5bc79dac8 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/case.json @@ -0,0 +1,14 @@ +{ + "spec_id": "bar-grouped", + "library": "matplotlib", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "category", "column": "Region"}, + {"role": "group", "column": "Ereignis"}, + {"role": "value", "column": "Anzahl"} + ], + "expected": "accepted", + "tags": ["categorical", "many-categories", "de", "language", "density"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-14 (B: 26 regions x 2 series)" +} diff --git a/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv new file mode 100644 index 00000000000..b721b778920 --- /dev/null +++ b/agents/evals/fixtures/cases/bar-grouped-matplotlib-many-categories-de/data.csv @@ -0,0 +1,53 @@ +Region,Ereignis,Anzahl +Aarwald,Geburten,12895 +Aarwald,Todesfälle,12435 +Bergfeld,Geburten,4925 +Bergfeld,Todesfälle,4499 +Brunnau,Geburten,3722 +Brunnau,Todesfälle,2612 +Dornegg,Geburten,6956 +Dornegg,Todesfälle,7264 +Eichtal,Geburten,13097 +Eichtal,Todesfälle,14382 +Felsach,Geburten,935 +Felsach,Todesfälle,990 +Grünmatt,Geburten,1359 +Grünmatt,Todesfälle,1512 +Hasenbühl,Geburten,14066 +Hasenbühl,Todesfälle,13537 +Ilfingen,Geburten,1591 +Ilfingen,Todesfälle,1436 +Juraweid,Geburten,14541 +Juraweid,Todesfälle,14518 +Kaltbrunn,Geburten,8252 +Kaltbrunn,Todesfälle,7096 +Lindau-Ost,Geburten,4217 +Lindau-Ost,Todesfälle,3909 +Mooshalden,Geburten,12995 +Mooshalden,Todesfälle,9336 +Nesselried,Geburten,12904 +Nesselried,Todesfälle,11989 +Oberried,Geburten,14029 +Oberried,Todesfälle,15283 +Pfannstiel,Geburten,1151 +Pfannstiel,Todesfälle,1247 +Quellmatt,Geburten,15082 +Quellmatt,Todesfälle,12359 +Rebhalde,Geburten,7120 +Rebhalde,Todesfälle,6801 +Seewinkel,Geburten,4894 +Seewinkel,Todesfälle,4904 +Tannegg,Geburten,14311 +Tannegg,Todesfälle,15669 +Unterau,Geburten,6607 +Unterau,Todesfälle,6048 +Vorderwald,Geburten,15897 +Vorderwald,Todesfälle,13608 +Weissbach,Geburten,7805 +Weissbach,Todesfälle,6402 +Zelgli,Geburten,11394 +Zelgli,Todesfälle,10652 +Ahornried,Geburten,5982 +Ahornried,Todesfälle,6609 +Buchsee,Geburten,4698 +Buchsee,Todesfälle,4461 diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json new file mode 100644 index 00000000000..0dc265c7a01 --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/case.json @@ -0,0 +1,14 @@ +{ + "spec_id": "line-multi", + "library": "matplotlib", + "locale": "de", + "language": "de", + "bindings": [ + {"role": "x", "column": "Jahr"}, + {"role": "y1", "column": "Bevölkerung"}, + {"role": "series", "column": "Staatsangehörigkeit"} + ], + "expected": "accepted", + "tags": ["numeric", "long-format", "de", "language", "number-format"], + "source": "hand-written synthetic data, generalised from the live test of 2026-10-14 (A: an axis in millions showed 6.0 M in a German session)" +} diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt new file mode 100644 index 00000000000..ceaf272addc --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/change_request.txt @@ -0,0 +1 @@ +Zeig die y-Achse in Millionen, zum Beispiel 6,5 Mio. diff --git a/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv new file mode 100644 index 00000000000..a77f2dacaaa --- /dev/null +++ b/agents/evals/fixtures/cases/line-multi-matplotlib-millions-de/data.csv @@ -0,0 +1,57 @@ +Jahr,Staatsangehörigkeit,Bevölkerung +1971,Schweiz,5090735 +1971,Ausland,1051054 +1973,Schweiz,5149288 +1973,Ausland,1091874 +1975,Schweiz,5212357 +1975,Ausland,1149602 +1977,Schweiz,5284253 +1977,Ausland,1178116 +1979,Schweiz,5337176 +1979,Ausland,1214283 +1981,Schweiz,5403798 +1981,Ausland,1279596 +1983,Schweiz,5480850 +1983,Ausland,1320628 +1985,Schweiz,5522363 +1985,Ausland,1372426 +1987,Schweiz,5586538 +1987,Ausland,1403167 +1989,Schweiz,5645642 +1989,Ausland,1443281 +1991,Schweiz,5731407 +1991,Ausland,1501651 +1993,Schweiz,5790434 +1993,Ausland,1519336 +1995,Schweiz,5845489 +1995,Ausland,1592226 +1997,Schweiz,5914754 +1997,Ausland,1610270 +1999,Schweiz,5974773 +1999,Ausland,1674374 +2001,Schweiz,6015759 +2001,Ausland,1710136 +2003,Schweiz,6077411 +2003,Ausland,1744682 +2005,Schweiz,6168536 +2005,Ausland,1784431 +2007,Schweiz,6216939 +2007,Ausland,1855071 +2009,Schweiz,6272127 +2009,Ausland,1887823 +2011,Schweiz,6341441 +2011,Ausland,1932134 +2013,Schweiz,6397995 +2013,Ausland,1981941 +2015,Schweiz,6473849 +2015,Ausland,2013750 +2017,Schweiz,6531505 +2017,Ausland,2047569 +2019,Schweiz,6576507 +2019,Ausland,2097893 +2021,Schweiz,6655738 +2021,Ausland,2164121 +2023,Schweiz,6719001 +2023,Ausland,2203491 +2025,Schweiz,6762891 +2025,Ausland,2231271 diff --git a/agents/evals/report.py b/agents/evals/report.py index 9078dd15bf1..2d14706df04 100644 --- a/agents/evals/report.py +++ b/agents/evals/report.py @@ -22,7 +22,7 @@ **Pass.** A run passes when it shipped a render (`ok` or `needs_attention`) that passed every deterministic gate: the host gates R1 to R3 without a padded canvas, and no ADAPTATION validator finding on the shipped code. The advisory probe gates (G3, -G5, G7, G8, G9, G10) and the reviewer's verdict do not decide a pass; the owner's blind +G5, G7, G8, G9, G10, G11, G12) and the reviewer's verdict do not decide a pass; the owner's blind judgement in the gallery does that for the sample, which holds passed renders only. Every rate counts all runs: a run that crashed in the harness did not pass. **Accept-match** compares the harness outcome (`accepted` when the status is `ok`, the diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 68077613d5e..a8ea714d0c5 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -310,10 +310,12 @@ The `remote` backend mints the ID token for `AGENT_RENDER_URL`, the renderer's s A run renders one theme, the one `PipelineArgs.theme` names, and the host gates judge exactly the themes of the job (an output for another theme is ignored, a missing one fails R1). The theme toggle renders the other theme of a finished version later, as a job of its own with the same code and `data.csv`, and adds its PNG to the version's render. Renders are strictly serial per instance, because spikes S and S2 showed one 4 GiB instance serves one sandbox at a time safely. No backend keeps a semaphore of its own: `Services.backend` wraps whichever backend `make_backend` built in `SerialRenderer` (`render/serial.py`), which renders one theme per slot under `AGENT_RENDER_CONCURRENCY` slots, 1 by default, so every backend is serial by construction, `adk web` runs included; the `remote` backend therefore sends one theme per request. The renderer holds one render slot of its own on top, because the deployed agents service and a developer's `adk web` call it independently. A freed slot goes to a waiting pipeline render before a waiting theme toggle, so a run waits for at most the render in progress; a render's timeout starts when it runs. The theme toggle records a theme that failed the host gates and renders it at most twice; a renderer that could not run is not recorded. `harness.py` (copied into the renderer image at `/opt/anyplot/harness.py`) prints its `HARNESS` start line, sets the resource limits soft and hard alike (CPU seconds and file size always; the address space and the process count when the renderer passes them, because outside a sandbox `RLIMIT_NPROC` counts every process of the user), copies the baked font cache into `MPLCONFIGDIR`, appends the installed fallback families (`FALLBACK_FAMILIES` in `harness.py`: Noto Sans, Noto Sans CJK JP, and the Noto Sans faces for Arabic, Hebrew, the nine Indic scripts, Thai, Khmer, Myanmar, Ethiopic and two symbol blocks) to every font lookup after the font the code chose, so DejaVu Sans stays the catalogue's first font and text in any of those scripts still draws, whatever `font.family` the code or seaborn's `set_theme` set; records every code point matplotlib reports as having no glyph in any font of its text, plain text and `$…$` math alike, as `missing_glyphs` in the probe; registers a `Figure.savefig` probe that writes `probe-.json`, compiles the plot file and runs it as `__main__` under the plot guard, and prints its peak memory and CPU time in a closing `HARNESS` line. -Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` with no symlinks, at most 10 MB, magic bytes, Pillow under `MAX_IMAGE_PIXELS`, non-blank with less than 98 % background, re-encode) are blocking; a render that fails either is discarded and never becomes `best`. R3 (canvas within 16 px through `core/canvas.py`, lifted from `.github/workflows/impl-review.yml`) is repair-triggering with a fallback: an R3 miss after normalisation is a defect line for the repair; if it persists after the last allowed attempt, the host pads the PNG of the rendered theme to the target canvas (never crops), the padded artifact counts as having passed the host gates, and the result ships as `needs_attention` with the residual line `canvas padded after render ()` and the VQ-05 line for that theme, never silently. A job with both themes pads each theme that missed and keeps the other theme's PNG as it rendered. The theme toggle answers a padded theme as `needs_attention` with the reason `canvas_padded`. The probe gates (G3 clipping, G5 annotation out of view, G7 overlapping text: tick labels among themselves, and since 2026-10-12 every other drawn text (axis label, title, legend text, annotation, or a text no role names) against any drawn text by more than 3 px, named by role and never by content, with a polar hint when an axis label of polar axes is involved, after the owner's polar case where the radial label `y` sat on the `270°` tick label and the reviewer called it missing; G8 row fidelity, G9 more than 500,000 plotted points, G10 characters without a glyph) are advisory because in-sandbox data can be tampered with: they can only trigger a repair or add notes, never fail a render. Since 2026-10-12 the probe also describes each polar axes (`polar`: the radial labels' angle, the visible theta span and a histogram of the plotted points by 5° of angle and by tenth of the radius range, all in the axes' own theta coordinates), and `agents/anyplot/render/polar.py` turns it into a measured target, not a gate: when the radial tick labels' ray holds plotted points and a ray every 22.5° inside the span holds at most half as many, `GateReport.repair_hints` gets one line such as "polar radial tick labels: at 67.5° they sit on 10 plotted points → ax.set_rlabel_position(270), the freest direction, 0 points there. Likely cause: ...". It never starts a repair; a repair attempt that runs for other findings carries the latest render's line first among its `hints`, and the adapter prompt says to apply it with exactly that value. The owner's case of that day (seaborn `polar-scatter`, `y = sin(x)`) had the repair guess 67.5°, onto the data; evenly spread angles give no line, but a cardioid does, because a point counts only when its distance from the ray as a share of the radius is small and it is outside the innermost tenth. The first attempt gets no such hint: nothing is rendered yet, and the angle of the user's column as the plot will draw it (degrees or radians, offset, direction) is only known from the render. A renderer limit that stopped a run (`memory`, `disk_budget`, `file_budget`, `output_rejected`) is recorded as `R1-` among the failed gates. The defect-line helpers (`DEFECT_RE`, `weakness_class`, `defect_ids`, and the `format_defect` builder) live in `core/defects.py`, the canonical stdlib-only implementation, and the PNG auto-reject checks (AR-04 blank and AR-07 format; R2 calls `check_blank` with its 98 % threshold) live next to the canvas gate in `core/canvas.py`, so the agents image needs no `automation/` or `scripts/` files. `automation/scripts/regen_gate.py` keeps a parity-tested copy of the grammar, because the workflows run it as a single-file copy with the runner's system Python, where `core` is not importable. The pipeline switches to `core.defects` in the later change that also switches the canvas gate and updates the copy sites: the `cp` step in `impl-review.yml`, the `git show` in `impl-generate.yml`, `review_retest.py materialize`, and the overlay in `review-retest.yml`. +Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` with no symlinks, at most 10 MB, magic bytes, Pillow under `MAX_IMAGE_PIXELS`, non-blank with less than 98 % background, re-encode) are blocking; a render that fails either is discarded and never becomes `best`. R3 (canvas within 16 px through `core/canvas.py`, lifted from `.github/workflows/impl-review.yml`) is repair-triggering with a fallback: an R3 miss after normalisation is a defect line for the repair; if it persists after the last allowed attempt, the host pads the PNG of the rendered theme to the target canvas (never crops), the padded artifact counts as having passed the host gates, and the result ships as `needs_attention` with the residual line `canvas padded after render ()` and the VQ-05 line for that theme, never silently. A job with both themes pads each theme that missed and keeps the other theme's PNG as it rendered. The theme toggle answers a padded theme as `needs_attention` with the reason `canvas_padded`. The probe gates (G3 clipping, G5 annotation out of view, G7 overlapping text: tick labels among themselves, and since 2026-10-12 every other drawn text (axis label, title, legend text, annotation, or a text no role names) against any drawn text by more than 3 px, named by role and never by content, with a polar hint when an axis label of polar axes is involved, after the owner's polar case where the radial label `y` sat on the `270°` tick label and the reviewer called it missing; G8 row fidelity, G9 more than 500,000 plotted points, G10 characters without a glyph, and since 2026-10-15 G11 per-mark value labels and decorations past what the marks leave room for, and G12 a decimal mark that is not the plot language's) are advisory because in-sandbox data can be tampered with: they can only trigger a repair or add notes, never fail a render. Since 2026-10-12 the probe also describes each polar axes (`polar`: the radial labels' angle, the visible theta span and a histogram of the plotted points by 5° of angle and by tenth of the radius range, all in the axes' own theta coordinates), and `agents/anyplot/render/polar.py` turns it into a measured target, not a gate: when the radial tick labels' ray holds plotted points and a ray every 22.5° inside the span holds at most half as many, `GateReport.repair_hints` gets one line such as "polar radial tick labels: at 67.5° they sit on 10 plotted points → ax.set_rlabel_position(270), the freest direction, 0 points there. Likely cause: ...". It never starts a repair; a repair attempt that runs for other findings carries the latest render's line first among its `hints`, and the adapter prompt says to apply it with exactly that value. The owner's case of that day (seaborn `polar-scatter`, `y = sin(x)`) had the repair guess 67.5°, onto the data; evenly spread angles give no line, but a cardioid does, because a point counts only when its distance from the ray as a share of the radius is small and it is outside the innermost tenth. The first attempt gets no such hint: nothing is rendered yet, and the angle of the user's column as the plot will draw it (degrees or radians, offset, direction) is only known from the render. A renderer limit that stopped a run (`memory`, `disk_budget`, `file_budget`, `output_rejected`) is recorded as `R1-` among the failed gates. The defect-line helpers (`DEFECT_RE`, `weakness_class`, `defect_ids`, and the `format_defect` builder) live in `core/defects.py`, the canonical stdlib-only implementation, and the PNG auto-reject checks (AR-04 blank and AR-07 format; R2 calls `check_blank` with its 98 % threshold) live next to the canvas gate in `core/canvas.py`, so the agents image needs no `automation/` or `scripts/` files. `automation/scripts/regen_gate.py` keeps a parity-tested copy of the grammar, because the workflows run it as a single-file copy with the runner's system Python, where `core` is not importable. The pipeline switches to `core.defects` in the later change that also switches the canvas gate and updates the copy sites: the `cp` step in `impl-review.yml`, the `git show` in `impl-generate.yml`, `review_retest.py materialize`, and the overlay in `review-retest.yml`. **Scripts and fonts.** Plot text can be in the scripts most users' data and languages bring: the renderer image installs Noto Sans, Noto Sans CJK and the Noto Sans faces of the scripts DejaVu Sans lacks (37 files, 46.8 MB, Regular and Bold where the family has both), and the harness appends them to every font lookup, so a Chinese title or a Devanagari tick label draws with the right glyphs while the code's own first font stays in charge of Latin text. Right-to-left scripts work: the pinned matplotlib lays text out through libraqm, so Arabic and Hebrew render shaped and in reading order, within one label aligned by the code's `ha`. Three limits remain, and G10 reports the first and the last: a `$…$` math string uses matplotlib's math fonts without fallback and draws `¤` for every character they lack, so non-Latin words belong outside math text; Han characters take the Japanese regional forms of the CJK face until a caller passes the plot's language (`text.language`, a follow-up with the language chip); and a character outside the installed families draws as a Last Resort placeholder box. Most pictographs are not in that last group, because Noto Sans Symbols2 or DejaVu Sans draws them as monochrome outlines (📈, 📊, ⚠, 😀), but 🔥, ✅, the flag letters, Tibetan and Mongolian have no glyph in any installed font. G10 counts those characters deterministically from matplotlib's own reports, names up to three of them with their code points (`U+1F525 🔥 fire`), tells the repair that a user's column name stays as it is, and keeps the line in the result's residual lines even when the reviewer passed the render, because a measured missing glyph is not a judgement. +**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits on a bar's end or a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels), the dots on bar ends and the shadowed bars, and measures the median bar label width against the space between bar centres. G11 writes a VQ-03 line when an axes carries more than `MAX_VALUE_LABELS` (25) value labels, from six labels on when a bar label is wider than 90 % of the space per bar, or more than `MAX_DECORATIONS` (25) dots or shadows. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string. G12 reads the probe's `labels` field (the drawn texts by kind) and writes a VQ-06 line when a numeric tick label or annotation uses the other decimal mark: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`), a point after a comma, or commas grouping thousands; dates, years, version strings, titles and legend entries never count, and no label text reaches the line. + Later languages reuse the CI commands: `Rscript plot.R`; `julia --project=/opt/julia plot.jl` with a stacked `JULIA_DEPOT_PATH` and a pinned `JULIA_CPU_TARGET`; `node /opt/anyplot/js-render/render.mjs plot.js` with `window.ANYPLOT_DATA` injected; bokeh with `BOKEH_RESOURCES=inline`, `SE_OFFLINE=true` and a shipped chromedriver; plotly with local or disabled MathJax. ### Footer strip @@ -382,13 +384,13 @@ Built since 2026-10-12, the capture half. Every result card carries the control Testing a newer model version, the other provider's arm, a different judge model, or a prompt or validator change is one command locally or one workflow dispatch in CI, and the answer is a diff against the pinned baseline. -Built on 2026-10-10: the harness, the two fixture generators and the 146 fixture cases (the 123 of the full matrix and the 23 stress cases with their model-free `--static` stage), the reports, the diff, the review gallery, the blind two-run gallery with its scoring, the model-free catalogue eligibility sweep `agents/evals/eligibility.py` (`agents/README.md`, "Run the regression harness"), and the judge-only scope eval `agents/evals/scope.py` with its refusal recall and false-refusal rate (`agents/README.md`, "Run the scope eval"; its first run passed both gates). Not built yet: `--prompts `, the Gemini baseline, and the CI workflow. +Built on 2026-10-10: the harness, the two fixture generators and the 146 fixture cases, 150 since 2026-10-15 (the 127 of the full matrix and the 23 stress cases with their model-free `--static` stage), the reports, the diff, the review gallery, the blind two-run gallery with its scoring, the model-free catalogue eligibility sweep `agents/evals/eligibility.py` (`agents/README.md`, "Run the regression harness"), and the judge-only scope eval `agents/evals/scope.py` with its refusal recall and false-refusal rate (`agents/README.md`, "Run the scope eval"; its first run passed both gates). Not built yet: `--prompts `, the Gemini baseline, and the CI workflow. -- **Fixture cases**, each a directory with `case.json` (spec, library, bindings, expected `accepted`, `rejected` or `unknown`, tags, the perturbation and the source of the data, and optionally a session `language` that the harness chooses through the language route before the first turn; a promoted case carries its session's when that was detected or chosen), `data.csv`, and an optional `change_request.txt`. Two sources: the committed synthetic fixtures in `agents/evals/fixtures/cases//`, which are the spike-X matrix (10 specs times matplotlib and seaborn times 6 perturbation datasets, written by the seeded generator `agents/evals/make_fixtures.py`) plus three hand-written cases (one of them, since 2026-10-12, a German follow-up message that sets the session language) and the 23 stress cases of `agents/evals/make_stress.py` (tag `stress`, left out of `full`; see [Bounds](#bounds)), and the promoted feedback cases synced from the private bucket into the gitignored `agents/evals/.cases/` directory (never committed, because they hold users' data). The tag `smoke` marks the 12 cases of the smoke set. The scope set is the judge-only case file `agents/evals/scope.evalset.json` (see Evals below); the injection and code-escape sets stay as flow evals. +- **Fixture cases**, each a directory with `case.json` (spec, library, bindings, expected `accepted`, `rejected` or `unknown`, tags, the perturbation and the source of the data, and optionally a session `language` that the harness chooses through the language route before the first turn; a promoted case carries its session's when that was detected or chosen), `data.csv`, and an optional `change_request.txt`. Two sources: the committed synthetic fixtures in `agents/evals/fixtures/cases//`, which are the spike-X matrix (10 specs times matplotlib and seaborn times 6 perturbation datasets, written by the seeded generator `agents/evals/make_fixtures.py`) plus seven hand-written cases (one of them, since 2026-10-12, a German follow-up message that sets the session language; four, since 2026-10-15, choose a language and cover label density and number formats) and the 23 stress cases of `agents/evals/make_stress.py` (tag `stress`, left out of `full`; see [Bounds](#bounds)), and the promoted feedback cases synced from the private bucket into the gitignored `agents/evals/.cases/` directory (never committed, because they hold users' data). The tag `smoke` marks the 12 cases of the smoke set. The scope set is the judge-only case file `agents/evals/scope.evalset.json` (see Evals below); the injection and code-escape sets stay as flow evals. - **Harness** (`agents/evals/matrix.py`, the spike-X runner made permanent): runs every case through the real `/v1` flow in-process (httpx `ASGITransport` against `agents.main:app`, the same plugins and run queue, the models from the settings, and the render backend `--renderer` names: `remote` against a deployed renderer, `local` in Docker, or `fake` for a dry run of everything but the render) with candidate settings that are passed only to the harness: `--provider` (alone it selects that provider's default models; the settings refuse a model from the other provider), `--model`, `--judge-model`, `--location`, `--cases smoke|full|stress|`, `--static` (only the model-free static stage of `agents/evals/stress.py`: parse, eligibility, bindings, loader, prompt sizes and the stand-in renders), `--repeats N`, `--budget-usd` (stop before a case that could take the estimated list-price cost past it), and `--gcloud-token` (mint the developer's renderer token with the gcloud CLI and renew it before its one-hour expiry). Before the first case it renders each library's catalogue file once through the configured backend, without a model call, and stops with exit code 2 when that fails; during the run a pipeline that ended in `RendererUnavailable` (the `error` class on its `pipeline_result` attribution line), or three errors in a row, stop it with exit code 4, so an outage never becomes a matrix of failed runs or a baseline. Per case it records the status and reason, attempts, gate failures by gate, validator rejections, edit-apply failures, the reviewer verdict, LLM calls, `model_version`, tokens (prompt, candidates, thoughts, cached, cache writes, the judge's input and output), cost at list price, the time to the first event, the end-to-end time and the render time, and, since report schema 2, the stage that stopped each attempt and the run (for example `adapter_truncated`, `edit_apply` or `reviewer_defects`, never a bare `validation`), the shipped and the last reviewed attempt, each attempt's adapter outcome, finish reason and plan shape, the edit-failure kinds, every model call's finish reason and tokens, the answers that failed their schema, by agent kind and outcome (`answer_outcomes`: `repaired` or `refused`) and by broken rule (`answer_rules`), the edits each tolerant matching tier applied (`edit_tolerant`), and the module labels of banned imports (`banned_imports`). It reads them from the stream and from the content-free attribution lines that the pipeline, the Budget plugin and the judges write per request id. It writes `/-.json`, a Markdown summary, and a review gallery of 30 renders sampled at random from the runs that passed, with an accept and reject pair each and a JSON export, and prints a diff table against `agents/evals/baselines/.json` (or `--baseline`) over the cases both reports hold: pass rate, accept-match rate (cases whose outcome matches the case's expected outcome), cost per successful plot, p50 and p95 latency, and every case that flipped. A run passes when it ships a render that passed every deterministic gate (no padded canvas, no ADAPTATION finding); every rate counts harness errors as runs that did not pass. The owner's judgement is the acceptance measure: to compare two arms, `python -m agents.evals.report blind` merges both reports into one gallery of the same cases with the arm hidden in a separate key file, and `report score` gives the acceptance per arm. The exit code is 1 when the pass rate on the shared cases drops below the baseline minus a tolerance (5 percentage points by default) or a run crashed in the harness, 2 for a setup error, 3 when the budget stopped the run, and 4 for an outage. - **Baselines.** `agents/evals/baselines/claude-haiku-5-5.json` is committed with the default model: the thinking run of 2026-10-11 on main at `e2e25ef61` (122 cases with one run each, root thinking 1,024 and adapter thinking 2,048, through the deployed renderer; 122 passed, 62 `ok`), which replaced the thinking-off rerun of spike X of 2026-10-10 (244 runs). `gemini-3.8-flash.json` follows with the Gemini arm, so the price and quality comparison is a diff between two reports. `--save-baseline` refuses a run that stopped early, holds promoted cases, or has a run that ended in an error. A model bump is a PR that changes the `AGENT_MODEL` default and adds the new baseline file; a unit test asserts that a baseline exists for the pinned model once any baseline is committed. The same mechanism covers judge-model bumps and ADK upgrades, because the ADK version is part of the report stamp. - **CI.** `.github/workflows/agents-eval.yml` has `workflow_dispatch` inputs `provider`, `model`, `judge_model`, `location`, `cases` (smoke or full) and `repeats`, and a nightly `smoke` run on the pinned model. It authenticates through Workload Identity Federation, sets `GOOGLE_CLOUD_LOCATION=eu`, uploads the report as an artifact and writes the diff table into the job summary. It mirrors the candidate-versus-baseline pattern of `review-retest.yml` (see [Review retest](../workflows/review-retest.md)). A lifecycle probe in the nightly run calls `models.get` on the pinned model and fails loudly on a 404 or a retirement notice, so a forced bump of the pinned model, or of the Gemini arm's short-term 3.8 Flash tier, is noticed before users are. -- **Cost.** A full run is 123 cases times at most 2 attempts. Measured on 2026-10-10 at list price for the full matrix: $0.52 on the Claude Haiku 5.5 arm ($0.0051 per passed plot, median $0.0041, 4.77 model calls per run) and $15.89 on the Gemini 3.8 Flash arm ($0.169 per passed plot, median $0.124, 4.80 model calls per run, $3.99 of it on failed runs); `smoke` (12 cases) is about a tenth of a full run. Both fit inside the spend cap. +- **Cost.** A full run is 127 cases (123 until 2026-10-15) times at most 2 attempts. Measured on 2026-10-10 at list price for the full matrix: $0.52 on the Claude Haiku 5.5 arm ($0.0051 per passed plot, median $0.0041, 4.77 model calls per run) and $15.89 on the Gemini 3.8 Flash arm ($0.169 per passed plot, median $0.124, 4.80 model calls per run, $3.99 of it on failed runs); `smoke` (12 cases) is about a tenth of a full run. Both fit inside the spend cap. ## Serving and infrastructure diff --git a/tests/unit/agents/evals/test_cases.py b/tests/unit/agents/evals/test_cases.py index 1d7595cbb39..924ce49e868 100644 --- a/tests/unit/agents/evals/test_cases.py +++ b/tests/unit/agents/evals/test_cases.py @@ -30,19 +30,27 @@ def test_every_committed_case_loads() -> None: cases = load_cases(promoted_dir=Path("/nonexistent")) stress = [case for case in cases if "stress" in case.tags] - assert len(cases) - len(stress) == 123 # 120 generated plus the three hand-written cases + assert len(cases) - len(stress) == 127 # 120 generated plus the seven hand-written cases assert len(stress) == 23 and all(case.case_id.startswith("stress-") for case in stress) assert [case.case_id for case in cases] == sorted(case.case_id for case in cases) assert {case.origin for case in cases} == {"fixtures"} hand_written = {case.case_id: case for case in cases if case.perturbation is None and "stress" not in case.tags} + chosen = { + "bar-grouped-matplotlib-many-categories-de": "de", + "bar-basic-seaborn-decimals-de": "de", + "bar-basic-seaborn-decimals-en": "en", + "line-multi-matplotlib-millions-de": "de", + } assert set(hand_written) == { "scatter-basic-matplotlib", "bar-grouped-seaborn", "scatter-basic-matplotlib-german-message", + *chosen, } assert hand_written["bar-grouped-seaborn"].locale == "de" - # No committed case chooses a language: the German case tests detection by its first typed message. - assert all(case.language is None for case in cases) + # Only the block H cases (density and number formats) choose a language; the German message case + # tests detection by its first typed message. + assert {case.case_id: case.language for case in cases if case.language is not None} == chosen def test_the_german_message_case_sets_the_language_by_its_first_typed_message() -> None: @@ -66,7 +74,7 @@ def test_selection_by_smoke_full_and_globs() -> None: smoke = select_cases(cases, "smoke") assert len(smoke) == 12 and all("smoke" in case.tags for case in smoke) full = select_cases(cases, "full") - assert len(full) == 123 and full == [case for case in cases if "stress" not in case.tags] + assert len(full) == 127 and full == [case for case in cases if "stress" not in case.tags] stress = select_cases(cases, "stress") assert len(stress) == 23 and all("stress" in case.tags for case in stress) assert len(select_cases(cases, "stress-compute-*")) == 3 diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py new file mode 100644 index 00000000000..859a28ea56a --- /dev/null +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -0,0 +1,299 @@ +"""G11 (per-mark labels and decorations past what the marks leave room for) and G12 (the plot language's decimal mark). + +The fixtures generalise two findings of the live tests of 2026-10-14/15 (block H): + +- Q3: bar-grouped on 26 cantons times two series kept the catalogue's per-bar value labels, + dots on the bar tops and drop shadows, styled for its 12 example bars, on all 52 bars; +- Q4: German sessions showed "6.0 M" on an axis (the user asked for "6,5 Mio.") and seaborn + bar labels "5.30" instead of "5,30". + +The density report runs in-process on real matplotlib figures (the harness's own code); the +label lists stand for the probe's `labels` field (`{"kind", "text"}` per drawn text). +""" + +import json +from typing import Any + +import matplotlib.patheffects as patheffects +import matplotlib.pyplot as plt +import numpy as np +import pytest +from matplotlib.figure import Figure +from matplotlib.ticker import FuncFormatter + +from agents.anyplot.language import NUMBER_FORMATS, number_format, number_format_table +from agents.anyplot.policy import adapter_instruction, adapter_rubric +from agents.anyplot.render import harness +from agents.anyplot.render.backends.fake import fixture_png +from agents.anyplot.render.contract import RenderResult, ThemeOutput +from agents.anyplot.render.gates import ( + MAX_VALUE_LABELS, + _density_lines, + _number_format_lines, + evaluate, + foreign_decimal, +) +from core.defects import DEFECT_RE + + +GERMAN = NUMBER_FORMATS["de"] +ENGLISH = NUMBER_FORMATS["en"] + + +def gate_lines(probe: dict[str, Any], plot_language: str | None = None) -> list[str]: + output = ThemeOutput("light", 0, png=fixture_png("light", (3200, 1800)), probe=probe) + report = evaluate( + RenderResult("p", {"light": output}), + themes=("light",), + library="matplotlib", + rows=100, + plot_language=plot_language, + ) + return report.advisory + + +def grouped_bars(categories: int, *, labels: bool = True, dots: bool = True, shadows: bool = True) -> plt.Figure: + """The catalogue's bar-grouped style (value labels, dots on the bar tops, shadows) on `categories` x 2 bars.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + x = np.arange(categories) + values = np.random.default_rng(7).integers(500, 90_000, size=(2, categories)) + for offset, series in zip((-0.2, 0.2), values, strict=True): + container = ax.bar(x + offset, series, 0.4) + for rect in container: + if shadows: + rect.set_path_effects([patheffects.withSimplePatchShadow(offset=(4, -4))]) + top = rect.get_height() + centre = rect.get_x() + rect.get_width() / 2 + if dots: + ax.scatter(centre, top, s=34) + if labels: + ax.annotate(f"{int(top)}", xy=(centre, top), xytext=(0, 6), textcoords="offset points", ha="center") + ax.set_xticks(x, [f"Region {index}" for index in range(categories)], rotation=90) + return fig + + +@pytest.fixture(autouse=True) +def _close_figures() -> Any: + yield + plt.close("all") + + +class TestDensityReport: + def test_the_live_case_52_bars_with_labels_dots_and_shadows(self) -> None: + fig = grouped_bars(26) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 52 and axes["labelled"] == 52 + assert axes["decorated"] == 104 # 52 dots on the bar tops and 52 shadowed bars + assert axes["marks"] == 52 # the dots are decorations, not data marks + assert axes["label_px"] > axes["slot_px"] + [line] = _density_lines({"density": [axes]}) + assert line[0] == "G11" + + def test_the_catalogue_scale_is_clean(self) -> None: + """The catalogue's own 4 categories x 3 series with 3-digit labels: what the style was made for.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + x = np.arange(4) + for index, series in enumerate(([245, 312, 287, 425], [178, 195, 285, 310], [125, 210, 195, 165])): + container = ax.bar(x + (index - 1) * 0.25, series, 0.25) + ax.bar_label(container, fmt="%d", fontsize=9) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 12 and axes["labelled"] == 12 and axes["decorated"] == 0 + assert _density_lines({"density": [axes]}) == [] + + def test_heatmap_cell_annotations_are_no_value_labels(self) -> None: + """Cell text sits on no bar or point: the rubric allows it (quality-criteria, cell annotations).""" + fig, ax = plt.subplots(figsize=(6, 6), dpi=400) + ax.imshow(np.ones((20, 20))) + for row in range(20): + for column in range(20): + ax.text(column, row, "1", ha="center", va="center") + fig.canvas.draw() + + assert harness.density_report(fig, 400) == [] + + def test_a_label_on_every_point_of_a_long_line(self) -> None: + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + values = np.arange(60) ** 1.1 + ax.plot(np.arange(60), values, marker="o") + for index, value in enumerate(values): + ax.text(index, value + 1, f"{value:.0f}", ha="center") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 0 and axes["labelled"] == 60 + [(gate, line)] = _density_lines({"density": [axes]}) + assert "60 marks carry 60 value labels" in line + + def test_one_callout_in_axes_coordinates_is_no_value_label(self) -> None: + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.bar(range(40), range(1, 41)) + ax.annotate("Peak", xy=(0.5, 0.9), xycoords="axes fraction") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 0 and _density_lines({"density": [axes]}) == [] + + def test_the_probe_writes_the_density_field(self, tmp_path: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Figure, "savefig", Figure.savefig) # restored after the test + harness.install_probe("light") + fig = grouped_bars(26) + fig.savefig("plot-light.png", dpi=100) + + probe = json.loads((tmp_path / "probe-light.json").read_text()) + assert probe["density"][0]["bars"] == 52 + + +class TestDensityGate: + def test_the_line_is_a_measured_vq03_defect_line(self) -> None: + probe = { + "density": [{"bars": 52, "marks": 52, "labelled": 52, "decorated": 104, "slot_px": 35.0, "label_px": 160.0}] + } + + [line] = gate_lines(probe) + + assert DEFECT_RE.match(line), line + assert line.startswith("VQ-03 (light): 52 bars carry 52 value labels (160 px wide on 35 px per bar)") + assert "104 dots or shadows" in line and f"at most {MAX_VALUE_LABELS} labels" in line + assert "Likely cause:" in line + + @pytest.mark.parametrize( + "entry", + [ + {"bars": "52", "marks": 52, "labelled": 52}, + {"bars": True, "marks": 52, "labelled": 99}, + {"bars": 52, "marks": 52, "labelled": 10, "slot_px": float("nan"), "label_px": 1e9}, + "not an entry", + ], + ) + def test_a_tampered_entry_yields_no_line(self, entry: Any) -> None: + assert gate_lines({"density": [entry]}) == [] + + def test_a_malformed_density_never_hides_another_gate(self) -> None: + lines = gate_lines({"density": "broken", "tick_overlaps": 3}) + + assert any(line.startswith("VQ-02") for line in lines) + + def test_a_few_cramped_labels_are_callouts(self) -> None: + entry = {"bars": 4, "marks": 4, "labelled": 3, "decorated": 0, "slot_px": 20.0, "label_px": 120.0} + + assert _density_lines({"density": [entry]}) == [] + + +class TestForeignDecimal: + @pytest.mark.parametrize("text", ["5.30", "6.0 M", "0.25", "−1.5", "1,234.5", "12.5 %", "$ 3.75", "1,234,567"]) + def test_english_numbers_in_a_german_plot(self, text: str) -> None: + assert foreign_decimal(text, GERMAN) + + @pytest.mark.parametrize( + "text", + [ + "5,30", + "6,5 Mio.", + "1.234", # a thousands mark + "7.184.003", + "1.234,5", + "2024", + "14.10.", # a date, never a number + "2024-10-14", + "Q3", + "3.11.1 Version", + "1e6", + "", + ], + ) + def test_german_numbers_and_non_numbers_pass(self, text: str) -> None: + assert not foreign_decimal(text, GERMAN) + + @pytest.mark.parametrize("text", ["5,30", "6,5 M", "1.234,5"]) + def test_german_numbers_in_an_english_plot(self, text: str) -> None: + assert foreign_decimal(text, ENGLISH) + + @pytest.mark.parametrize("text", ["5.30", "6.5 M", "1,234", "1,234,567", "1,234.5", "7.184.003", "2024"]) + def test_english_numbers_pass_in_english(self, text: str) -> None: + assert not foreign_decimal(text, ENGLISH) + + +def seaborn_style_bar_labels(fmt_label: Any, language: str) -> list[dict[str, str]]: + """The live seaborn case: bar labels and y tick labels as the probe's `labels` field lists them.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(["A", "B", "C", "D"], [5.3, 4.25, 3.1, 2.05]) + ax.bar_label(container, labels=[fmt_label(value) for value in container.datavalues]) + ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: fmt_label(value))) + fig.canvas.draw() + labels = [{"kind": "annotation", "text": text.get_text()} for text in ax.texts] + labels += [{"kind": "tick label", "text": tick.get_text()} for tick in ax.get_yticklabels() if tick.get_text()] + labels += [{"kind": "tick label", "text": tick.get_text()} for tick in ax.get_xticklabels()] + labels.append({"kind": "title", "text": f"Version 2.5 ({language})"}) + return labels + + +class TestNumberFormatGate: + def test_the_live_german_case_is_a_vq06_line(self) -> None: + labels = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "de") + + [line] = gate_lines({"labels": labels}, plot_language="de") + + assert DEFECT_RE.match(line), line + assert line.startswith("VQ-06 (light): ") + assert "use a decimal point; German writes a decimal comma" in line + assert "Mio." in line and "annotations and tick labels" in line + assert "5.30" not in line # no label text reaches the line + + def test_the_fixed_german_case_is_clean(self) -> None: + german = str.maketrans(",.", ".,") + labels = seaborn_style_bar_labels(lambda value: f"{value:,.2f}".translate(german), "de") + + assert gate_lines({"labels": labels}, plot_language="Deutsch") == [] + + def test_the_english_control_is_clean_and_german_marks_in_english_are_named(self) -> None: + english = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "en") + german = seaborn_style_bar_labels(lambda value: f"{value:.2f}".replace(".", ","), "en") + + assert gate_lines({"labels": english}, plot_language="en") == [] + [line] = gate_lines({"labels": german}, plot_language="en") + assert "English writes a decimal point" in line + + def test_a_language_without_a_number_format_and_no_language_have_no_gate(self) -> None: + labels = [{"kind": "tick label", "text": "5.30"}] + + assert _number_format_lines({"labels": labels}, number_format("Swedish"), "Swedish") == [] + assert gate_lines({"labels": labels}) == [] + + def test_titles_and_legend_entries_are_never_checked(self) -> None: + labels = [{"kind": "title", "text": "2.5"}, {"kind": "legend text", "text": "1.5"}] + + assert gate_lines({"labels": labels}, plot_language="de") == [] + + @pytest.mark.parametrize("labels", ["broken", [None, {"kind": "tick label", "text": 5.3}], [{"text": "5.30"}]]) + def test_a_tampered_labels_field_yields_no_line(self, labels: Any) -> None: + assert gate_lines({"labels": labels}, plot_language="de") == [] + + +class TestNumberFormatTable: + def test_every_named_language_has_marks_and_words(self) -> None: + for code, spec in NUMBER_FORMATS.items(): + assert spec.decimal in ".," and spec.decimal != spec.thousands, code + assert spec.suffixes and all(word for _, word in spec.suffixes), code + + def test_names_and_codes_share_a_format(self) -> None: + assert number_format("German") is number_format("de") is number_format("Deutsch") is GERMAN + assert number_format("Klingon") is None + + def test_the_adapter_reads_the_table_and_the_rules(self) -> None: + instruction = adapter_instruction("matplotlib") + + assert number_format_table() in instruction + assert "| `de` (German) | `,` | `.` | 10^3: Tsd., 10^6: Mio., 10^9: Mrd. |" in instruction + assert "**Numbers.**" in instruction and "**Per-mark labels and decorations.**" in instruction + + def test_the_adapter_reads_the_narrowed_house_style_rule(self) -> None: + assert "not those drawn once per bar or point" in adapter_rubric() From 2db63c1f9f10964bb1540d5411f22ebb8990ec25 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:56:30 +0200 Subject: [PATCH 2/8] docs(changelog): fragment for the density and number-format gates Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- changelog.d/agent-density-locale-gates.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog.d/agent-density-locale-gates.md diff --git a/changelog.d/agent-density-locale-gates.md b/changelog.d/agent-density-locale-gates.md new file mode 100644 index 00000000000..2e7ef45c3a8 --- /dev/null +++ b/changelog.d/agent-density-locale-gates.md @@ -0,0 +1,13 @@ +### Added + +- **Agent chat: gates for crowded value labels and the plot language's + number format.** G11 counts the value labels, dots and shadows the + catalogue draws once per bar or point and writes a measured VQ-03 line + when more than 25 crowd a plot, or bar labels are wider than the space + per bar (the live case: 52 labelled, dotted, shadowed bars of 26 + regions). G12 writes a VQ-06 line when a numeric tick label or annotation + uses the other decimal mark than the plot language (`5.30` and `6.0 M` + in a German plot). A server-side table of decimal and thousands marks and + large-number words per language (`de`: `5,30`, `6,5 Mio.`) goes into the + adapter's instruction, and the house-style exemption of the reviewer and + scorer no longer covers elements whose count grows with the data. From b13ab36751e5a3abb9d68b990ba3b83ee343bc6e Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:34:10 +0200 Subject: [PATCH 3/8] fix(agents): G12 reads block F's tick label kinds `harness.drawn_labels` (#12183) names tick labels by axis (`x tick label`, `y tick label`, `z tick label`, `colorbar tick label`); G12 now checks every kind ending in "tick label" plus annotations and texts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/render/gates.py | 19 +++++++++++++++---- .../runtime/test_density_locale_gates.py | 15 +++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 3eb59af1119..3c0a8ce37fb 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -533,8 +533,18 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: ) """A label that is one number: an optional currency sign and sign, digits with marks, an optional unit or word (`%`, `Mio.`, `k`, `mil M`). Dates (`14.10.`, `2024-10-14`), ranges, times and words with digits never match.""" -NUMBER_LABEL_KINDS = frozenset({"tick label", "annotation", "text"}) -"""The label kinds whose numbers G12 checks; titles, axis titles and legend entries may name a version or a code.""" +NUMBER_LABEL_KINDS = frozenset({"annotation", "text"}) +"""The label kinds whose numbers G12 checks besides every tick label (`x tick label`, `y tick label`, `z tick label`, +`colorbar tick label`, `harness.drawn_labels`); titles, axis titles and legend entries may name a version or a code.""" + + +def _number_label_kind(kind: Any) -> str | None: + """`tick labels` or `annotations` for a label kind G12 checks, else None.""" + if not isinstance(kind, str): + return None + if kind.endswith("tick label"): + return "tick labels" + return "annotations" if kind in NUMBER_LABEL_KINDS else None def foreign_decimal(text: str, numbers: NumberFormat) -> bool: @@ -580,7 +590,8 @@ def _number_format_lines( numeric = foreign = 0 kinds: set[str] = set() for label in labels: - if not isinstance(label, dict) or label.get("kind") not in NUMBER_LABEL_KINDS: + kind = _number_label_kind(label.get("kind")) if isinstance(label, dict) else None + if kind is None: continue text = label.get("text") if not isinstance(text, str) or _NUMBER_LABEL.match(text.strip()) is None: @@ -588,7 +599,7 @@ def _number_format_lines( numeric += 1 if foreign_decimal(text, numbers): foreign += 1 - kinds.add("tick labels" if label["kind"] == "tick label" else "annotations") + kinds.add(kind) if not foreign: return [] name = english_name(plot_language or "") or "the plot language" diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py index 859a28ea56a..f081a98d9a9 100644 --- a/tests/unit/agents/runtime/test_density_locale_gates.py +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -230,8 +230,8 @@ def seaborn_style_bar_labels(fmt_label: Any, language: str) -> list[dict[str, st ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: fmt_label(value))) fig.canvas.draw() labels = [{"kind": "annotation", "text": text.get_text()} for text in ax.texts] - labels += [{"kind": "tick label", "text": tick.get_text()} for tick in ax.get_yticklabels() if tick.get_text()] - labels += [{"kind": "tick label", "text": tick.get_text()} for tick in ax.get_xticklabels()] + labels += [{"kind": "y tick label", "text": tick.get_text()} for tick in ax.get_yticklabels() if tick.get_text()] + labels += [{"kind": "x tick label", "text": tick.get_text()} for tick in ax.get_xticklabels()] labels.append({"kind": "title", "text": f"Version 2.5 ({language})"}) return labels @@ -263,7 +263,7 @@ def test_the_english_control_is_clean_and_german_marks_in_english_are_named(self assert "English writes a decimal point" in line def test_a_language_without_a_number_format_and_no_language_have_no_gate(self) -> None: - labels = [{"kind": "tick label", "text": "5.30"}] + labels = [{"kind": "y tick label", "text": "5.30"}] assert _number_format_lines({"labels": labels}, number_format("Swedish"), "Swedish") == [] assert gate_lines({"labels": labels}) == [] @@ -273,7 +273,14 @@ def test_titles_and_legend_entries_are_never_checked(self) -> None: assert gate_lines({"labels": labels}, plot_language="de") == [] - @pytest.mark.parametrize("labels", ["broken", [None, {"kind": "tick label", "text": 5.3}], [{"text": "5.30"}]]) + def test_colorbar_and_3d_tick_labels_count(self) -> None: + labels = [{"kind": "colorbar tick label", "text": "0.25"}, {"kind": "z tick label", "text": "1.5"}] + + [line] = gate_lines({"labels": labels}, plot_language="de") + + assert line.startswith("VQ-06 (light): 2 of 2 numeric tick labels use a decimal point") + + @pytest.mark.parametrize("labels", ["broken", [None, {"kind": "x tick label", "text": 5.3}], [{"text": "5.30"}]]) def test_a_tampered_labels_field_yields_no_line(self, labels: Any) -> None: assert gate_lines({"labels": labels}, plot_language="de") == [] From dd2756d6685568d23b3f08e5275948f6e6490dbe Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:34:48 +0200 Subject: [PATCH 4/8] docs(changelog): reference #12184 in the fragment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- changelog.d/agent-density-locale-gates.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.d/agent-density-locale-gates.md b/changelog.d/agent-density-locale-gates.md index 2e7ef45c3a8..18686aeaaa5 100644 --- a/changelog.d/agent-density-locale-gates.md +++ b/changelog.d/agent-density-locale-gates.md @@ -10,4 +10,5 @@ in a German plot). A server-side table of decimal and thousands marks and large-number words per language (`de`: `5,30`, `6,5 Mio.`) goes into the adapter's instruction, and the house-style exemption of the reviewer and - scorer no longer covers elements whose count grows with the data. + scorer no longer covers elements whose count grows with the data + (#12184). From 761144b0470e6ab7fab7b7cfed962927bd4bf75e Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:57:16 +0200 Subject: [PATCH 5/8] fix(agents): Copilot round 1 on the density and number-format gates - density_report: bars, points and annotation anchors outside the axes view (a zoomed plot) and non-finite bar geometry (a NaN value) are left out, so they neither count nor void the probe's JSON; scatter marks are matched to bar ends in chunks of 2,000, so the pairwise arrays stay bounded however many marks a plot draws. - New probe field `tick_axes` (harness.tick_axes): per axes, whether each axis's tick labels are numbers, dates, categories or labels the code set itself. G12 checks tick labels of numeric axes only, so a German `%d.%m` date (14.10) or a category the data names (3.11) is never a finding; without the field only annotations count. - G12 reads a mathtext label as drawn (`$\mathdefault{5.30}$` is 5.30). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/render/gates.py | 54 +++++-- agents/anyplot/render/harness.py | 141 ++++++++++++++--- docs/concepts/agent-network.md | 2 +- .../runtime/test_density_locale_gates.py | 148 +++++++++++++++--- 4 files changed, 287 insertions(+), 58 deletions(-) diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 3c0a8ce37fb..7d65074cc5f 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -12,7 +12,7 @@ | G9 | probe: scatter marks plus line vertices over `MAX_PLOTTED_POINTS` (computed mass data) | advisory: a DQ-03 line | | G10 | probe: characters without a glyph in any font of their text, plain text and mathtext (the harness records matplotlib's own reports) | advisory: a VQ-01 line | | G11 | probe (`density`): value labels at a data coordinate on a bar's end or point, dots on bar ends and shadowed bars, per axes: more than `MAX_VALUE_LABELS` labels, bar labels wider than `LABEL_SLOT_FILL` of the space per bar, or more than `MAX_DECORATIONS` decorations | advisory: a VQ-03 line | -| G12 | probe (`labels`): numeric tick labels and annotations whose decimal mark is not the plot language's (`language.NUMBER_FORMATS`; `foreign_decimal`) | advisory: a VQ-06 line | +| G12 | probe (`labels`, `tick_axes`): numeric annotations, and tick labels of numeric axes (never dates or categories), whose decimal mark is not the plot language's (`language.NUMBER_FORMATS`; `foreign_decimal`, mathtext read as drawn) | advisory: a VQ-06 line | A pipeline run renders one theme (`PipelineArgs.theme`) and the theme toggle renders the other one later, so `evaluate` judges exactly the themes of the job it is given: @@ -538,13 +538,44 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: `colorbar tick label`, `harness.drawn_labels`); titles, axis titles and legend entries may name a version or a code.""" -def _number_label_kind(kind: Any) -> str | None: - """`tick labels` or `annotations` for a label kind G12 checks, else None.""" +NUMERIC_TICKS = "number" +"""`harness.tick_axes`'s kind for an axis whose tick labels are numbers; dates, categories and labels the code set +itself (`fixed`) are never checked.""" +_MATH_MARKUP = re.compile(r"\\mathdefault|\\mathrm|\\,|[{}$]") + + +def plain_text(text: str) -> str: + """A label as it reads on the canvas when it is one mathtext string: `$\\mathdefault{5.30}$` reads `5.30` and + `$5{,}30$` reads `5,30`. Any other text stays as it is, a currency dollar included.""" + stripped = text.strip() + if len(stripped) > 2 and stripped.startswith("$") and stripped.endswith("$") and stripped.count("$") == 2: + return _MATH_MARKUP.sub("", stripped).replace("\\ ", " ").strip() + return stripped + + +def _number_label_kind(label: dict[str, Any], ticks: list[Any] | None) -> str | None: + """`tick labels` or `annotations` for a label G12 checks, else None. + + An annotation or text always counts. A tick label counts only when the probe's + `tick_axes` (`harness.tick_axes`) names its axis numeric: an axis of dates (`14.10`) or + of the data's own categories (`3.11`) writes no decimals. Without `tick_axes` (a renderer + older than G12) no tick label counts. + """ + kind = label.get("kind") if not isinstance(kind, str): return None - if kind.endswith("tick label"): - return "tick labels" - return "annotations" if kind in NUMBER_LABEL_KINDS else None + if kind in NUMBER_LABEL_KINDS: + return "annotations" + if not kind.endswith("tick label") or ticks is None: + return None + index = label.get("axes") + if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(ticks): + return None + entry = ticks[index] + if not isinstance(entry, dict): + return None + names = ("x", "y") if kind == "colorbar tick label" else (kind[0],) + return "tick labels" if any(entry.get(name) == NUMERIC_TICKS for name in names) else None def foreign_decimal(text: str, numbers: NumberFormat) -> bool: @@ -556,7 +587,7 @@ def foreign_decimal(text: str, numbers: NumberFormat) -> bool: digits (`1.234`) is a thousands mark and passes. A decimal point mirrors this, except that several points (`7.184.003`) pass, because a version-like label writes them too. """ - match = _NUMBER_LABEL.match(text.strip()) + match = _NUMBER_LABEL.match(plain_text(text)) if match is None: return False body = match.group("body") @@ -580,21 +611,24 @@ def _number_format_lines( ) -> list[tuple[str, str]]: """The G12 line: numeric tick labels or annotations whose decimal mark is not the plot language's. - Reads the probe's `labels` field (`{"kind", "text"}` per drawn text). No label text + Reads the probe's `labels` field (`{"kind", "text", "axes"}` per drawn text) and its + `tick_axes`, which say which tick labels are numbers (`_number_label_kind`). No label text reaches the line: a tick label can be a cell of the user's data. The language is named by its English name from the server's table, never by the session's free-text value. """ labels = probe.get("labels") if isinstance(probe, dict) else None if numbers is None or not isinstance(labels, list): return [] + ticks = probe.get("tick_axes") if isinstance(probe, dict) else None + ticks = ticks if isinstance(ticks, list) else None numeric = foreign = 0 kinds: set[str] = set() for label in labels: - kind = _number_label_kind(label.get("kind")) if isinstance(label, dict) else None + kind = _number_label_kind(label, ticks) if isinstance(label, dict) else None if kind is None: continue text = label.get("text") - if not isinstance(text, str) or _NUMBER_LABEL.match(text.strip()) is None: + if not isinstance(text, str) or _NUMBER_LABEL.match(plain_text(text)) is None: continue numeric += 1 if foreign_decimal(text, numbers): diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 0a02f3b9a79..a4311ff24d2 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -106,7 +106,8 @@ plotted points than any dataset within the parser's caps can hold) and G10 (characters without a glyph in any font of the text); the `density` field each save adds after the probe (`density_report`: per axes, the bars and points and the value labels and decorations on them) -feeds G11. It is +feeds G11, and `tick_axes` (what each axis's tick labels write: numbers, dates, categories) tells G12 +which tick labels are numbers. It is advisory because the code under test could tamper with it; the host never fails a render on it. The same holds for the `HARNESS` lines: the code under test shares stdout, so the host reads them as hints (a retry decision, a memory figure), never @@ -1201,12 +1202,19 @@ def polar_report(figure): return report -def _bar_ends(axes): - """Every visible bar of the axes as `(centre, end, width)` in display pixels along its orientation, plus a flag - whether it draws a path effect (a shadow), and the orientation of the first bar container. +def _in_view(x, y, box, slack=1.0): + """Whether a display point lies within the axes box (a mark or anchor outside it is clipped, never drawn).""" + return box.x0 - slack <= x <= box.x1 + slack and box.y0 - slack <= y <= box.y1 + slack + + +def _bar_ends(axes, box): + """Every visible bar of the axes inside its view as `(centre, end, width)` in display pixels along its + orientation, plus a flag whether it draws a path effect (a shadow), and the orientation of the first bar container. For a vertical bar the centre is its x centre, the end the y of the side away from the - baseline and the width its pixel width; a horizontal bar swaps the axes. + baseline and the width its pixel width; a horizontal bar swaps the axes. A bar with a + missing value (NaN geometry) or a centre outside the axes box (a zoomed view) is left + out: it draws nothing, and a NaN would void the probe's JSON. """ bars = [] orientation = None @@ -1218,18 +1226,23 @@ def _bar_ends(axes): for patch in container.patches: if not patch.get_visible(): continue - box = patch.get_window_extent() + extent = patch.get_window_extent() negative = (patch.get_width() if horizontal else patch.get_height()) < 0 if horizontal: - bar = ((box.y0 + box.y1) / 2, box.x0 if negative else box.x1, box.y1 - box.y0) + bar = ((extent.y0 + extent.y1) / 2, extent.x0 if negative else extent.x1, extent.y1 - extent.y0) + centre = ((extent.x0 + extent.x1) / 2, bar[0]) else: - bar = ((box.x0 + box.x1) / 2, box.y0 if negative else box.y1, box.x1 - box.x0) + bar = ((extent.x0 + extent.x1) / 2, extent.y0 if negative else extent.y1, extent.x1 - extent.x0) + centre = (bar[0], (extent.y0 + extent.y1) / 2) + if not all(math.isfinite(value) for value in bar) or not _in_view(*centre, box): + continue bars.append((*bar, bool(patch.get_path_effects()))) return bars, orientation -def _point_marks(axes): - """The display positions of the axes' visible scatter marks and of the vertices of its lines of three or more.""" +def _point_marks(axes, box): + """The display positions of the axes' visible scatter marks and of the vertices of its lines of three or more, + finite and inside the axes box only.""" import numpy as np scatter, vertices = [], [] @@ -1243,15 +1256,29 @@ def _point_marks(axes): if line.get_visible() and len(xy) >= 3: # two-vertex lines are rules, brackets and reference lines vertices.append(line.get_transform().transform(xy)) empty = np.empty((0, 2)) - return (np.concatenate(scatter) if scatter else empty), (np.concatenate(vertices) if vertices else empty) + + def kept(parts): + points = np.concatenate(parts) if parts else empty + inside = ( + np.isfinite(points).all(axis=1) + & (points[:, 0] >= box.x0 - 1) + & (points[:, 0] <= box.x1 + 1) + & (points[:, 1] >= box.y0 - 1) + & (points[:, 1] <= box.y1 + 1) + ) + return points[inside] + + return kept(scatter), kept(vertices) -def _label_anchors(axes): +def _label_anchors(axes, box): """The visible texts of the axes placed at a data coordinate, with their anchors in display pixels. An annotation counts when its `xycoords` is "data" (`ax.annotate`, `ax.bar_label`), any other text when it uses the data transform (`ax.text` at a data position). A callout in - axes coordinates is no value label. + axes coordinates is no value label. Like matplotlib's own draw, an annotation whose point + lies outside the axes is skipped unless it is set to draw there (`annotation_clip=False`); + a plain text at an infinite position is skipped too. """ found = [] for text in axes.texts: @@ -1261,16 +1288,38 @@ def _label_anchors(axes): if text.xycoords != "data": continue anchor = axes.transData.transform(text.xy) + clipped = text.get_annotation_clip() is not False elif text.get_transform() == axes.transData: anchor = axes.transData.transform(text.get_position()) + clipped = False else: continue - found.append((text, float(anchor[0]), float(anchor[1]))) + x, y = float(anchor[0]), float(anchor[1]) + if not (math.isfinite(x) and math.isfinite(y)) or (clipped and not _in_view(x, y, box)): + continue + found.append((text, x, y)) if len(found) >= DENSITY_LABEL_LIMIT: break return found +def _on_bar_ends(scatter, bar_array, horizontal, value_extent): + """Which scatter marks sit on a bar's end, matched in chunks of `DENSITY_MARK_LIMIT` marks so the pairwise + arrays stay at most `DENSITY_MARK_LIMIT` squared, however many marks the plot draws.""" + import numpy as np + + on_bar = np.zeros(len(scatter), dtype=bool) + if not len(bar_array) or not len(scatter): + return on_bar + for start in range(0, len(scatter), DENSITY_MARK_LIMIT): + chunk = scatter[start : start + DENSITY_MARK_LIMIT] + along, across = (chunk[:, 1], chunk[:, 0]) if horizontal else (chunk[:, 0], chunk[:, 1]) + near_centre = np.abs(along[:, None] - bar_array[None, :, 0]) <= bar_array[None, :, 2] / 2 + 1 + near_end = np.abs(across[:, None] - bar_array[None, :, 1]) <= 0.03 * value_extent + on_bar[start : start + len(chunk)] = (near_centre & near_end).any(axis=1) + return on_bar + + def density_report(figure, dpi): """Per axes (at most `DENSITY_AXES_LIMIT`, only those with bars or point marks): how many marks it draws and how many of them carry a value label or a decoration, with the space a label has. @@ -1292,9 +1341,9 @@ def density_report(figure, dpi): if len(report) >= DENSITY_AXES_LIMIT: break try: - bars, orientation = _bar_ends(axes) - scatter, vertices = _point_marks(axes) box = axes.get_window_extent(renderer) + bars, orientation = _bar_ends(axes, box) + scatter, vertices = _point_marks(axes, box) except Exception: continue if not bars and not len(scatter) and not len(vertices): @@ -1304,15 +1353,10 @@ def density_report(figure, dpi): value_extent = width if horizontal else height bar_array = np.asarray([bar[:3] for bar in bars[:DENSITY_MARK_LIMIT]], dtype=float).reshape(-1, 3) # A scatter mark on a bar's end is a decoration (the catalogue's dots on its bar tops), not a data mark. - on_bar = np.zeros(len(scatter), dtype=bool) - if len(bar_array) and len(scatter): - along, across = (scatter[:, 1], scatter[:, 0]) if horizontal else (scatter[:, 0], scatter[:, 1]) - near_centre = np.abs(along[:, None] - bar_array[None, :, 0]) <= bar_array[None, :, 2] / 2 + 1 - near_end = np.abs(across[:, None] - bar_array[None, :, 1]) <= 0.03 * value_extent - on_bar = (near_centre & near_end).any(axis=1) + on_bar = _on_bar_ends(scatter, bar_array, horizontal, value_extent) points = np.concatenate([scatter[~on_bar], vertices])[:DENSITY_MARK_LIMIT] labelled, label_sizes = 0, [] - for text, x, y in _label_anchors(axes): + for text, x, y in _label_anchors(axes, box): along, across = (y, x) if horizontal else (x, y) on_a_bar = len(bar_array) and bool( ( @@ -1334,7 +1378,9 @@ def density_report(figure, dpi): extent = text.get_window_extent(renderer) except Exception: continue - label_sizes.append(float(extent.height if horizontal else extent.width)) + size = float(extent.height if horizontal else extent.width) + if math.isfinite(size): + label_sizes.append(size) centres = np.unique(np.round(bar_array[:, 0], 1)) if len(bar_array) else np.empty(0) slot = float(np.median(np.diff(centres))) if len(centres) > 1 else None report.append( @@ -1350,6 +1396,49 @@ def density_report(figure, dpi): return report +def _tick_kind(axis): + """What an axis's tick labels write: `number` (a numeric scale and formatter), `date`, `category`, `fixed` + (labels the code set itself, `set_xticks(positions, labels)`: often the data's own categories), `none` (no labels) + or `other`.""" + major = axis.get_major_formatter() + formatter = type(major).__name__ + if formatter == "NullFormatter": + return "none" + # `set_ticklabels` (and `set_xticks(positions, labels)`) installs a FuncFormatter over a dict of the labels. + labels_dict = getattr(getattr(major, "func", None), "func", None) + if formatter == "FixedFormatter" or getattr(labels_dict, "__name__", "") == "_format_with_dict": + return "fixed" + get_converter = getattr(axis, "get_converter", None) + converter = get_converter() if get_converter is not None else getattr(axis, "converter", None) + if converter is None: + return "number" + module = type(converter).__module__ + if "dates" in module: + return "date" + if "category" in module: + return "category" + return "other" + + +def tick_axes(figure): + """Per axes of `figure.get_axes()`, in that order (the `axes` index of `drawn_labels`), what each of its axes' + tick labels write (`_tick_kind`), so the host checks the decimal mark (G12) of numeric ticks only, never of a + date (`14.10`) or a category the data names (`3.11`).""" + report = [] + for axes in figure.get_axes(): + entry = {} + for name in ("x", "y", "z"): + axis = getattr(axes, f"{name}axis", None) + if axis is None: + continue + try: + entry[name] = _tick_kind(axis) + except Exception: + entry[name] = "other" + report.append(entry) + return report + + def install_probe(theme): try: from matplotlib.figure import Figure @@ -1376,6 +1465,10 @@ def savefig(self, *args, **kwargs): density = [] if density: report["density"] = density + try: + report["tick_axes"] = tick_axes(self) + except Exception: # nor does the tick-axes report + pass try: text = json.dumps(report, allow_nan=False) # NaN or Infinity (a text at an infinite coordinate) except (TypeError, ValueError) as exc: diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index a8ea714d0c5..7bc75c08c47 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -314,7 +314,7 @@ Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` wi **Scripts and fonts.** Plot text can be in the scripts most users' data and languages bring: the renderer image installs Noto Sans, Noto Sans CJK and the Noto Sans faces of the scripts DejaVu Sans lacks (37 files, 46.8 MB, Regular and Bold where the family has both), and the harness appends them to every font lookup, so a Chinese title or a Devanagari tick label draws with the right glyphs while the code's own first font stays in charge of Latin text. Right-to-left scripts work: the pinned matplotlib lays text out through libraqm, so Arabic and Hebrew render shaped and in reading order, within one label aligned by the code's `ha`. Three limits remain, and G10 reports the first and the last: a `$…$` math string uses matplotlib's math fonts without fallback and draws `¤` for every character they lack, so non-Latin words belong outside math text; Han characters take the Japanese regional forms of the CJK face until a caller passes the plot's language (`text.language`, a follow-up with the language chip); and a character outside the installed families draws as a Last Resort placeholder box. Most pictographs are not in that last group, because Noto Sans Symbols2 or DejaVu Sans draws them as monochrome outlines (📈, 📊, ⚠, 😀), but 🔥, ✅, the flag letters, Tibetan and Mongolian have no glyph in any installed font. G10 counts those characters deterministically from matplotlib's own reports, names up to three of them with their code points (`U+1F525 🔥 fire`), tells the repair that a user's column name stays as it is, and keeps the line in the result's residual lines even when the reviewer passed the render, because a measured missing glyph is not a judgement. -**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits on a bar's end or a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels), the dots on bar ends and the shadowed bars, and measures the median bar label width against the space between bar centres. G11 writes a VQ-03 line when an axes carries more than `MAX_VALUE_LABELS` (25) value labels, from six labels on when a bar label is wider than 90 % of the space per bar, or more than `MAX_DECORATIONS` (25) dots or shadows. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string. G12 reads the probe's `labels` field (the drawn texts by kind) and writes a VQ-06 line when a numeric tick label or annotation uses the other decimal mark: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`), a point after a comma, or commas grouping thousands; dates, years, version strings, titles and legend entries never count, and no label text reaches the line. +**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits on a bar's end or a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels), the dots on bar ends and the shadowed bars, and measures the median bar label width against the space between bar centres. G11 writes a VQ-03 line when an axes carries more than `MAX_VALUE_LABELS` (25) value labels, from six labels on when a bar label is wider than 90 % of the space per bar, or more than `MAX_DECORATIONS` (25) dots or shadows. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string. G12 reads the probe's `labels` field (the drawn texts by kind, from block F) and its `tick_axes` field (`harness.tick_axes`: whether each axis's tick labels are numbers, dates, categories or labels the code set itself), and writes a VQ-06 line when an annotation, or a tick label of a numeric axis, uses the other decimal mark, mathtext read as drawn: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`), a point after a comma, or commas grouping thousands; dates, years, version strings, titles and legend entries never count, and no label text reaches the line. Later languages reuse the CI commands: `Rscript plot.R`; `julia --project=/opt/julia plot.jl` with a stacked `JULIA_DEPOT_PATH` and a pinned `JULIA_CPU_TARGET`; `node /opt/anyplot/js-render/render.mjs plot.js` with `window.ANYPLOT_DATA` injected; bokeh with `BOKEH_RESOURCES=inline`, `SE_OFFLINE=true` and a shipped chromedriver; plotly with local or disabled MathJax. diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py index f081a98d9a9..2803da3d158 100644 --- a/tests/unit/agents/runtime/test_density_locale_gates.py +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -141,6 +141,43 @@ def test_one_callout_in_axes_coordinates_is_no_value_label(self) -> None: assert axes["labelled"] == 0 and _density_lines({"density": [axes]}) == [] + def test_a_missing_bar_value_keeps_the_report_valid_json(self) -> None: + """Copilot on #12184: a NaN bar (a group missing after a pivot) would void the probe's JSON.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(4), [3.0, float("nan"), 2.0, 1.0]) + ax.bar_label(container) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 3 + json.dumps(axes, allow_nan=False) + + def test_bars_and_labels_outside_a_zoomed_view_are_not_counted(self) -> None: + """Copilot on #12184: 50 labelled bars zoomed to five draw five labels.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(50), range(1, 51)) + ax.bar_label(container) + ax.set_xlim(-0.5, 4.5) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 5 and axes["labelled"] == 5 + + def test_many_scatter_marks_are_matched_in_bounded_chunks(self) -> None: + """Copilot on #12184: the pairwise arrays stay bounded however many marks the plot draws.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(30), range(1, 31)) + rng = np.random.default_rng(3) + ax.scatter(rng.uniform(0, 29, 9_000), rng.uniform(0, 30, 9_000), s=1) + ax.scatter([rect.get_x() + rect.get_width() / 2 for rect in container], range(1, 31), s=30) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["decorated"] >= 30 and axes["marks"] < 9_030 + def test_the_probe_writes_the_density_field(self, tmp_path: Any, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) monkeypatch.setattr(Figure, "savefig", Figure.savefig) # restored after the test @@ -150,6 +187,7 @@ def test_the_probe_writes_the_density_field(self, tmp_path: Any, monkeypatch: py probe = json.loads((tmp_path / "probe-light.json").read_text()) assert probe["density"][0]["bars"] == 52 + assert probe["tick_axes"] == [{"x": "fixed", "y": "number"}] class TestDensityGate: @@ -222,25 +260,34 @@ def test_english_numbers_pass_in_english(self, text: str) -> None: assert not foreign_decimal(text, ENGLISH) -def seaborn_style_bar_labels(fmt_label: Any, language: str) -> list[dict[str, str]]: - """The live seaborn case: bar labels and y tick labels as the probe's `labels` field lists them.""" +def probe_of(fig: plt.Figure, extra: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """The probe's `labels` (as `harness.drawn_labels` writes them, axes index included) and `tick_axes`.""" + fig.canvas.draw() + labels: list[dict[str, Any]] = [] + for index, ax in enumerate(fig.get_axes()): + labels += [{"kind": "annotation", "text": text.get_text()} for text in ax.texts] + for name in ("x", "y"): + ticks = getattr(ax, f"get_{name}ticklabels")() + labels += [ + {"kind": f"{name} tick label", "text": t.get_text(), "axes": index} for t in ticks if t.get_text() + ] + return {"labels": labels + (extra or []), "tick_axes": harness.tick_axes(fig)} + + +def seaborn_style_bar_labels(fmt_label: Any, language: str) -> dict[str, Any]: + """The live seaborn case: bar labels and y tick labels, and a title that names a version.""" fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) container = ax.bar(["A", "B", "C", "D"], [5.3, 4.25, 3.1, 2.05]) ax.bar_label(container, labels=[fmt_label(value) for value in container.datavalues]) ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: fmt_label(value))) - fig.canvas.draw() - labels = [{"kind": "annotation", "text": text.get_text()} for text in ax.texts] - labels += [{"kind": "y tick label", "text": tick.get_text()} for tick in ax.get_yticklabels() if tick.get_text()] - labels += [{"kind": "x tick label", "text": tick.get_text()} for tick in ax.get_xticklabels()] - labels.append({"kind": "title", "text": f"Version 2.5 ({language})"}) - return labels + return probe_of(fig, [{"kind": "title", "text": f"Version 2.5 ({language})"}]) class TestNumberFormatGate: def test_the_live_german_case_is_a_vq06_line(self) -> None: - labels = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "de") + probe = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "de") - [line] = gate_lines({"labels": labels}, plot_language="de") + [line] = gate_lines(probe, plot_language="de") assert DEFECT_RE.match(line), line assert line.startswith("VQ-06 (light): ") @@ -250,39 +297,94 @@ def test_the_live_german_case_is_a_vq06_line(self) -> None: def test_the_fixed_german_case_is_clean(self) -> None: german = str.maketrans(",.", ".,") - labels = seaborn_style_bar_labels(lambda value: f"{value:,.2f}".translate(german), "de") + probe = seaborn_style_bar_labels(lambda value: f"{value:,.2f}".translate(german), "de") - assert gate_lines({"labels": labels}, plot_language="Deutsch") == [] + assert gate_lines(probe, plot_language="Deutsch") == [] def test_the_english_control_is_clean_and_german_marks_in_english_are_named(self) -> None: english = seaborn_style_bar_labels(lambda value: f"{value:.2f}", "en") german = seaborn_style_bar_labels(lambda value: f"{value:.2f}".replace(".", ","), "en") - assert gate_lines({"labels": english}, plot_language="en") == [] - [line] = gate_lines({"labels": german}, plot_language="en") + assert gate_lines(english, plot_language="en") == [] + [line] = gate_lines(german, plot_language="en") assert "English writes a decimal point" in line + def test_date_ticks_and_category_ticks_are_never_checked(self) -> None: + """Copilot on #12184: a German `%d.%m` date (`14.10`) and a category the data names (`3.11`).""" + import matplotlib.dates as mdates + import pandas as pd + + fig, (dates, categories) = plt.subplots(1, 2, figsize=(8, 4.5), dpi=400) + days = pd.date_range("2026-10-01", periods=20, freq="D") + dates.plot(days, range(20)) + dates.xaxis.set_major_formatter(mdates.DateFormatter("%d.%m")) + dates.set_yticks([0, 10, 20]) + categories.bar(["3.11", "3.12", "3.13"], [1, 2, 3]) + categories.set_yticks([0, 1, 2, 3]) + probe = probe_of(fig) + + assert {"date", "category"} <= {entry["x"] for entry in probe["tick_axes"]} + assert any(label["text"] == "3.11" for label in probe["labels"]) + assert gate_lines(probe, plot_language="de") == [] + + def test_mathtext_reads_as_drawn(self) -> None: + """Copilot on #12184: `$\\mathdefault{5.30}$` shows a decimal point like `5.30`.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.plot([0, 1, 2], [0, 1, 2]) + ax.set_xticks([0, 1, 2]) + ax.set_yticks([0.25, 1.5], [r"$\mathdefault{0.25}$", r"$1.50$"]) + ax.text(1, 1, r"$5{,}30$") + probe = probe_of(fig) + assert probe["tick_axes"] == [{"x": "number", "y": "fixed"}] + probe["tick_axes"][0]["y"] = "number" # as a ScalarFormatter with mathtext would write them + + [line] = gate_lines(probe, plot_language="de") + + # x: 0, 1, 2 (no marks); y: the two mathtext labels; the annotation `$5{,}30$` reads 5,30. + assert line.startswith("VQ-06 (light): 2 of 6 numeric tick labels use a decimal point") + + def test_without_tick_axes_only_annotations_count(self) -> None: + labels = [{"kind": "y tick label", "text": "5.30", "axes": 0}, {"kind": "annotation", "text": "4.85"}] + + [line] = gate_lines({"labels": labels}, plot_language="de") + + assert line.startswith("VQ-06 (light): 1 of 1 numeric annotations use a decimal point") + def test_a_language_without_a_number_format_and_no_language_have_no_gate(self) -> None: - labels = [{"kind": "y tick label", "text": "5.30"}] + probe = {"labels": [{"kind": "annotation", "text": "5.30"}]} - assert _number_format_lines({"labels": labels}, number_format("Swedish"), "Swedish") == [] - assert gate_lines({"labels": labels}) == [] + assert _number_format_lines(probe, number_format("Swedish"), "Swedish") == [] + assert gate_lines(probe) == [] def test_titles_and_legend_entries_are_never_checked(self) -> None: labels = [{"kind": "title", "text": "2.5"}, {"kind": "legend text", "text": "1.5"}] - assert gate_lines({"labels": labels}, plot_language="de") == [] + assert gate_lines({"labels": labels, "tick_axes": []}, plot_language="de") == [] def test_colorbar_and_3d_tick_labels_count(self) -> None: - labels = [{"kind": "colorbar tick label", "text": "0.25"}, {"kind": "z tick label", "text": "1.5"}] + labels = [ + {"kind": "colorbar tick label", "text": "0.25", "axes": 1}, + {"kind": "z tick label", "text": "1.5", "axes": 0}, + ] + ticks = [{"x": "number", "y": "number", "z": "number"}, {"x": "none", "y": "number"}] - [line] = gate_lines({"labels": labels}, plot_language="de") + [line] = gate_lines({"labels": labels, "tick_axes": ticks}, plot_language="de") assert line.startswith("VQ-06 (light): 2 of 2 numeric tick labels use a decimal point") - @pytest.mark.parametrize("labels", ["broken", [None, {"kind": "x tick label", "text": 5.3}], [{"text": "5.30"}]]) - def test_a_tampered_labels_field_yields_no_line(self, labels: Any) -> None: - assert gate_lines({"labels": labels}, plot_language="de") == [] + @pytest.mark.parametrize( + "probe", + [ + {"labels": "broken"}, + {"labels": [None, {"kind": "annotation", "text": 5.3}]}, + {"labels": [{"text": "5.30"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": True}], "tick_axes": [{"x": "number"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": 3}], "tick_axes": [{"x": "number"}]}, + {"labels": [{"kind": "x tick label", "text": "5.30", "axes": 0}], "tick_axes": "broken"}, + ], + ) + def test_a_tampered_labels_field_yields_no_line(self, probe: Any) -> None: + assert gate_lines(probe, plot_language="de") == [] class TestNumberFormatTable: From 4ed334d2fef19130fe40354beefeb4c4219b4297 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:10:21 +0200 Subject: [PATCH 6/8] fix(agents): Copilot round 2 on the density and number-format gates - A bar counts when its rectangle meets the view, so bars zoomed on their value axis (ylim 900 to 1100 over bars ending at 1000) keep their labels. - A date formatter marks the axis as dates before the converter check, so date2num coordinates under DateFormatter("%d.%m") are never numbers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/render/harness.py | 14 +++++---- .../runtime/test_density_locale_gates.py | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index a4311ff24d2..2af57840f73 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -1213,8 +1213,9 @@ def _bar_ends(axes, box): For a vertical bar the centre is its x centre, the end the y of the side away from the baseline and the width its pixel width; a horizontal bar swaps the axes. A bar with a - missing value (NaN geometry) or a centre outside the axes box (a zoomed view) is left - out: it draws nothing, and a NaN would void the probe's JSON. + missing value (NaN geometry) or a rectangle outside the axes box (a zoomed view) is left + out: it draws nothing, and a NaN would void the probe's JSON. A bar whose rectangle + meets the view counts, even when its centre lies past a value-axis limit. """ bars = [] orientation = None @@ -1230,11 +1231,11 @@ def _bar_ends(axes, box): negative = (patch.get_width() if horizontal else patch.get_height()) < 0 if horizontal: bar = ((extent.y0 + extent.y1) / 2, extent.x0 if negative else extent.x1, extent.y1 - extent.y0) - centre = ((extent.x0 + extent.x1) / 2, bar[0]) else: bar = ((extent.x0 + extent.x1) / 2, extent.y0 if negative else extent.y1, extent.x1 - extent.x0) - centre = (bar[0], (extent.y0 + extent.y1) / 2) - if not all(math.isfinite(value) for value in bar) or not _in_view(*centre, box): + # Drawn when its rectangle meets the view: a bar zoomed past on its value axis still shows its end. + meets = extent.x0 <= box.x1 and extent.x1 >= box.x0 and extent.y0 <= box.y1 and extent.y1 >= box.y0 + if not all(math.isfinite(value) for value in bar) or not meets: continue bars.append((*bar, bool(patch.get_path_effects()))) return bars, orientation @@ -1408,6 +1409,9 @@ def _tick_kind(axis): labels_dict = getattr(getattr(major, "func", None), "func", None) if formatter == "FixedFormatter" or getattr(labels_dict, "__name__", "") == "_format_with_dict": return "fixed" + # A date formatter over coordinates converted beforehand (`mdates.date2num`) leaves the converter unset. + if "Date" in formatter or "dates" in type(major).__module__: + return "date" get_converter = getattr(axis, "get_converter", None) converter = get_converter() if get_converter is not None else getattr(axis, "converter", None) if converter is None: diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py index 2803da3d158..51c38973525 100644 --- a/tests/unit/agents/runtime/test_density_locale_gates.py +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -165,6 +165,19 @@ def test_bars_and_labels_outside_a_zoomed_view_are_not_counted(self) -> None: assert axes["bars"] == 5 and axes["labelled"] == 5 + def test_bars_zoomed_on_their_value_axis_still_count(self) -> None: + """Copilot round 2 on #12184: 30 bars ending at 1000 under `ylim(900, 1100)` still draw their ends.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(range(30), [1000] * 30) + ax.bar_label(container) + ax.set_ylim(900, 1100) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["bars"] == 30 and axes["labelled"] == 30 + assert _density_lines({"density": [axes]}) + def test_many_scatter_marks_are_matched_in_bounded_chunks(self) -> None: """Copilot on #12184: the pairwise arrays stay bounded however many marks the plot draws.""" fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) @@ -327,6 +340,23 @@ def test_date_ticks_and_category_ticks_are_never_checked(self) -> None: assert any(label["text"] == "3.11" for label in probe["labels"]) assert gate_lines(probe, plot_language="de") == [] + def test_preconverted_date_coordinates_are_dates(self) -> None: + """Copilot round 2 on #12184: `date2num` coordinates under a DateFormatter leave the converter unset.""" + import datetime + + import matplotlib.dates as mdates + + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + start = datetime.datetime(2026, 10, 1) + days = [mdates.date2num(start + datetime.timedelta(days=offset)) for offset in range(20)] + ax.plot(days, range(20)) + ax.xaxis.set_major_formatter(mdates.DateFormatter("%d.%m")) + ax.set_yticks([0, 10, 20]) + probe = probe_of(fig) + + assert probe["tick_axes"] == [{"x": "date", "y": "number"}] + assert gate_lines(probe, plot_language="de") == [] + def test_mathtext_reads_as_drawn(self) -> None: """Copilot on #12184: `$\\mathdefault{5.30}$` shows a decimal point like `5.30`.""" fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) From 52ca686cc68be77e4bf3cc541873dd0385e74d61 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:18:19 +0200 Subject: [PATCH 7/8] test(agents): G12 end to end through the real probe after #12183 The merged harness writes block F's `labels` next to `tick_axes`; a German bar chart with "5.30" labels fires G12 and the comma-formatted one does not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- .../runtime/test_density_locale_gates.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py index 51c38973525..22403ef7e8a 100644 --- a/tests/unit/agents/runtime/test_density_locale_gates.py +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -340,6 +340,32 @@ def test_date_ticks_and_category_ticks_are_never_checked(self) -> None: assert any(label["text"] == "3.11" for label in probe["labels"]) assert gate_lines(probe, plot_language="de") == [] + @pytest.mark.parametrize(("fmt", "fires"), [("{:.2f}", True), ("{:,.2f}", False)]) + def test_the_real_probe_end_to_end( + self, fmt: str, fires: bool, tmp_path: Any, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The harness's own probe file (block F's `labels`, this gate's `tick_axes`) through `evaluate`.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Figure, "savefig", Figure.savefig) # restored after the test + harness.install_probe("light") + german = str.maketrans(",.", ".,") + + def label(value: float) -> str: + text = fmt.format(value) + return text if fires else text.translate(german) + + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + container = ax.bar(["Altbrunn", "Seefeld", "Hochmatt"], [5.3, 4.85, 6.12]) + ax.bar_label(container, labels=[label(value) for value in container.datavalues]) + ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: label(value))) + fig.savefig("plot-light.png", dpi=100) + probe = json.loads((tmp_path / "probe-light.json").read_text()) + + lines = [line for line in gate_lines(probe, plot_language="de") if line.startswith("VQ-06")] + + assert probe["labels"] and probe["tick_axes"] == [{"x": "category", "y": "number"}] + assert bool(lines) is fires + def test_preconverted_date_coordinates_are_dates(self) -> None: """Copilot round 2 on #12184: `date2num` coordinates under a DateFormatter leave the converter unset.""" import datetime From 480cc620bb4bf241fa5934328cbbc987dac95d71 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:43:55 +0200 Subject: [PATCH 8/8] fix(agents): Fable and Gemini review of the density and number-format gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit G11: - Named points no longer count against the 25-label cap: without bars only numeric labels count (`numeric_labels`); scatter-annotated asks for 10 to 50. - With bars the cap counts labelled bar positions (`labelled_bars`), so a stacked bar's segment labels count once. - Labels inside a bar segment (`bar_label(label_type="center")`, a callable xycoords) are counted; they were missed. - The space per bar label comes from the labelled bars' nearest neighbours, so offset shadow bars or a chart that labels only its extremes no longer shrink it. - Bar ends come from the bars' data coordinates (inverted axes), markers of `plot(..., "o")` count as dots, a clipped `ax.text` outside the view is skipped. G12: - A trailing currency (`5.30 €`) and a sign before the currency (`-$5.30`) are numbers; `0.125` is never a thousands group; a zero-padded `14.10` annotation reads as a day and month. Adapter prompt: the marks are swapped through a placeholder (`.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`), exact for space-grouped languages. Each reproducer is a unit test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/prompts/adapter.md | 2 +- agents/anyplot/render/gates.py | 42 +++-- agents/anyplot/render/harness.py | 144 +++++++++++++----- docs/concepts/agent-network.md | 2 +- .../runtime/test_density_locale_gates.py | 132 +++++++++++++++- 5 files changed, 269 insertions(+), 53 deletions(-) diff --git a/agents/anyplot/prompts/adapter.md b/agents/anyplot/prompts/adapter.md index e966e329583..6321e2b4ce2 100644 --- a/agents/anyplot/prompts/adapter.md +++ b/agents/anyplot/prompts/adapter.md @@ -68,7 +68,7 @@ These rules replace the catalogue's own rules where they differ: - **Callouts and statistics.** A label or callout that states a number must compute it from `df` in an f-string (`f"r = {df[x].corr(df[y]):.2f}"`, written with the column variables of the code). A callout target comes from a rule over `df` (the maximum, the latest row), and a conditional claim is drawn only when `df` meets it. Remove a callout that cannot be derived. - **Density.** Adapt marker sizes, line widths and alpha to the user's row count (see the VQ-03 table below). - **Per-mark labels and decorations.** "The same style" covers elements whose count stays fixed (a title, a legend, one callout), not elements the catalogue draws once per bar or point of its few example marks: value labels, dots on bar ends, shadows. Their count grows with the user's data, so keep them only while each label fits the space of its mark and the plot stays legible, which rarely holds past about 25 marks; with more marks, drop them, or label only the extremes (the largest and the smallest) when the plot needs numbers. The server measures this (a `VQ-03` gate line counts the labels, the decorations and the pixels each label has). -- **Numbers.** Write every number in the plot text (tick labels, value labels, annotations, a number in the title) with the plot language's decimal and thousands marks and its words for large numbers, from the table "Number formats by plot language" below: `de` writes `5,30` and `6,5 Mio.`, `en` writes `5.30` and `6.5 M`. Python's `locale` is not available: format with an f-string and swap the marks (`f"{v:,.2f}".translate(str.maketrans(",.", ".,"))` for `,` decimals and `.` thousands; the table's thousands mark goes in place of the first `.`), through a `matplotlib.ticker.FuncFormatter` for the axes and the same function for value labels (`ax.bar_label(container, labels=[...])`, never `fmt="%.2f"`, which always writes a point). Years, dates, identifiers and the cells of the user's data stay as they are. A plot language the table does not list keeps the catalogue's formats. The server checks the decimal mark of every numeric tick label and annotation (a `VQ-06` gate line). +- **Numbers.** Write every number in the plot text (tick labels, value labels, annotations, a number in the title) with the plot language's decimal and thousands marks and its words for large numbers, from the table "Number formats by plot language" below: `de` writes `5,30` and `6,5 Mio.`, `en` writes `5.30` and `6.5 M`. Python's `locale` is not available: format with an f-string, which writes `,` for thousands and `.` for decimals, then swap both marks through a placeholder: `f"{v:,.2f}".replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`, with `DECIMAL` and `THOUSANDS` set once to the table's marks (`de`: `","` and `"."`; `fr`: `","` and `" "`; `en` keeps the f-string as it is), through a `matplotlib.ticker.FuncFormatter` for the axes and the same function for value labels (`ax.bar_label(container, labels=[...])`, never `fmt="%.2f"`, which always writes a point). Years, dates, identifiers and the cells of the user's data stay as they are. A plot language the table does not list keeps the catalogue's formats. The server checks the decimal mark of every numeric tick label and annotation (a `VQ-06` gate line). - **Dates.** Use `matplotlib.dates.AutoDateLocator` with `ConciseDateFormatter` instead of a fixed locator, unless the change request asks for a specific one. - **Safe code.** Use only the imports the code already has, plus, when needed, `numpy`, `pandas`, the plotting library, scipy's computing modules (`scipy.stats`, `scipy.interpolate`, `scipy.signal`, `scipy.optimize`, `scipy.fft`, `scipy.linalg`, ...) and these standard modules: `math`, `cmath`, `statistics`, `decimal`, `fractions`, `numbers`, `datetime`, `calendar`, `zoneinfo`, `collections`, `itertools`, `functools`, `operator`, `bisect`, `heapq`, `typing`, `warnings`, `textwrap`, `colorsys` and `re`. Never import `io`, `sys`, `pathlib`, `random`, `locale` or `string`. No file or network access, no `os` beyond the THEME line, no `.format(...)` (use f-strings), no `eval`, `exec`, `getattr`, `operator.attrgetter` or other reflection, no names with double underscores or private names of a module (`module._name`), no `class` statements, and no string literal longer than 200 characters. diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 7d65074cc5f..935dc31f10e 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -481,7 +481,10 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: """The G11 line: per-mark value labels or decorations past what the plot's marks leave room for. The probe's `density` field (`harness.density_report`) describes each axes; the line - names the worst one. Labels crowd when there are more than `MAX_VALUE_LABELS` of them, + names the worst one. Labels crowd when more than `MAX_VALUE_LABELS` bar positions carry + one (`labelled_bars`: a stacked bar's segment labels count once), or, on an axes without + bars, more than that many numeric labels sit on points (`numeric_labels`: named points + are the point of scatter-annotated), or when a bar label is wider than `LABEL_SLOT_FILL` of the space between bar centres (from `MIN_CRAMPED_LABELS` labels on); decorations (dots on bar ends, shadowed bars) when there are more than `MAX_DECORATIONS`. Only counts and pixels reach the line. @@ -497,6 +500,14 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: labelled, decorated = _count(entry.get("labelled")) or 0, _count(entry.get("decorated")) or 0 if bars is None or marks is None: continue + numeric = _count(entry.get("numeric_labels")) + positions = _count(entry.get("labelled_bars")) + # With bars the cap counts labelled bar positions, so a stacked bar's segment labels count once; without + # bars only numbers count: named points (scatter-annotated asks for 10 to 50) do not. + if bars: + capped = labelled if positions is None else min(positions, labelled) + else: + capped = labelled if numeric is None else min(numeric, labelled) slot, label = _pixels(entry.get("slot_px")), _pixels(entry.get("label_px")) cramped = ( bars > 0 @@ -505,14 +516,14 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: and label is not None and label > slot * LABEL_SLOT_FILL ) - many_labels = labelled > MAX_VALUE_LABELS + many_labels = capped > MAX_VALUE_LABELS many_decorations = decorated > MAX_DECORATIONS if not (cramped or many_labels or many_decorations): continue observed: list[str] = [] if cramped or many_labels: room = f" ({round(label)} px wide on {round(slot)} px per bar)" if cramped and label and slot else "" - observed.append(f"{labelled} value labels{room}") + observed.append(f"{labelled if bars else capped} value labels{room}") if many_decorations: observed.append(f"{decorated} dots or shadows") marks_text = f"{bars} bars" if bars else f"{marks} marks" @@ -528,11 +539,15 @@ def _density_lines(probe: dict[str, Any] | None) -> list[tuple[str, str]]: _NUMBER_LABEL = re.compile( - r"^(?:[$€£¥]\s?)?[+\-−]?(?P\d[\d.,    ']*\d|\d)" - r"(?:\s?(?:%|‰|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?$" + r"^[+\-−]?(?:[$€£¥]\s?)?[+\-−]?(?P\d[\d.,    ']*\d|\d)" + r"(?:\s?(?:%|‰|[$€£¥]|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?$" ) -"""A label that is one number: an optional currency sign and sign, digits with marks, an optional unit or word -(`%`, `Mio.`, `k`, `mil M`). Dates (`14.10.`, `2024-10-14`), ranges, times and words with digits never match.""" +"""A label that is one number: an optional sign and currency sign in either order, digits with marks, an optional +unit, word or trailing currency sign (`%`, `Mio.`, `k`, `mil M`, `5,30 €`, `-$5.30`). Dates (`14.10.`, +`2024-10-14`), ranges, times and words with digits never match.""" +_DAY_MONTH = re.compile(r"^(0[1-9]|[12]\d|3[01])[.](0[1-9]|1[0-2])$") +"""`14.10`: a day and month as `%d.%m` writes them, both zero-padded. Read as a date in an annotation, where no axis +says what it is; a decimal written with two digits each side (`10.12`) is the one number this lets through.""" NUMBER_LABEL_KINDS = frozenset({"annotation", "text"}) """The label kinds whose numbers G12 checks besides every tick label (`x tick label`, `y tick label`, `z tick label`, `colorbar tick label`, `harness.drawn_labels`); titles, axis titles and legend entries may name a version or a code.""" @@ -584,11 +599,13 @@ def foreign_decimal(text: str, numbers: NumberFormat) -> bool: Only a label that is one number counts (`_NUMBER_LABEL`). With a decimal comma, a single point followed by other than three digits (`5.30`, `6.0 M`) is foreign, and so is a point after a comma (`1,234.5`) or more than one comma (`1,234,567`); a single point with three - digits (`1.234`) is a thousands mark and passes. A decimal point mirrors this, except that + digits (`1.234`) is a thousands mark and passes, unless the whole part is 0 (`0.125`). A + zero-padded day and month (`14.10`, `_DAY_MONTH`) is a date and passes. A decimal point mirrors this, except that several points (`7.184.003`) pass, because a version-like label writes them too. """ - match = _NUMBER_LABEL.match(plain_text(text)) - if match is None: + shown = plain_text(text) + match = _NUMBER_LABEL.match(shown) + if match is None or _DAY_MONTH.match(shown): return False body = match.group("body") marks = [char for char in body if char in ".,"] @@ -602,8 +619,9 @@ def foreign_decimal(text: str, numbers: NumberFormat) -> bool: return own == "," and marks.count(own) > 1 # `1,234,567` groups with commas: English if marks.count(other) > 1: return False # `7.184.003` or `1,234,567` group thousands the other way: ambiguous, never named - decimals = body.rsplit(other, 1)[1] - return len(decimals) != 3 + whole, decimals = body.rsplit(other, 1) + # `0.125` is never a thousands group: a leading 0 makes the mark a decimal one. + return len(decimals) != 3 or whole == "0" def _number_format_lines( diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 86a9cbc795f..0428d2df7c5 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -122,6 +122,7 @@ import logging import math import os +import re import resource import shutil import sys @@ -1299,14 +1300,16 @@ def _in_view(x, y, box, slack=1.0): def _bar_ends(axes, box): - """Every visible bar of the axes inside its view as `(centre, end, width)` in display pixels along its + """Every visible bar of the axes inside its view as `(centre, end, width, base)` in display pixels along its orientation, plus a flag whether it draws a path effect (a shadow), and the orientation of the first bar container. For a vertical bar the centre is its x centre, the end the y of the side away from the - baseline and the width its pixel width; a horizontal bar swaps the axes. A bar with a - missing value (NaN geometry) or a rectangle outside the axes box (a zoomed view) is left - out: it draws nothing, and a NaN would void the probe's JSON. A bar whose rectangle - meets the view counts, even when its centre lies past a value-axis limit. + baseline, the base the y of the baseline side and the width its pixel width; a + horizontal bar swaps the axes. End and base come from the bar's data coordinates, so an + inverted or logarithmic value axis puts the end where the bar ends on the canvas. A bar + with a missing value (NaN geometry) or a rectangle outside the axes box (a zoomed view) + is left out: it draws nothing, and a NaN would void the probe's JSON. A bar whose + rectangle meets the view counts, even when its centre lies past a value-axis limit. """ bars = [] orientation = None @@ -1319,22 +1322,25 @@ def _bar_ends(axes, box): if not patch.get_visible(): continue extent = patch.get_window_extent() - negative = (patch.get_width() if horizontal else patch.get_height()) < 0 + x, y, w, h = patch.get_x(), patch.get_y(), patch.get_width(), patch.get_height() if horizontal: - bar = ((extent.y0 + extent.y1) / 2, extent.x0 if negative else extent.x1, extent.y1 - extent.y0) + (end, _), (base, _) = axes.transData.transform([(x + w, y), (x, y)]) + bar = ((extent.y0 + extent.y1) / 2, end, extent.y1 - extent.y0, base) else: - bar = ((extent.x0 + extent.x1) / 2, extent.y0 if negative else extent.y1, extent.x1 - extent.x0) + (_, end), (_, base) = axes.transData.transform([(x, y + h), (x, y)]) + bar = ((extent.x0 + extent.x1) / 2, end, extent.x1 - extent.x0, base) # Drawn when its rectangle meets the view: a bar zoomed past on its value axis still shows its end. meets = extent.x0 <= box.x1 and extent.x1 >= box.x0 and extent.y0 <= box.y1 and extent.y1 >= box.y0 - if not all(math.isfinite(value) for value in bar) or not meets: + if not all(math.isfinite(float(value)) for value in bar) or not meets: continue - bars.append((*bar, bool(patch.get_path_effects()))) + bars.append((*(float(value) for value in bar), bool(patch.get_path_effects()))) return bars, orientation def _point_marks(axes, box): - """The display positions of the axes' visible scatter marks and of the vertices of its lines of three or more, - finite and inside the axes box only.""" + """The display positions of the axes' visible point marks (scatter marks, and the markers of a `plot` call that + draws no line, `ax.plot(x, y, "o")`) and of the vertices of its lines of three or more, finite and inside the + axes box only.""" import numpy as np scatter, vertices = [], [] @@ -1344,8 +1350,13 @@ def _point_marks(axes, box): offsets = np.asarray(collection.get_offsets(), dtype=float).reshape(-1, 2) scatter.append(collection.get_offset_transform().transform(offsets)) for line in axes.get_lines(): + if not line.get_visible(): + continue xy = np.asarray(line.get_xydata(), dtype=float).reshape(-1, 2) - if line.get_visible() and len(xy) >= 3: # two-vertex lines are rules, brackets and reference lines + markers_only = line.get_linestyle() in ("None", "none", "", " ") and line.get_marker() not in (None, "None", "") + if markers_only: + scatter.append(line.get_transform().transform(xy)) + elif len(xy) >= 3: # two-vertex lines are rules, brackets and reference lines vertices.append(line.get_transform().transform(xy)) empty = np.empty((0, 2)) @@ -1363,27 +1374,36 @@ def kept(parts): return kept(scatter), kept(vertices) -def _label_anchors(axes, box): +def _label_anchors(axes, box, renderer): """The visible texts of the axes placed at a data coordinate, with their anchors in display pixels. An annotation counts when its `xycoords` is "data" (`ax.annotate`, `ax.bar_label`), any other text when it uses the data transform (`ax.text` at a data position). A callout in axes coordinates is no value label. Like matplotlib's own draw, an annotation whose point - lies outside the axes is skipped unless it is set to draw there (`annotation_clip=False`); - a plain text at an infinite position is skipped too. + lies outside the axes is skipped unless it is set to draw there (`annotation_clip=False`), + and so is a plain text clipped to the axes (`clip_on=True`) outside them; a text at an + infinite position is skipped too. """ found = [] for text in axes.texts: if not text.get_visible() or not text.get_text().strip(): continue if type(text).__name__ == "Annotation": - if text.xycoords != "data": + if callable(text.xycoords): + # `bar_label(label_type="center")` anchors at a share of its bar's box: a callable returning it. + try: + frame = text.xycoords(renderer) + anchor = (frame.x0 + text.xy[0] * frame.width, frame.y0 + text.xy[1] * frame.height) + except Exception: + continue + elif text.xycoords == "data": + anchor = axes.transData.transform(text.xy) + else: continue - anchor = axes.transData.transform(text.xy) clipped = text.get_annotation_clip() is not False elif text.get_transform() == axes.transData: anchor = axes.transData.transform(text.get_position()) - clipped = False + clipped = bool(text.get_clip_on()) else: continue x, y = float(anchor[0]), float(anchor[1]) @@ -1417,12 +1437,15 @@ def density_report(figure, dpi): many of them carry a value label or a decoration, with the space a label has. `bars` counts the visible bars, `marks` bars plus scatter marks plus line vertices - (scatter marks on a bar's end count as decorations instead), `labelled` the texts at a - data coordinate whose anchor sits on a mark (a bar's end or a point; `_label_anchors`), - `decorated` the scatter marks on a bar's end plus the bars that draw a path effect such as - a shadow, `slot_px` the median distance between neighbouring bar centres (the space one - bar's label has along the category axis) and `label_px` the median extent of the bar - labels along that axis. Pixels are the saved image's. The host turns this into G11. + (point marks on a bar's end count as decorations instead), `labelled` the texts at a + data coordinate whose anchor sits on a mark (at or past a bar's end, inside a bar's + segment, or on a point; `_label_anchors`), `numeric_labels` those of them that are one + number (`_is_number_text`: a named point, `Firm 3`, is not), `labelled_bars` the bar + positions that carry a label (a stacked bar's segments share one), `decorated` the point marks + on a bar's end plus the bars that draw a path effect such as a shadow, `slot_px` the + space one bar label has along the category axis (`_label_slot`, from the labelled bars + only) and `label_px` the median extent of the bar labels along that axis. Pixels are the + saved image's. The host turns this into G11. """ import numpy as np @@ -1447,25 +1470,33 @@ def density_report(figure, dpi): # A scatter mark on a bar's end is a decoration (the catalogue's dots on its bar tops), not a data mark. on_bar = _on_bar_ends(scatter, bar_array, horizontal, value_extent) points = np.concatenate([scatter[~on_bar], vertices])[:DENSITY_MARK_LIMIT] - labelled, label_sizes = 0, [] - for text, x, y in _label_anchors(axes, box): + segments = np.asarray([bar[3] for bar in bars[:DENSITY_MARK_LIMIT]], dtype=float) + slack = DENSITY_BAR_SLACK * value_extent + labelled, numeric, label_sizes, labelled_bars = 0, 0, [], [] + for text, x, y in _label_anchors(axes, box, renderer): along, across = (y, x) if horizontal else (x, y) - on_a_bar = len(bar_array) and bool( - ( - (np.abs(along - bar_array[:, 0]) <= bar_array[:, 2] / 2 + 1) - & (np.abs(across - bar_array[:, 1]) <= DENSITY_BAR_SLACK * value_extent) - ).any() - ) + hit = None + if len(bar_array): + # On a bar: past its end by at most `slack`, or inside its segment (`label_type="center"`). + low = np.minimum(bar_array[:, 1], segments) - slack + high = np.maximum(bar_array[:, 1], segments) + slack + matches = ( + (np.abs(along - bar_array[:, 0]) <= bar_array[:, 2] / 2 + 1) & (across >= low) & (across <= high) + ) + if matches.any(): + hit = int(np.argmax(matches)) on_a_point = len(points) and bool( ( (np.abs(x - points[:, 0]) <= DENSITY_POINT_SLACK[0] * width + 2) & (np.abs(y - points[:, 1]) <= DENSITY_POINT_SLACK[1] * height + 2) ).any() ) - if not (on_a_bar or on_a_point): + if hit is None and not on_a_point: continue labelled += 1 - if on_a_bar: + numeric += _is_number_text(text.get_text()) + if hit is not None: + labelled_bars.append((float(bar_array[hit, 0]), float(bar_array[hit, 2]))) try: extent = text.get_window_extent(renderer) except Exception: @@ -1473,14 +1504,15 @@ def density_report(figure, dpi): size = float(extent.height if horizontal else extent.width) if math.isfinite(size): label_sizes.append(size) - centres = np.unique(np.round(bar_array[:, 0], 1)) if len(bar_array) else np.empty(0) - slot = float(np.median(np.diff(centres))) if len(centres) > 1 else None + slot = _label_slot(labelled_bars) report.append( { "bars": len(bars), "marks": len(bars) + int((~on_bar).sum()) + len(vertices), "labelled": labelled, - "decorated": int(on_bar.sum()) + sum(1 for bar in bars if bar[3]), + "numeric_labels": numeric, + "labelled_bars": len({round(centre, 1) for centre, _ in labelled_bars}), + "decorated": int(on_bar.sum()) + sum(1 for bar in bars if bar[4]), "slot_px": None if slot is None else round(slot * scale, 1), "label_px": round(float(np.median(label_sizes)) * scale, 1) if label_sizes else None, } @@ -1488,6 +1520,42 @@ def density_report(figure, dpi): return report +_NUMBER_TEXT = re.compile( + r"[$€£¥]?\s?[+\-−]?\s?[$€£¥]?\s?\d[\d.,'    ]*" + r"(?:\s?(?:%|‰|[$€£¥]|[^\W\d_]{1,5}\.?(?:\s[^\W\d_]{1,5}\.?)?))?" +) +_MATH_MARKUP = re.compile(r"\\mathdefault|\\mathrm|\\,|[{}$]") + + +def _is_number_text(text): + """Whether a label is one number (`5.30`, `6,5 Mio.`, `12 %`, `5,30 €`), mathtext read as drawn; a name is not.""" + stripped = text.strip() + if len(stripped) > 2 and stripped.startswith("$") and stripped.endswith("$") and stripped.count("$") == 2: + stripped = _MATH_MARKUP.sub("", stripped).strip() + return _NUMBER_TEXT.fullmatch(stripped) is not None + + +def _label_slot(labelled_bars): + """The space one bar label has along the category axis: the median distance from each labelled bar's centre to + the nearest other labelled bar's centre, in display pixels, or None for fewer than two. + + Only labelled bars count, so unlabelled bars between them (an offset shadow series, a + chart that labels only its extremes) leave the space as wide as it looks. Distances + below half a bar width are one bar labelled twice (a stacked bar's segments) and are skipped. + """ + import numpy as np + + if len(labelled_bars) < 2: + return None + centres = np.asarray([centre for centre, _ in labelled_bars], dtype=float) + widths = np.asarray([bar_width for _, bar_width in labelled_bars], dtype=float) + distances = np.abs(centres[:, None] - centres[None, :]) + distances[distances < np.median(widths) / 2] = np.inf + nearest = distances.min(axis=1) + nearest = nearest[np.isfinite(nearest)] + return float(np.median(nearest)) if len(nearest) else None + + def _tick_kind(axis): """What an axis's tick labels write: `number` (a numeric scale and formatter), `date`, `category`, `fixed` (labels the code set itself, `set_xticks(positions, labels)`: often the data's own categories), `none` (no labels) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 2eb4f98d9c1..3bdb656849a 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -323,7 +323,7 @@ Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` wi **Scripts and fonts.** Plot text can be in the scripts most users' data and languages bring: the renderer image installs Noto Sans, Noto Sans CJK and the Noto Sans faces of the scripts DejaVu Sans lacks (37 files, 46.8 MB, Regular and Bold where the family has both), and the harness appends them to every font lookup, so a Chinese title or a Devanagari tick label draws with the right glyphs while the code's own first font stays in charge of Latin text. Right-to-left scripts work: the pinned matplotlib lays text out through libraqm, so Arabic and Hebrew render shaped and in reading order, within one label aligned by the code's `ha`. Three limits remain, and G10 reports the first and the last: a `$…$` math string uses matplotlib's math fonts without fallback and draws `¤` for every character they lack, so non-Latin words belong outside math text; Han characters take the Japanese regional forms of the CJK face until a caller passes the plot's language (`text.language`, a follow-up with the language chip); and a character outside the installed families draws as a Last Resort placeholder box. Most pictographs are not in that last group, because Noto Sans Symbols2 or DejaVu Sans draws them as monochrome outlines (📈, 📊, ⚠, 😀), but 🔥, ✅, the flag letters, Tibetan and Mongolian have no glyph in any installed font. G10 counts those characters deterministically from matplotlib's own reports, names up to three of them with their code points (`U+1F525 🔥 fire`), tells the repair that a user's column name stays as it is, and keeps the line in the result's residual lines even when the reviewer passed the render, because a measured missing glyph is not a judgement. -**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits on a bar's end or a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels), the dots on bar ends and the shadowed bars, and measures the median bar label width against the space between bar centres. G11 writes a VQ-03 line when an axes carries more than `MAX_VALUE_LABELS` (25) value labels, from six labels on when a bar label is wider than 90 % of the space per bar, or more than `MAX_DECORATIONS` (25) dots or shadows. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string. G12 reads the probe's `labels` field (the drawn texts by kind, from block F) and its `tick_axes` field (`harness.tick_axes`: whether each axis's tick labels are numbers, dates, categories or labels the code set itself), and writes a VQ-06 line when an annotation, or a tick label of a numeric axis, uses the other decimal mark, mathtext read as drawn: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`), a point after a comma, or commas grouping thousands; dates, years, version strings, titles and legend entries never count, and no label text reaches the line. +**Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits at a bar's end, inside a bar's segment (`bar_label(label_type="center")`) or on a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels, and neither is a text matplotlib clips away in a zoomed view), which of them are numbers, the dots on bar ends (scatter marks or `plot(..., "o")` markers) and the shadowed bars, and measures the median bar label width against the space between neighbouring labelled bars (unlabelled bars between them, such as an offset shadow series, do not shrink it). Bar ends come from the bars' data coordinates, so an inverted axis works. G11 writes a VQ-03 line (owner, 2026-10-15: the thresholds stay) when more than `MAX_VALUE_LABELS` (25) bar positions carry a label (a stacked bar's segment labels count once), when more than 25 numeric labels sit on points (named points, the point of scatter-annotated, never count), from six labels on when a bar label is wider than 90 % of the space per bar, or when more than `MAX_DECORATIONS` (25) dots or shadows are drawn. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string whose marks are swapped through a placeholder (`.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`), exact for the space-grouped languages too. A region such as `de-CH` with its apostrophe (7'184'003) is a follow-up of its own setting (owner, 2026-10-15). G12 reads the probe's `labels` field (the drawn texts by kind, from block F) and its `tick_axes` field (`harness.tick_axes`: whether each axis's tick labels are numbers, dates, categories or labels the code set itself), and writes a VQ-06 line when an annotation, or a tick label of a numeric axis, uses the other decimal mark, mathtext read as drawn: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`, `5.30 €`), a point after a whole part of 0 (`0.125`), a point after a comma, or commas grouping thousands; dates (a zero-padded `14.10` in an annotation reads as a day and month), years, version strings, titles, legend entries and tick labels the code set itself never count, and no label text reaches the line. Later languages reuse the CI commands: `Rscript plot.R`; `julia --project=/opt/julia plot.jl` with a stacked `JULIA_DEPOT_PATH` and a pinned `JULIA_CPU_TARGET`; `node /opt/anyplot/js-render/render.mjs plot.js` with `window.ANYPLOT_DATA` injected; bokeh with `BOKEH_RESOURCES=inline`, `SE_OFFLINE=true` and a shipped chromedriver; plotly with local or disabled MathJax. diff --git a/tests/unit/agents/runtime/test_density_locale_gates.py b/tests/unit/agents/runtime/test_density_locale_gates.py index 22403ef7e8a..0ed94c7be05 100644 --- a/tests/unit/agents/runtime/test_density_locale_gates.py +++ b/tests/unit/agents/runtime/test_density_locale_gates.py @@ -141,6 +141,104 @@ def test_one_callout_in_axes_coordinates_is_no_value_label(self) -> None: assert axes["labelled"] == 0 and _density_lines({"density": [axes]}) == [] + def test_named_points_are_no_value_labels_review_d02(self) -> None: + """Fable g11/d02 and perf/p04: 30 and 40 named scatter points (scatter-annotated asks for 10 to 50).""" + for count in (30, 40): + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + rng = np.random.default_rng(9) + x, y = rng.random(count) * 100, rng.random(count) * 100 + ax.scatter(x, y, s=80) + for index, (xi, yi) in enumerate(zip(x, y, strict=True)): + ax.text(xi + 1, yi + 1, f"Firm {index}", fontsize=9) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == count and axes["numeric_labels"] == 0 + assert _density_lines({"density": [axes]}) == [] + + def test_offset_shadow_bars_leave_the_space_per_label_review_d01(self) -> None: + """Fable g11/d01: 12 labelled bars over 12 offset shadow bars once measured 8 px per bar.""" + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + x = np.arange(12) + y = np.random.default_rng(1).integers(50, 500, 12) + ax.bar(x + 0.04, y - 4, color="#00000033", zorder=1) + container = ax.bar(x, y, zorder=2) + ax.bar_label(container, fmt="%d") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == 12 and axes["slot_px"] > 150 + assert _density_lines({"density": [axes]}) == [] + + def test_labelling_only_the_extremes_of_52_bars_is_clean(self) -> None: + """Gemini on #12184: the fix G11 asks for must pass it.""" + fig = grouped_bars(26, labels=False, dots=False, shadows=False) + ax = fig.axes[0] + rects = [rect for container in ax.containers for rect in container] + for rect in (max(rects, key=lambda r: r.get_height()), min(rects, key=lambda r: r.get_height())): + ax.annotate(f"{int(rect.get_height())}", xy=(rect.get_x() + rect.get_width() / 2, rect.get_height())) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 2 and _density_lines({"density": [axes]}) == [] + + def test_stacked_centre_labels_count_once_per_bar_review_p05(self) -> None: + """Fable perf/p05 and g11/a05: `bar_label(label_type="center")` labels were not counted at all.""" + for positions, fires in ((8, False), (40, True)): + fig, ax = plt.subplots(figsize=(16, 9), dpi=100) + x = np.arange(positions) + rng = np.random.default_rng(2) + bottom = np.zeros(positions) + for _ in range(4): + values = rng.integers(20, 100, positions) + container = ax.bar(x, values, bottom=bottom) + ax.bar_label(container, label_type="center", fmt="%d", fontsize=6) + bottom += values + fig.canvas.draw() + + [axes] = harness.density_report(fig, 200) + + assert axes["labelled"] == 4 * positions and axes["labelled_bars"] == positions + assert bool(_density_lines({"density": [axes]})) is fires + + def test_an_inverted_value_axis_keeps_the_bar_ends(self) -> None: + """Gemini on #12184: the end of a bar on an inverted axis is its lower edge on the canvas.""" + fig = grouped_bars(26, labels=True, dots=True, shadows=False) + fig.axes[0].invert_yaxis() + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 52 and axes["decorated"] == 52 + + def test_dots_drawn_with_plot_markers_are_decorations(self) -> None: + """Gemini on #12184: `ax.plot(x, top, "o")` per bar, a Line2D of one vertex.""" + fig = grouped_bars(26, labels=False, dots=False, shadows=False) + ax = fig.axes[0] + for rect in [rect for container in ax.containers for rect in container]: + ax.plot(rect.get_x() + rect.get_width() / 2, rect.get_height(), "o") + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["decorated"] == 52 and axes["marks"] == 52 + + def test_a_clipped_text_outside_a_zoomed_view_is_not_counted(self) -> None: + """Gemini on #12184: `ax.text(..., clip_on=True)` outside the view is never drawn.""" + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) + ax.bar(range(50), range(1, 51)) + for index in range(50): + ax.text(index, index + 1.5, str(index + 1), ha="center", clip_on=True) + ax.set_xlim(-0.5, 4.5) + fig.canvas.draw() + + [axes] = harness.density_report(fig, 400) + + assert axes["labelled"] == 5 + def test_a_missing_bar_value_keeps_the_report_valid_json(self) -> None: """Copilot on #12184: a NaN bar (a group missing after a pivot) would void the probe's JSON.""" fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) @@ -240,10 +338,32 @@ def test_a_few_cramped_labels_are_callouts(self) -> None: class TestForeignDecimal: - @pytest.mark.parametrize("text", ["5.30", "6.0 M", "0.25", "−1.5", "1,234.5", "12.5 %", "$ 3.75", "1,234,567"]) + @pytest.mark.parametrize( + "text", + [ + "5.30", + "6.0 M", + "0.25", + "−1.5", + "1,234.5", + "12.5 %", + "$ 3.75", + "1,234,567", + "5.30 €", # review: a trailing currency, the German order + "-$5.30", + "0.125", # review: a leading 0 is never a thousands group + "−0.125", + ], + ) def test_english_numbers_in_a_german_plot(self, text: str) -> None: assert foreign_decimal(text, GERMAN) + def test_a_zero_padded_day_and_month_is_a_date(self) -> None: + """Review of #12184: `14.10` in an annotation is a `%d.%m` date; `12.50` stays a decimal.""" + assert not foreign_decimal("14.10", GERMAN) + assert foreign_decimal("12.50", GERMAN) + assert not foreign_decimal("5,30 €", GERMAN) + @pytest.mark.parametrize( "text", [ @@ -460,5 +580,15 @@ def test_the_adapter_reads_the_table_and_the_rules(self) -> None: assert "| `de` (German) | `,` | `.` | 10^3: Tsd., 10^6: Mio., 10^9: Mrd. |" in instruction assert "**Numbers.**" in instruction and "**Per-mark labels and decorations.**" in instruction + def test_the_adapter_pattern_writes_every_table_language_and_passes_g12(self) -> None: + """Gemini on #12184: the old hint (`maketrans(",.", ".,")`, then the thousands mark for "the first `.`") was + ambiguous for space-grouped languages; the placeholder pattern is exact for every row.""" + assert '.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)' in adapter_instruction("matplotlib") + for code, spec in NUMBER_FORMATS.items(): + text = f"{1234567.25:,.2f}".replace(",", "_").replace(".", spec.decimal).replace("_", spec.thousands) + + assert text == f"1{spec.thousands}234{spec.thousands}567{spec.decimal}25", code + assert not foreign_decimal(f"{0.5:,.2f}".replace(".", spec.decimal), spec), code + def test_the_adapter_reads_the_narrowed_house_style_rule(self) -> None: assert "not those drawn once per bar or point" in adapter_rubric()