From f126ed70d411ba6aea87ea9c470019d4e756192e Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:46:58 +0200 Subject: [PATCH 1/5] fix(agents): keep pasted data exact or warn about every change Owner rule of 2026-10-15: precision is never lost silently. - loader: read data.csv with float_precision="round_trip" (pandas 3's default reader misread ~23 % of 17-digit floats by one ulp) - parse: count numbers stored rounded (more digits than a float64), keep JSON/literal float texts, whole numbers past int64 stay text, warn on NFC, trimmed text cells and missing tokens in text columns, exact integer min/max in the profile - prepare: exact whole numbers in set_type number, aggregate sum (no Int64 wraparound), cumsum; change lines count rounded values; keep sub-seconds and offsets; year-first ISO not read day-first; mixed UTC offsets refused instead of crashing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/code/loader.py | 23 ++- agents/anyplot/data/parse.py | 202 +++++++++++++++++++++++-- agents/anyplot/data/prepare.py | 197 +++++++++++++++++++----- agents/anyplot/schemas.py | 4 +- agents/evals/stress.py | 9 +- changelog.d/agent-data-fidelity.md | 17 +++ docs/concepts/agent-network.md | 6 +- tests/unit/agents/code/test_loader.py | 44 +++++- tests/unit/agents/data/test_parse.py | 145 +++++++++++++++++- tests/unit/agents/data/test_prepare.py | 110 +++++++++++++- tests/unit/agents/evals/test_stress.py | 3 +- 11 files changed, 688 insertions(+), 72 deletions(-) create mode 100644 changelog.d/agent-data-fidelity.md diff --git a/agents/anyplot/code/loader.py b/agents/anyplot/code/loader.py index b05a748437c..202953760b3 100644 --- a/agents/anyplot/code/loader.py +++ b/agents/anyplot/code/loader.py @@ -7,8 +7,10 @@ ISO dates), so no `sep`, `decimal` or date format is needed. The column dtypes from the dataset profile map to pandas dtypes that keep missing values (`integer` to `Int64`, `number` to `float64`, `boolean` to `boolean`, `text` to `string`); datetime -columns go to `parse_dates`. `import pandas as pd` is added after the last module-level -import when the file does not have it. +columns go to `parse_dates`. A dataset with a number column is read with +`float_precision="round_trip"`, so every float is the one `data.csv` writes in its +shortest round-trip form (`ROUND_TRIP`). `import pandas as pd` is added after the last +module-level import when the file does not have it. `data.csv` writes a missing cell as an empty cell and nothing else, so a text cell such as `NA` (Namibia, North America), `None` or `null` is a value there. pandas reads those tokens @@ -24,7 +26,7 @@ import ast import json -from typing import get_args +from typing import Literal, get_args from ..data.parse import MISSING_VALUES from ..schemas import ColumnDtype @@ -113,6 +115,16 @@ def na_values(dtype_map: dict[str, str], dates: list[str]) -> dict[str, list[str return values +ROUND_TRIP: Literal["round_trip"] = "round_trip" +"""pandas' exact float reader: its default C reader misreads about one in four 17-digit numbers by one unit in +the last place (measured on pandas 3.0, 2026-10-15), so the plot would not draw the values `data.csv` holds.""" + + +def round_trip(dtype_map: dict[str, str]) -> bool: + """Whether the read needs `float_precision="round_trip"`: some column is read as float64.""" + return PANDAS_DTYPES["number"] in dtype_map.values() + + def _literal(text: str) -> str: """A double-quoted Python string literal (JSON escapes are valid Python escapes).""" return json.dumps(text, ensure_ascii=False) @@ -132,11 +144,14 @@ def _statement(indent: str, dtype_map: dict[str, str], dates: list[str]) -> str: f"{_literal(name)}: " + ('[""]' if dtype_map.get(name) == TEXT_DTYPE else NA_NAME) for name in [*dtype_map, *(name for name in dates if name not in dtype_map)] ] + exact = round_trip(dtype_map) arguments = [_literal(DATA_FILE)] if dtype_items: arguments.append("dtype={" + ", ".join(dtype_items) + "}") if date_items: arguments.append("parse_dates=[" + ", ".join(date_items) + "]") + if exact: + arguments.append(f'float_precision="{ROUND_TRIP}"') if per_column: arguments += ["keep_default_na=False", "na_values={" + ", ".join(na_items) + "}"] prefix = _tokens_statement(indent) + "\n" + indent if per_column else "" @@ -150,6 +165,8 @@ def _statement(indent: str, dtype_map: dict[str, str], dates: list[str]) -> str: lines += [f"{inner}dtype={{", *(f"{entry}{item}," for item in dtype_items), f"{inner}}},"] if date_items: lines += [f"{inner}parse_dates=[", *(f"{entry}{item}," for item in date_items), f"{inner}],"] + if exact: + lines.append(f'{inner}float_precision="{ROUND_TRIP}",') if per_column: lines += [f"{inner}keep_default_na=False,", f"{inner}na_values={{", *(f"{entry}{item}," for item in na_items)] lines.append(f"{inner}}},") diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 5d9900b909d..a47c32df153 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -11,7 +11,8 @@ `MAX_COMPUTED_BYTES` (2 MiB) for a table the service computed (`too_long`; the canonical CSV the parse writes is checked against `MAX_CANONICAL_BYTES` too); no control character other than tab, line feed and carriage return (`control_chars`: every Unicode `Cc` character, which is C0, DEL and C1). -2. **Normalise.** Strip a leading BOM and apply NFC. +2. **Normalise.** Strip a leading BOM and apply NFC, with one warning that counts the + lines NFC changed (a decomposed `e` and accent became `é`: the text reads the same). 3. **Read every cell as text.** A literal when the first non-blank character is `[`, `{` or `(`, also behind a Python assignment (`cat_day = {`), which is stripped: `json.loads` first, then `ast.literal_eval` for a Python literal @@ -23,7 +24,8 @@ or an object of scalars or an array of `[category, value]` pairs (`json_pairs`: the columns `category` and `value`, one row per entry, in the pasted order; the keys stay text, the values are typed); `null`, `NaN` and - `Infinity` are missing; a key repeated in an object keeps its last value, with + `Infinity` are missing; a float keeps the text it was written with (`_SourceFloat`), + so the typing sees every digit; a key repeated in an object keeps its last value, with a warning. Since 2026-10-15 a record's nested objects become columns named with dots (`country.value`), down to `MAX_FLATTEN_DEPTH` levels (`_flatten`), and a field that holds a list of values expands to one row per value (`_records`); the records @@ -62,7 +64,10 @@ characters), turn line and paragraph separators (U+2028, U+2029) into spaces, turn CRLF and CR inside a quoted cell into LF, strip surrounding whitespace, and map `MISSING_VALUES` to an empty cell. One warning per column that lost - invisible characters. A cell (header or value) longer than + invisible characters; in a column that stays text, one warning counts the cells + whose surrounding whitespace went (not in a space-aligned table, whose padding is + layout) and one the cells read as missing, naming the tokens (`None`, `-`), since + there they could have been categories (`_fidelity_notes`). A cell (header or value) longer than `MAX_INPUT_CELL_CHARS` after cleaning is `cell_too_long`, checked before any typing reads the cell. A column without a header whose cells are all missing (the trailing-delimiter column of many spreadsheet exports) is dropped with a @@ -112,12 +117,14 @@ import math import re import statistics +import sys import tokenize import unicodedata from collections import Counter from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime +from decimal import Decimal, InvalidOperation from itertools import pairwise, product from typing import Any, Literal @@ -448,6 +455,10 @@ class _Column: mark_rows: list[int] = field(default_factory=list) # the rows whose cell is a declared missing-value mark id_like: bool = False # the name says ids, codes, versions or phone numbers (`_id_like`): no thousands reading group_mark: str | None = None # the thousands mark `_as_grouped_numbers` read, evidence for other columns + integers: list[int] = field(default_factory=list) # an integer column's values, exact (`numbers` are floats) + rounded: int = 0 # the cells a number column holds only rounded: more digits than a float64 keeps + trimmed: int = 0 # the cells whose surrounding whitespace the cleaning removed + missing_tokens: Counter[str] = field(default_factory=Counter) # `MISSING_VALUES` tokens read as missing def pandas_dtypes(column_dtypes: Mapping[str, ColumnDtype]) -> dict[str, str]: @@ -483,12 +494,20 @@ def parse_dataset( _check_size(text, max_bytes) text = text.removeprefix(_BOM) _check_controls(text) - text = unicodedata.normalize("NFC", text) - stripped = text.strip() + composed = unicodedata.normalize("NFC", text) + stripped = composed.strip() if not stripped: raise ParseError("unparseable", "the data is empty") warnings: list[str] = [] + if composed != text: + # The same letters, one code point instead of two (`e` + U+0301 is `é`): said, as every change to the text is. + changed = sum(1 for line in text.splitlines() if not unicodedata.is_normalized("NFC", line)) + warnings.append( + f"{_count(changed, 'line')} held decomposed characters (such as e and a separate accent), now written " + "in Unicode's composed form (NFC); the text reads the same" + ) + text = composed context: list[str] = [] table: tuple[list[str], list[list[str]], SourceFormat] | None = None literal = _LITERAL_START.match(stripped) @@ -1115,17 +1134,67 @@ def objects(pairs: list[tuple[str, Any]]) -> dict[str, Any]: return dict(pairs) try: - return json.loads(text, parse_constant=lambda _: None, object_pairs_hook=objects), repeated + loaded = json.loads(text, parse_constant=lambda _: None, parse_float=_SourceFloat, object_pairs_hook=objects) + return loaded, repeated except (ValueError, RecursionError): pass try: - tree = ast.parse(text, mode="eval") + tree = _FloatTexts(text).visit(ast.parse(text, mode="eval")) value = ast.literal_eval(tree) except (ValueError, SyntaxError, TypeError, RecursionError, MemoryError): return None return value, _repeated_keys(tree) +class _SourceFloat(float): + """A float of a JSON or Python literal that remembers how it was written. + + `json.loads` and `ast.literal_eval` read `0.1234567890123456789` as a float64 of 17 + digits; the cell takes the written text instead (`_json_cell`), so the typing step + sees every digit and says when a number column holds one only rounded. + """ + + text: str + + def __new__(cls, text: str) -> "_SourceFloat": + number = super().__new__(cls, text.replace("_", "")) + number.text = text + return number + + +class _FloatTexts(ast.NodeTransformer): + """Turn the float constants of a Python literal into `_SourceFloat`s of their written text. + + A signed float (`-0.5`) becomes one constant with its sign, since `literal_eval` applies + a sign only to a plain `float`. A complex literal (`1.5+2j`) is left as Python reads it. + """ + + def __init__(self, source: str) -> None: + self.source = source + + def _constant(self, node: ast.AST, sign: str, constant: ast.Constant) -> ast.AST: + written = ast.get_source_segment(self.source, constant) + if not written: + return node + return ast.copy_location(ast.Constant(_SourceFloat(sign + written)), node) + + def visit_BinOp(self, node: ast.BinOp) -> ast.AST: + return node + + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: + operand = node.operand + if ( + isinstance(node.op, ast.USub | ast.UAdd) + and isinstance(operand, ast.Constant) + and type(operand.value) is float + ): + return self._constant(node, "-" if isinstance(node.op, ast.USub) else "", operand) + return self.generic_visit(node) + + def visit_Constant(self, node: ast.Constant) -> ast.AST: + return self._constant(node, "", node) if type(node.value) is float else node + + def _repeated_keys(tree: ast.AST) -> int: """How many dict keys of an evaluated literal repeat an earlier key of the same dict.""" repeated = 0 @@ -1624,6 +1693,8 @@ def _json_cell(value: object, position: int) -> str: return "true" if value else "false" if isinstance(value, int): return str(value) + if isinstance(value, _SourceFloat) and _FLOAT.fullmatch(value.text): + return value.text # every written digit: a JSON number's text is a dot-decimal number already if isinstance(value, float): return repr(value) if math.isfinite(value) else "" if isinstance(value, str): @@ -1870,8 +1941,19 @@ def _clean_key(raw: str) -> tuple[str, bool]: A cell of canonical CSV (`parse_dataset(canonical=True)`) is cleaned the same way: there the empty cell is the only missing one.""" + text, invisible, _ = _cleaned(raw) + return text, invisible + + +def _cleaned(raw: str) -> tuple[str, bool, bool]: + """The cleaned cell, whether invisible characters went, and whether surrounding whitespace went. + + A cell of nothing but whitespace is empty, not trimmed: it was blank to begin with. + """ text, invisible = _strip_invisible(raw) - return text.replace("\r\n", "\n").replace("\r", "\n").strip(), invisible + unstripped = text.replace("\r\n", "\n").replace("\r", "\n") + stripped = unstripped.strip() + return stripped, invisible, bool(stripped) and stripped != unstripped def canonical_names(headers: Sequence[str]) -> list[str]: @@ -1956,9 +2038,12 @@ def _type_column(column: _Column) -> None: for the whole column when any cell has a time. 2. **boolean**: `true` / `false` in any letter case, written `True` / `False`. 3. **integer**: ASCII digits with an optional sign within int64. A column with a - leading zero (`007`) stays text, because the zeros are part of the value. + leading zero (`007`) stays text, because the zeros are part of the value, and so + does a column with a whole number past int64 (`BIG_INTEGERS`): a float64 would + round its last digits. 4. **number**: dot decimals with an optional exponent, written in Python's - shortest round-trip form; then decimal commas when every cell has the + shortest round-trip form (the value is kept, not the spelling: `1.50` is `1.5`; + a cell with more digits than a float64 holds is counted, `rounded_note`); then decimal commas when every cell has the `1.234,56` or `1234,56` shape and at least one has a comma. Dates were tried first, so `DD.MM.YYYY` never reads as a thousands-dot number. Then numbers whose thousands are grouped unambiguously by a dot, a comma, an apostrophe or a space @@ -2002,6 +2087,10 @@ def _as_plain_numbers(column: _Column, cells: list[str], present: list[str]) -> return False if _as_integers(column, cells): return True + # Past int64 a whole number only fits a float64, which keeps 15 to 17 digits: an id or a count + # of 20 digits would lose its last ones. Text keeps every digit; `set_type number` can still read it. + column.notes.append(BIG_INTEGERS) + return False if all(_FLOAT.fullmatch(cell) for cell in present): return _as_numbers(column, cells) if all(_DECIMAL_COMMA.fullmatch(cell) for cell in present) and any("," in cell for cell in present): @@ -2017,6 +2106,17 @@ def _as_plain_numbers(column: _Column, cells: list[str], present: list[str]) -> AMBIGUOUS_COMMAS = "commas read as decimal commas; remove them first if they separate thousands" """The note of a column whose commas could be decimals or thousands (`1,234`); `_commas_as_thousands` replaces it.""" +BIG_INTEGERS = "whole numbers beyond the 64-bit range kept as text, so no digit is lost" +"""The note of an integer column with a value past int64, which a number column could hold only rounded.""" + + +def rounded_note(count: int) -> str: + """The note of a number column whose cells hold more significant digits than a float64 keeps.""" + return ( + f"{_count(count, 'value')} with more significant digits than a number holds (15 to 17) " + f"{'is' if count == 1 else 'are'} stored rounded to the nearest number" + ) + def _as_grouped_numbers(column: _Column, cells: list[str], present: list[str]) -> bool: """Numbers whose thousands are grouped by a mark the rules above do not read (live test of 2026-10-15). @@ -2173,11 +2273,21 @@ def _as_integers(column: _Column, cells: list[str]) -> bool: return False column.dtype = "integer" column.cells = ["" if value is None else str(value) for value in values] - column.numbers = [float(value) for value in values if value is not None] + column.integers = [value for value in values if value is not None] + column.numbers = [float(value) for value in column.integers] + column.rounded = 0 return True def _as_numbers(column: _Column, cells: list[str]) -> bool: + """The dot-decimal `cells` as floats, written in their shortest round-trip form (`repr`). + + The value is what counts, not the spelling: `1.50`, `1.5e0` and `+1.5` are all `1.5`. A + cell whose value a float64 does not hold (more than about 15 to 17 significant digits, + `0.1234567890123456789`, or a whole number past 2**53 beside decimals) is counted in + `_Column.rounded`, and `_build` warns about it (owner rule of 2026-10-15: precision is + never lost silently). + """ values = [float(cell) if cell else None for cell in cells] finite = [value for value in values if value is not None] if not all(math.isfinite(value) for value in finite): @@ -2185,9 +2295,35 @@ def _as_numbers(column: _Column, cells: list[str]) -> bool: column.dtype = "number" column.cells = ["" if value is None else repr(value) for value in values] column.numbers = finite + column.integers = [] + column.rounded = sum( + 1 for cell, value in zip(cells, values, strict=True) if value is not None and not keeps_value(cell, value) + ) return True +_FLOAT_DIGITS = 15 +"""Every decimal of at most this many significant digits survives a float64 and its `repr` (`DBL_DIG`).""" + + +def keeps_value(text: str, value: float) -> bool: + """Whether the finite float `value` read from the dot-decimal number `text` has exactly the value of `text`. + + `1.50` and `1.5` are the same value, so are `1e3` and `1000.0`; `0.1234567890123456789` + (19 digits) and `9007199254740993` (2**53 + 1) are not, since a float64 keeps 15 to 17 + significant digits. A short number is decided by its digit count; a long one, or one so + small that the float is subnormal, by comparing the decimal values exactly. + """ + mantissa = text.lower().partition("e")[0] + digits = mantissa.lstrip("+-").replace(".", "").lstrip("0") + if len(digits) <= _FLOAT_DIGITS and (abs(value) >= sys.float_info.min or (value == 0 and not digits)): + return True + try: + return Decimal(text) == Decimal(repr(value)) + except InvalidOperation: + return False + + def _match_all(pattern: re.Pattern[str], cells: list[str]) -> dict[str, tuple[int, ...]] | None: """Each distinct cell's date and time fields as integers (an absent time is 0), or None on a miss.""" fields: dict[str, tuple[int, ...]] = {} @@ -2267,12 +2403,17 @@ def _build( cells: list[str] = [] na_rows: list[int] = [] mark_rows: list[int] = [] + trimmed = 0 + tokens: Counter[str] = Counter() for row_index, row in enumerate(raw_rows): - text, cell_invisible = _clean_key(row[position]) + text, cell_invisible, cell_trimmed = _cleaned(row[position]) + trimmed += cell_trimmed if text_only or canonical: cell = text else: cell = "" if text in MISSING_VALUES else text + if text and not cell: + tokens[text] += 1 if text == NA_TOKEN: na_rows.append(row_index) if missing_marks and _matches_mark(text, missing_marks): @@ -2289,6 +2430,8 @@ def _build( text_only=text_only, na_rows=na_rows, mark_rows=mark_rows, + trimmed=trimmed, + missing_tokens=tokens, ) read.append((header, broken, column)) @@ -2347,6 +2490,7 @@ def _build( _retype_thousands_dots(column, decimal_comma=decimal_comma) for column in columns: _keep_na_codes(column) + _fidelity_notes(column, layout_padding=source_format == "whitespace") # A header that already names the unit (`Anteil (%)`, `Preis in €`) gets no suffix, unless it is # longer than a name may be: then the suffix is passed so the cut keeps the unit at the end. @@ -2455,9 +2599,33 @@ def _keep_na_codes(column: _Column) -> None: return for row in column.na_rows: column.cells[row] = NA_TOKEN + column.missing_tokens.pop(NA_TOKEN, None) column.notes.append(f"{NA_TOKEN} read as a code (such as Namibia or North America), not as a missing cell") +def _fidelity_notes(column: _Column, *, layout_padding: bool) -> None: + """The column's notes for every cell the parse changed beyond its spelling (owner rule of 2026-10-15). + + A number keeps its value and a date its moment (`1.50` is written `1.5`, `5.3.2024` + `2024-03-05`): those are the canonical spelling, not a change. What is a change gets + a note with its count: a number stored rounded (`_as_numbers`); in a text column, a + cell whose surrounding whitespace was removed (not in a space-aligned table, whose + padding is layout) and a token such as `None` or `-` read as missing, since there it + could have been a category. In number and date columns those tokens are the usual + marks of a missing value, and whitespace is no part of a number. + """ + if column.dtype == "number" and column.rounded: + column.notes.append(rounded_note(column.rounded)) + if column.dtype != "text": + return + if column.trimmed and not layout_padding: + column.notes.append(f"spaces around the text removed in {_count(column.trimmed, 'cell')}") + if column.missing_tokens: + total = sum(column.missing_tokens.values()) + named = ", ".join(f"'{token}'" for token, _ in column.missing_tokens.most_common(3)) + column.notes.append(f"{_count(total, 'cell')} read as missing ({named})") + + def _without_empty_columns( read: list[tuple[str, bool, _Column]], warnings: list[str] ) -> list[tuple[str, bool, _Column]]: @@ -2532,12 +2700,16 @@ def _constants_to_notes( def _profile_column(column: _Column) -> ColumnProfile: present = [cell for cell in column.cells if cell] - minimum: float | str | None = None - maximum: float | str | None = None + minimum: int | float | str | None = None + maximum: int | float | str | None = None mean: float | None = None median: float | None = None top: list[str] = [] - if column.dtype in ("integer", "number") and column.numbers: + if column.dtype == "integer" and column.integers: + # Exact: a float64 would round an id or a count past 2**53 in the profile the root reads. + minimum, maximum = min(column.integers), max(column.integers) + mean, median = _statistics(column.numbers) + elif column.dtype in ("integer", "number") and column.numbers: minimum, maximum = min(column.numbers), max(column.numbers) mean, median = _statistics(column.numbers) elif column.dtype == "datetime" and column.dates: diff --git a/agents/anyplot/data/prepare.py b/agents/anyplot/data/prepare.py index b689eb267e5..5c99218ea96 100644 --- a/agents/anyplot/data/prepare.py +++ b/agents/anyplot/data/prepare.py @@ -22,7 +22,9 @@ decimal (`12,5` is no dot-decimal number, never 125). When cells fail with `decimal` and fewer fail with the other mark, the whole column is read with the other mark and the change line says so ("read as numbers (decimal comma)"); the unit must be the - same wherever it occurs. To datetime: with `date_format` the cells are read in + same wherever it occurs. Whole numbers are read exactly (`Int64`, never through a + float64), and the change line counts the cells a float64 holds only rounded (more + than 15 to 17 significant digits). To datetime: with `date_format` the cells are read in exactly that format, with an optional time (`HH:MM[:SS]`) after it (a format without a day, such as `YYYY-MM` or `YYYY`, reads the first day of the month or the year; `YYYY` on a number column says "read as years", since a measure of 1450 to 2100 reads @@ -35,7 +37,10 @@ only 1677 to 2262; a date pandas still cannot hold is an `invalid_operation`, never a crash). Without `date_format` pandas infers the format per cell, day before month when ambiguous, as the parser - reads a slash date. Either way a cell that cannot be read becomes missing and the + reads a slash date, except in a column whose every cell starts with a year + (`2024-06-01T13:00:00Z` is 1 June); a fraction of a second and a UTC offset are kept + (`moment_text`), and different offsets in one column are a `type_mismatch`, never a + silent conversion to one time zone. Either way a cell that cannot be read becomes missing and the change line counts it; when more than 10 % of the non-missing cells fail, the operation is a `type_mismatch` instead. To text: every cell becomes its canonical text and the column is kept text through the re-parse (`Prepared.text_columns`, @@ -85,9 +90,11 @@ rows with the same group and order are an `invalid_operation`, since "previous" would be a guess (aggregate first); a row without an order value gets an empty cell and takes no place in the order. `percent` multiplies a `ratio` or a `share` by 100; - a percent change is always in percent. A difference or a sum of whole-number columns - is computed in Python integers and stays whole (a float64 holds integers only up to - 2**53). A division by zero or a missing input gives a + a percent change is always in percent. A difference, a sum or a `cumsum` of + whole-number columns is computed in Python integers and stays whole (a float64 holds + integers only up to 2**53), as does an `aggregate` `sum` (pandas' `Int64` sum wraps + around past 2**63); a result past int64 makes the column float64, and the change line + counts the values it rounds (`whole_rounded`). A division by zero or a missing input gives a missing cell, and the change line counts them. A question such as "which year has the largest share of X" takes the same operations through `query_data` (`query.py`), on a copy that is never stored (live tests of 2026-10-14 and 2026-10-15). @@ -138,9 +145,10 @@ import re import unicodedata import warnings -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from dataclasses import dataclass, field, replace from datetime import datetime +from decimal import Decimal as ExactDecimal from typing import Annotated, Literal import numpy as np @@ -155,7 +163,9 @@ MISSING_VALUES, ParsedDataset, canonical_names, + keeps_value, pandas_dtypes, + rounded_note, ) @@ -729,7 +739,7 @@ def _set_type(frame: pd.DataFrame, op: SetType, carried: _Carried) -> tuple[pd.D # The canonical text of a number is a dot decimal; re-reading it under a decimal comma would be wrong. return frame, f"{op.column}: already numbers, nothing changed" requested = op.decimal or carried.decimal - converted, unit, failed, used = _to_numbers(series, requested) + converted, unit, failed, used, rounded = _to_numbers(series, requested) _check_failed(op.column, failed, present, "numbers") result[op.column] = converted carried.with_time.pop(op.column, None) @@ -739,6 +749,8 @@ def _set_type(frame: pd.DataFrame, op: SetType, carried: _Carried) -> tuple[pd.D change += " (decimal comma)" if used == "," else " (decimal point)" if unit: change += f", unit '{_clip(unit, MAX_CELL_CHARS)}' removed" + if rounded: + change += f", {rounded_note(rounded)}" else: if kind == "datetime": return _reread_dates(frame, op, present, carried) @@ -822,7 +834,13 @@ def _aggregate(frame: pd.DataFrame, op: Aggregate, carried: _Carried) -> tuple[p if value_kind == "boolean" and op.how in ("sum", "mean"): work[op.column] = work[op.column].astype("Int64") grouped = work.groupby(keys, dropna=False, sort=True)[op.column] + rounded = 0 match op.how: + case "sum" if is_integer_dtype(work[op.column].dtype): + # In Python integers: pandas' Int64 sum wraps around silently past 2**63 (review of 2026-10-15). + # Summed outside pandas, which would infer a dtype for the totals; the groups come in the order of `size`. + totals = [sum(int(value) for value in values.dropna()) for _, values in grouped] + aggregated, rounded = _whole_numbers(totals, grouped.size().index) case "sum": aggregated = grouped.sum() case "mean": @@ -844,6 +862,8 @@ def _aggregate(frame: pd.DataFrame, op: Aggregate, carried: _Carried) -> tuple[p other = frame.shape[1] - len(keys) - 1 if other: change += f", {_count(other, 'other column')} dropped" + if rounded: + change += f", {whole_rounded(rounded)}" return result, change @@ -879,9 +899,15 @@ def _pivot_longer(frame: pd.DataFrame, op: PivotLonger, carried: _Carried) -> tu kinds = {_kind(frame[name]) for name in columns} times = [_with_time(frame[name], carried) for name in columns] work = frame + rounded = 0 if len(kinds) > 1: # Unlike kinds melt into one text column: each column's canonical texts, a datetime with its time. work = frame.assign(**{name: _texts(frame[name], time) for name, time in zip(columns, times, strict=True)}) + elif kinds == {"number"} and len({is_integer_dtype(frame[name].dtype) for name in columns}) > 1: + # Whole numbers beside decimals share one float64 column, which rounds a whole number past 2**53: counted. + whole = [name for name in columns if is_integer_dtype(frame[name].dtype)] + rounded = sum(int(value) != int(float(value)) for name in whole for value in frame[name].dropna()) + work = frame.assign(**{name: frame[name].astype("float64") for name in whole}) melted = work.melt(id_vars=rest, value_vars=columns, var_name=op.names_to, value_name=op.values_to) # The melted column stays text through the re-parse when a source was kept text by request # (ids that look like numbers) or unlike kinds were written as text: by content it could type again. @@ -891,7 +917,13 @@ def _pivot_longer(frame: pd.DataFrame, op: PivotLonger, carried: _Carried) -> tu carried.text_columns.add(op.values_to) if kinds == {"datetime"}: carried.with_time[op.values_to] = any(bool(time) for time in times) - return melted, f"pivoted {len(columns)} columns into {op.names_to}/{op.values_to}: {_count(rows, 'row')}" + change = f"pivoted {len(columns)} columns into {op.names_to}/{op.values_to}: {_count(rows, 'row')}" + if rounded: + change += ( + f", {_count(rounded, 'whole number')} past 2^53 {_be(rounded)} stored rounded to the nearest number " + f"beside the decimals of {op.values_to}" + ) + return melted, change def _pivot_wider(frame: pd.DataFrame, op: PivotWider, carried: _Carried) -> tuple[pd.DataFrame, str]: @@ -1013,8 +1045,10 @@ def _derive(frame: pd.DataFrame, op: Derive, carried: _Carried) -> tuple[pd.Data raise PrepareError("type_mismatch", f"column '{name}' is {kind}; derive {op.how} needs number columns") if op.how in ("difference", "sum") and all(is_integer_dtype(frame[name].dtype) for name in columns): # Whole numbers stay exact: a float64 holds integers only up to 2**53. - result = _integer_result(frame, columns, op.how) + result, rounded = _integer_result(frame, columns, op.how) what = f"{columns[0]} - {columns[1]}" if op.how == "difference" else f"sum of {', '.join(columns)}" + if rounded: + what += f", {whole_rounded(rounded)}" return _derived(frame, op, result, what, "", carried) values = [frame[name].astype("float64") for name in columns] note = "" @@ -1042,6 +1076,8 @@ def _derive(frame: pd.DataFrame, op: Derive, carried: _Carried) -> tuple[pd.Data note = f"a row without a {op.order}, " if op.how == "percent_change": note = "the first row of each group, " + note + if is_integer_dtype(result.dtype): # an exact running total of whole numbers (`_integer_cumsum`) + return _derived(frame, op, result, what, note, carried) if op.percent: result = result * 100 what += ", in percent" @@ -1065,8 +1101,8 @@ def _derived( return frame.assign(**{op.name: result}), change -def _integer_result(frame: pd.DataFrame, columns: list[str], how: str) -> pd.Series: - """A difference or a row sum of whole-number columns in Python integers: `Int64`, or float64 past its range.""" +def _integer_result(frame: pd.DataFrame, columns: list[str], how: str) -> tuple[pd.Series, int]: + """A difference or a row sum of whole-number columns in Python integers, see `_whole_numbers`.""" rows = zip(*([None if pd.isna(value) else int(value) for value in frame[name]] for name in columns), strict=True) results: list[int | None] = [] for row in rows: @@ -1075,9 +1111,51 @@ def _integer_result(frame: pd.DataFrame, columns: list[str], how: str) -> pd.Ser results.append(None if len(present) < 2 else present[0] - present[1]) else: results.append(sum(present) if present else None) + return _whole_numbers(results, frame.index) + + +def _whole_numbers(results: Sequence[int | None], index: pd.Index) -> tuple[pd.Series, int]: + """Exact whole-number results as `Int64`; past its range float64, with the count of values it rounds. + + A float64 holds whole numbers exactly only up to 2**53, so once one result leaves the + int64 range the column is float64 and every result past 2**53 is rounded: the count + goes into the change line (`whole_rounded`), never silently (owner rule of 2026-10-15). + """ if all(value is None or _INT64_MIN <= value <= _INT64_MAX for value in results): - return pd.Series(pd.array(results, dtype="Int64"), index=frame.index) - return pd.Series([np.nan if value is None else float(value) for value in results], index=frame.index) + return pd.Series(pd.array(results, dtype="Int64"), index=index), 0 + floats = [np.nan if value is None else float(value) for value in results] + rounded = sum( + 1 for value, number in zip(results, floats, strict=True) if value is not None and int(number) != value + ) + return pd.Series(floats, index=index, dtype="float64"), rounded + + +def whole_rounded(count: int) -> str: + """The change-line clause of whole-number results that left the int64 range and are stored rounded.""" + return ( + f"{_count(count, 'value')} past the 64-bit whole-number range {_be(count)} stored rounded " + "to the nearest number (15 to 17 significant digits)" + ) + + +def _integer_cumsum(frame: pd.DataFrame, column: str, keys: list[str], ordered: pd.Index) -> tuple[pd.Series, int]: + """The running total of a whole-number column in Python integers, in the order `ordered`, within each group. + + A missing cell adds nothing and keeps an empty cell, as pandas' `cumsum` does; a row + outside `ordered` (no order value) stays empty. + """ + wholes = {row: None if pd.isna(value) else int(value) for row, value in frame[column].items()} + totals: dict[tuple[str | None, ...], int] = {} + running: dict[object, int | None] = {} + for row in ordered: + whole = wholes[row] + if whole is None: + running[row] = None + continue + group = tuple(_cell_text(frame.at[row, key], True) for key in keys) + totals[group] = totals.get(group, 0) + whole + running[row] = totals[group] + return _whole_numbers([running.get(row) for row in frame.index], frame.index) def _ordered( @@ -1113,8 +1191,12 @@ def _ordered( result = (current - previous) / previous.where(previous != 0) * 100 what = f"percent change of {column} from the previous {order}{within}" elif how == "cumsum": - result = current.groupby(groups, dropna=False).cumsum() if keys else current.cumsum() what = f"running total of {column} in the order of {order}{within}" + if is_integer_dtype(frame[column].dtype): + # Whole numbers add up exactly: a float64 running total rounds past 2**53. + exact, rounded = _integer_cumsum(frame, column, keys, ordered) + return exact, what + (f", {whole_rounded(rounded)}" if rounded else "") + result = current.groupby(groups, dropna=False).cumsum() if keys else current.cumsum() else: if keys: # `first` takes the group's first value that is there base = current.groupby(groups, dropna=False).transform("first") @@ -1364,8 +1446,9 @@ def _bools(mask: pd.Series) -> pd.Series: # --- Conversions ----------------------------------------------------------------------------- -def _to_numbers(series: pd.Series, decimal: str) -> tuple[pd.Series, str | None, int, str]: - """The cells as numbers (`Int64` when every value is whole, else `float64`), the unit removed, the failures, the decimal used. +def _to_numbers(series: pd.Series, decimal: str) -> tuple[pd.Series, str | None, int, str, int]: + """The cells as numbers (`Int64` when every value is whole, else `float64`), the unit removed, the failures, the decimal used, + and the cells a float64 holds only rounded (more significant digits than it keeps, see `parse.keeps_value`). The cells are read with `decimal` first. When some fail and fewer fail with the other mark, the column is written that way (a `12,5 %` column in a dot-decimal @@ -1373,44 +1456,51 @@ def _to_numbers(series: pd.Series, decimal: str) -> tuple[pd.Series, str | None, one decimal per column, never a mix. """ if _kind(series) == "boolean": - return series.astype("Int64"), None, 0, decimal + return series.astype("Int64"), None, 0, decimal, 0 texts = [text if isinstance(text, str) else None for text in _texts(series)] - values, units, failed = _read_numbers(texts, decimal) + numbers, units, failed = _read_numbers(texts, decimal) if failed: other = "," if decimal == "." else "." alternative = _read_numbers(texts, other) if alternative[2] < failed: - (values, units, failed), decimal = alternative, other + (numbers, units, failed), decimal = alternative, other if len(units) > 1: raise PrepareError("type_mismatch", f"column '{series.name}' mixes {len(units)} different units") unit = next(iter(units)) if units else None - whole = all(value is None or (value.is_integer() and _INT64_MIN <= value <= _INT64_MAX) for value in values) - if whole: - integers = [None if value is None else int(value) for value in values] - return pd.Series(pd.array(integers, dtype="Int64"), index=series.index), unit, failed, decimal - return pd.Series(values, index=series.index, dtype="float64"), unit, failed, decimal - - -def _read_numbers(texts: list[str | None], decimal: str) -> tuple[list[float | None], set[str], int]: - """Every cell read with `decimal`: the values (None for missing and failed cells), the units seen, the failures.""" - values: list[float | None] = [] + # Whole numbers are read exactly, never through a float64, which rounds past 2**53 (`9007199254740993`). + exact = [None if number is None else ExactDecimal(number) for number in numbers] + integers = [None if value is None or value != value.to_integral_value() else int(value) for value in exact] + if all( + number is None or (integer is not None and _INT64_MIN <= integer <= _INT64_MAX) + for number, integer in zip(numbers, integers, strict=True) + ): + return pd.Series(pd.array(integers, dtype="Int64"), index=series.index), unit, failed, decimal, 0 + values = [None if number is None else float(number) for number in numbers] + rounded = sum(not keeps_value(number, float(number)) for number in numbers if number is not None) + return pd.Series(values, index=series.index, dtype="float64"), unit, failed, decimal, rounded + + +def _read_numbers(texts: list[str | None], decimal: str) -> tuple[list[str | None], set[str], int]: + """Every cell read with `decimal`: the dot-decimal numbers (None for missing and failed cells), the units seen, + the failures.""" + numbers: list[str | None] = [] units: set[str] = set() failed = 0 for text in texts: if text is None: - values.append(None) + numbers.append(None) continue number, unit = _read_number(text, decimal) if number is None: failed += 1 elif unit: units.add(unit) - values.append(number) - return values, units, failed + numbers.append(number) + return numbers, units, failed -def _read_number(text: str, decimal: str) -> tuple[float | None, str | None]: - """One cell as a finite number and the unit token taken off it; `(None, None)` when it cannot be read.""" +def _read_number(text: str, decimal: str) -> tuple[str | None, str | None]: + """One cell as a finite dot-decimal number and the unit token taken off it; `(None, None)` when it cannot be read.""" body, unit = text.strip(), None if _plain_number(body, decimal) is None: match = _TRAILING_UNIT.match(body) or _LEADING_UNIT.match(body) @@ -1421,12 +1511,14 @@ def _read_number(text: str, decimal: str) -> tuple[float | None, str | None]: return number, (unit if number is not None else None) -def _plain_number(body: str, decimal: str) -> float | None: - """`body` read with `decimal` as the parser reads a number; None when it is no finite number that way. +def _plain_number(body: str, decimal: str) -> str | None: + """`body` read with `decimal` as the parser reads a number, as dot-decimal text without groups; None when it is + no finite number that way. Spaces and apostrophes separate thousands. The other mark does so only between three-digit groups (`_GROUPED`); anywhere else it makes the cell unreadable with this decimal (`12,5` is no dot-decimal number), so a decimal mark is never dropped. + The text, not a float, goes back: the caller reads whole numbers exactly. """ thousands = "." if decimal == "," else "," cleaned = _SPACES.sub("", body).replace("'", "") @@ -1438,8 +1530,7 @@ def _plain_number(body: str, decimal: str) -> float | None: cleaned = cleaned.replace(",", ".") if not _NUMBER.fullmatch(cleaned): return None - number = float(cleaned) - return number if math.isfinite(number) else None + return cleaned if math.isfinite(float(cleaned)) else None def _to_dates(series: pd.Series, date_format: str | None) -> tuple[pd.Series, int, bool]: @@ -1454,9 +1545,22 @@ def _to_dates(series: pd.Series, date_format: str | None) -> tuple[pd.Series, in values = [_read_date(text, date_format) if isinstance(text, str) else None for text in texts] result = pd.Series(values, index=series.index, dtype="datetime64[us]") else: + # A year-first cell (`2024-06-01T13:00:00Z`) reads year, month, day: with `dayfirst` pandas would read it as + # 6 January (review of 2026-10-15). Day before month is for the other shapes only. + present = [text for text in texts if isinstance(text, str)] + day_first = not (present and all(_YEAR_FIRST.match(text) for text in present)) with warnings.catch_warnings(): warnings.simplefilter("ignore") - result = pd.to_datetime(texts, errors="coerce", format="mixed", dayfirst=True) + try: + result = pd.to_datetime(texts, errors="coerce", format="mixed", dayfirst=day_first) + except ValueError: + # Different UTC offsets (summer and winter time) or times with and without one: one column needs + # one time zone, and converting them would change the clock times without a word (review of 2026-10-15). + raise PrepareError( + "type_mismatch", + f"column '{series.name}' holds times with different UTC offsets (or some without one); " + "they are not converted to one time zone, so the column stays as it is", + ) from None failed = int((result.isna() & texts.notna()).sum()) with_time = any(":" in text for text in texts if isinstance(text, str)) return result, failed, with_time @@ -1588,10 +1692,23 @@ def _cell_text(value: object, with_time: bool) -> str | None: number = float(value) return None if math.isnan(number) else repr(number) if isinstance(value, pd.Timestamp): - return value.isoformat(timespec="seconds") if with_time else value.date().isoformat() + return moment_text(value) if with_time else value.date().isoformat() return str(value) +def moment_text(value: pd.Timestamp) -> str: + """A date-time as ISO text with every digit it holds: seconds, or the fraction of a second it has. + + `2024-01-05T10:00:00`, `2024-01-05T10:00:00.250000` (any fraction is written in + microseconds, as the file converters write it), `…T10:00:00.000000001` for a + nanosecond; an offset stays (`+02:00`). Writing seconds only would drop a sub-second + reading without a word (owner rule of 2026-10-15). + """ + if value.nanosecond: + return value.isoformat(timespec="nanoseconds") + return value.isoformat(timespec="microseconds" if value.microsecond else "seconds") + + def _require(frame: pd.DataFrame, name: str) -> None: if name not in frame.columns: raise PrepareError("unknown_column", f"there is no column '{name}'") diff --git a/agents/anyplot/schemas.py b/agents/anyplot/schemas.py index 76813c94982..2f8503ee412 100644 --- a/agents/anyplot/schemas.py +++ b/agents/anyplot/schemas.py @@ -191,8 +191,8 @@ class ColumnProfile(_ServerContract): dtype: ColumnDtype missing: NonNegativeInt unique: NonNegativeInt - min: float | Cell | None = None - max: float | Cell | None = None + min: int | float | Cell | None = None # an integer column's exactly: a float rounds past 2**53 + max: int | float | Cell | None = None mean: float | None = None median: float | None = None top: list[Cell] = Field(default_factory=list, max_length=MAX_TOP_VALUES) diff --git a/agents/evals/stress.py b/agents/evals/stress.py index 972b15001c8..ba652008a82 100644 --- a/agents/evals/stress.py +++ b/agents/evals/stress.py @@ -54,7 +54,7 @@ from agents.anyplot.briefs import adapter_profile, profile_json, profile_summary, spec_brief from agents.anyplot.code.edits import apply_plan, new_literal_chars -from agents.anyplot.code.loader import loader_arguments, na_values +from agents.anyplot.code.loader import ROUND_TRIP, loader_arguments, na_values, round_trip from agents.anyplot.code.regions import find_regions from agents.anyplot.data.bindings import check_bindings, default_bindings from agents.anyplot.data.parse import MAX_INPUT_BYTES, ParsedDataset, ParseError, parse_dataset @@ -171,7 +171,12 @@ def loader_outcome(parsed: ParsedDataset) -> tuple[str, float]: try: missing = na_values(dtypes, dates) frame = pd.read_csv( - io.StringIO(parsed.csv), dtype=dtypes, parse_dates=dates, keep_default_na=missing is None, na_values=missing + io.StringIO(parsed.csv), + dtype=dtypes, + parse_dates=dates, + keep_default_na=missing is None, + na_values=missing, + float_precision=ROUND_TRIP if round_trip(dtypes) else None, ) except Exception as exc: # the class is the finding; the message could quote the data return type(exc).__name__, (time.perf_counter() - started) * 1000 diff --git a/changelog.d/agent-data-fidelity.md b/changelog.d/agent-data-fidelity.md new file mode 100644 index 00000000000..7a87189e596 --- /dev/null +++ b/changelog.d/agent-data-fidelity.md @@ -0,0 +1,17 @@ +### Fixed + +- **Pasted data keeps every digit, or a warning says what changed.** Owner rule of + 2026-10-15: precision is never lost silently. The exported `plot.py` and the render + read `data.csv` with `float_precision="round_trip"` (pandas' default reader had + misread about one in four 17-digit numbers); a number with more digits than a float + holds, from CSV, JSON or a Python literal, is stored rounded with a warning that + counts the cells; whole numbers past the 64-bit range stay text; a text cell that + lost surrounding spaces or became missing (`None`, `-`), and NFC composition, are + counted in warnings; the profile's integer `min` and `max` are exact. +- **Data operations no longer round or wrap numbers without a word.** `set_type number` + reads whole numbers exactly; an `aggregate` sum of whole numbers no longer wraps + around past 2^63; `cumsum`, row sums and differences stay whole; `pivot_longer` + counts the whole numbers its shared column rounds. Date-times keep their fraction of a + second and their UTC offset, a year-first time such as `2024-06-01T13:00:00Z` is no + longer read as 6 January, and a column with different UTC offsets is refused instead + of crashing the operation. diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 2d980a94947..fc1ea561602 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -208,7 +208,7 @@ Principles: Built since 2026-10-11 (the gate): `POST /v1/sessions/{sid}/dataset/confirm` stores a fingerprint of the dataset id and the bindings as the user saw them (`anyplot.confirmed`, `confirmation_fingerprint`); the pipeline runs only while the fingerprint of the current state equals it. A new dataset gets a new id, so a paste or a transcription's `.use()` drops the confirmation by itself, and so does any binding change through the `PUT` route or the root's `set_bindings` tool, also one that lands between the confirmation's read and its write, because the stored fingerprint then names bindings the session no longer holds; a library switch keeps the data, the bindings and the confirmation. Since 2026-10-15 (live test round 2) there is one exception for the root's `set_bindings`: bindings that only leave out members of a confirmed number family (`y2`, `y3`, …), every other binding exactly as confirmed, keep the confirmation (`session_state.narrows` and `narrowed_confirmation`, `AGENT_NARROWED_BINDINGS=keep`; `reset` restores the old rule; owner decision of 2026-10-15: "e.g. drops the 2nd line"), so a step-2 note such as "only co2_per_capita, title …" plots in the same turn, and the tool result says `data_confirmed`. Leaving out a grouping, text or boolean role such as `series` (which merged two countries into one line in the review) or any single role, or renumbering a member so that `y1` plots another column, asks for `.confirm()` again. A first plot's `change_request` that stops on `data_unconfirmed` or `incomplete_bindings`, or gets no queue slot, is kept in `anyplot.pending_request`, its `expected_texts` with it in `anyplot.pending_expected_texts`, and the bindings of that moment in `anyplot.pending_bindings` (`pipeline.keep_pending_request`, `hold_request`). A bare `.create_plot()` checks the render for the kept texts; a run that brings a request or texts of its own checks only its own, since the root never sees the kept ones and a withdrawn title must not count as a miss (`chosen_texts`); stored texts are read through the `PipelineArgs` validator (`kept_texts`). A binding write that keeps every binding of that moment and only fills roles open then (the step `incomplete_bindings` waits for) keeps the note (`session_state.fills_open_roles`); a change of an existing version neither takes nor clears it: the next first-plot run joins it to its own request (`merged_request`, at most 600 characters, the older note cut with `…` rather than dropped), and it stays until a run ships a plot (`settle_pending_request`), so a failed run loses nothing. Any binding write (the root's or the panel's), any new data version (a paste, `prepare_data`, `generate_data`, `run_data_code`, `.to_long()`, undo) and a library switch drop it, since it was written for the data and bindings before. The session block tells the root whether the data is confirmed, and the root answers `data_unconfirmed` by asking the user to confirm in the data panel, which it cannot do itself. The page shows a step header `1 data · 2 plot`, `.confirm()` after the bindings, and `.create_plot()`, which starts a run only after the confirmation (before it, the button explains the step instead of calling the server); the eval harness and the development fixture confirm on the user's behalf, because their data is the developer's own. -Built since 2026-10-11 (the operations): the root tool `prepare_data(operations)` with at most five operations per call, each a Pydantic model of the fixed set in `agents/anyplot/data/prepare.py`: `rename` (column, name), `drop_columns` (1 to 10 columns), `drop_rows` (`where` conditions on one column each, all of which must hold: `eq`, `ne`, `contains`, `is_missing`, `gt`, `lt`, at most three; a value is compared, never evaluated), `set_type` (number, datetime or text, with the date format and the decimal separator; to number is for the columns the profile still shows as text, because a column whose every cell carries the same unit is already numbers after the paste, the unit in its name such as `Anteil (%)`: one unit such as `%` or `EUR` is removed and the cells are read as the parser reads a number, the other mark a thousands separator between three-digit groups only, and when more cells read with the other decimal mark the whole column is read that way and the change line says so; a cell that cannot be read becomes missing, more than 10 % of them is a `type_mismatch`; to text the column stays text through the re-parse of every later version, stored with the version as `text_columns`; `MM/DD/YYYY` on a column that is already dates swaps day and month only for a slash column the parser flagged as ambiguous), `aggregate` (by one or two columns, a value column, `sum`, `mean`, `count`, `min` or `max`, and a `month`, `quarter` or `year` period for a datetime key), `sort`, `pivot_longer` (2 to 20 columns into a name/value pair; a long format over the 2 MB cap of computed data, `MAX_COMPUTED_BYTES`, is an `invalid_operation`; 200 KB until 2026-10-14) and `pivot_wider` (the distinct values of a column become at most 50 columns; duplicate pairs are refused, never aggregated silently), and, since 2026-10-12, `classify` (a new text column whose cells are labels: each row gets the label of the first of 1 to 10 rules whose `drop_rows`-style conditions all hold, else `otherwise` or a missing cell; labels are at most 40 characters, never a token the parser reads as a missing cell such as `NA` or `-`, exempt from ToolSafety's link rule like a rename's name, and the column stays text through every re-parse). `classify` came from the owner's case of 2026-10-12: a generated sine table on `bar-feature-importance` had no text column for the `feature` role, and "positive is class 1, negative class 2" had no operation, so the root had to refuse. Since 2026-10-14, `derive` adds a number column from number columns by one formula of a closed set: `ratio` (`a / b`) and `difference` (`a - b`) of two columns, `sum` of 2 to 10 columns in the same row, `share` of one column in the total of its group (`within` 0 to 2 key columns), and three formulas that walk each group in the order of `order`, a number or datetime column: `percent_change` from the previous row, `cumsum` (the running total) and `index` (the group's first row = 100; the last two by owner decision of 2026-10-14). A text order column is a `type_mismatch` with a hint to `set_type` it, because it would sort alphabetically (`April` before `Januar`, `10` before `9`); repeated order values within a group are refused (aggregate first), and a row without an order value keeps an empty cell; `percent` multiplies a ratio or a share by 100, and a division by zero or a missing input leaves the cell empty, which the change line counts. It came from the owner's live test of 2026-10-14: "which year has the largest share of foreigners?" on a long table (year, origin, count), where the root could only ask the user to compute the column; now it derives the share, drops the other rows, sorts, and answers from the new profile. Since 2026-10-15, `make_date` adds a date column built from whole-number part columns of the same row: a year (four digits), then a month (1 to 12) and optionally a day, or a quarter (1 to 4), or an ISO week (1 to 53), a part left out being its first (a year and a month give the first day of each month); a day without a month, a quarter or a week beside another part, a value out of its range and a row that is no calendar date (31 April) are refused, never moved; a text month column may hold month names and abbreviations in the eleven refusal languages (en, de, fr, es, it, pt, nl, pl, zh, vi, ru), compared without case and diacritics, an unknown name refusing it (owner decision of 2026-10-15); ISO week 1 can start in late December, which the change line says. `set_type` to datetime gained the formats `DD/MM/YYYY`, `YYYYMMDD` and `YYYYMM` (also on a number column), `YYYY-MM`, `MM/YYYY`, `MM.YYYY` and `YYYY`; the compact ones take exactly 8, 6 or 4 digits (`2024115` is refused, never guessed), `YYYY` on numbers says "read as years", and `MM.YYYY` on a column the parser read as numbers (`03.2020` as 3.202) pads four decimals only when every cell then is a month of a year 1000 to 2200, at most 300 years apart, so a column of ordinary decimals (`2.2`, `3.5`) is refused. Dates are read with Python's parser and stored at microseconds, so years outside pandas' nanosecond range (1677 to 2262) no longer crash the tool. A date column whose every day is at most 12 keeps its day and month order open from version to version (a warning carried by `with_date_notes`), and `set_type` swaps it in either direction: `MM/DD/YYYY` on a column read day first, `DD/MM/YYYY` or `DD.MM.YYYY` on one read month first. All this came from live test round 3 and its review: a MeteoSwiss table of `Year` and `Month` had no date for `line-timeseries-rolling`, the root tried `run_data_code`, which cannot read the user's table, and then asked the user to add the column by hand; the root prompt now names the operation and forbids handing data work back when an operation can do it. A date column the change made (a datetime column the version before did not have) fills an open required date role in `carry_bindings` (owner decision of 2026-10-15: code, not the model). No expression language (an injection surface), no joins, no model-guessed fills: the operations only move, type, aggregate, drop, label or compute from cells that are there, and `ToolSafety` validates the list against the same models before the tool runs. The server applies them with pandas to the stored canonical CSV, re-parses the result with `parse_dataset` (the dataset judge does not run again: the only new text is a rename's name or a `classify` label, which the model writes from a user message the scope judge already checked) and stores it as the next dataset version (`agents/anyplot/dataset_versions.py`): the previous version joins `anyplot.dataset_history`, which keeps the last five (older ones leave the store; the idle sweep takes a session's history only together with its current version), `anyplot.dataset_version` counts up from the paste's 1, the bindings are carried over (`carry_bindings`: the ones still valid stay, the roles left without one get the defaults), and the confirmation drops by itself because its fingerprint names the dataset id. The tool answers the change lines (one English line per operation with its counts, fenced as ``) and the new profile (fenced as ``), or a `PrepareError` code (`unknown_column`, `type_mismatch`, `empty_result`, `too_wide`, `invalid_operation`) with a message that names columns at most. The page learns about a changed dataset from the `done` event: after a turn that changed the dataset id it carries `dataset: {preview, profile, bindings, warnings, roles, changes, version, context_lines}`, the dataset route's shape plus the turn's change lines (`roles` since the formulas below), and `data_confirmed: false`; the BFF relays a `done` of up to 512 KiB whole (every other event stops at 64 KiB), which the largest dataset the parser accepts fits (`agents/stream.py`, `MAX_DONE_EVENT_CHARS`). `POST /v1/sessions/{sid}/dataset/undo` (the user's `.undo()` button; the model has no undo) restores the newest history entry as the current dataset, bindings carried over, unconfirmed again, and answers like the dataset route (`422 nothing_to_undo` for a paste that was never changed). The step-1 summary is the action `{"action": "summarize_data"}` on the messages route (judge skipped, like `create_plot`; since 2026-10-14 the turn also carries the notes pasted with the table, already judged at the paste, after a preamble and fenced as ``, and `get_dataset_profile` returns them under `user_notes` for later turns, with a fixed session-block line saying they exist, so the root uses them as context for the whole chat and acts on a clear request in them at once, the plot itself still waiting for the user's confirmation): the root reads the profile, which now carries `mean` and `median` for numeric columns, and answers in two to four sentences (rows, columns with types and units, missing values, values that look like percentages, the proposed bindings, one question when something is ambiguous), never claiming the data is right: the user confirms. The live eval `agents/evals/prep.py` scores the root's choices on `agents/evals/fixtures/prep/` by exact operation match, order-insensitive, with an 80 % gate (see [Run the prep eval](../../agents/README.md#run-the-prep-eval)). The page asks for the summary right after every parse that stores a dataset (a paste, or a transcription's `.use()`; never after an undo or a library switch), so it is the first turn in the thread, and the composer takes data requests in step 1. A `done` with `dataset` replaces the preview, the profile and the bindings, takes the page back to step 1, and shows a `# changed` block above the preview with the change lines and `.undo()`; after an undo the block names the restored version and keeps `.undo()` while the version is above 1, until the service answers `nothing_to_undo`. +Built since 2026-10-11 (the operations): the root tool `prepare_data(operations)` with at most five operations per call, each a Pydantic model of the fixed set in `agents/anyplot/data/prepare.py`: `rename` (column, name), `drop_columns` (1 to 10 columns), `drop_rows` (`where` conditions on one column each, all of which must hold: `eq`, `ne`, `contains`, `is_missing`, `gt`, `lt`, at most three; a value is compared, never evaluated), `set_type` (number, datetime or text, with the date format and the decimal separator; to number is for the columns the profile still shows as text, because a column whose every cell carries the same unit is already numbers after the paste, the unit in its name such as `Anteil (%)`: one unit such as `%` or `EUR` is removed and the cells are read as the parser reads a number, the other mark a thousands separator between three-digit groups only, and when more cells read with the other decimal mark the whole column is read that way and the change line says so; a cell that cannot be read becomes missing, more than 10 % of them is a `type_mismatch`; to text the column stays text through the re-parse of every later version, stored with the version as `text_columns`; `MM/DD/YYYY` on a column that is already dates swaps day and month only for a slash column the parser flagged as ambiguous), `aggregate` (by one or two columns, a value column, `sum`, `mean`, `count`, `min` or `max`, and a `month`, `quarter` or `year` period for a datetime key), `sort`, `pivot_longer` (2 to 20 columns into a name/value pair; a long format over the 2 MB cap of computed data, `MAX_COMPUTED_BYTES`, is an `invalid_operation`; 200 KB until 2026-10-14) and `pivot_wider` (the distinct values of a column become at most 50 columns; duplicate pairs are refused, never aggregated silently), and, since 2026-10-12, `classify` (a new text column whose cells are labels: each row gets the label of the first of 1 to 10 rules whose `drop_rows`-style conditions all hold, else `otherwise` or a missing cell; labels are at most 40 characters, never a token the parser reads as a missing cell such as `NA` or `-`, exempt from ToolSafety's link rule like a rename's name, and the column stays text through every re-parse). `classify` came from the owner's case of 2026-10-12: a generated sine table on `bar-feature-importance` had no text column for the `feature` role, and "positive is class 1, negative class 2" had no operation, so the root had to refuse. Since 2026-10-14, `derive` adds a number column from number columns by one formula of a closed set: `ratio` (`a / b`) and `difference` (`a - b`) of two columns, `sum` of 2 to 10 columns in the same row, `share` of one column in the total of its group (`within` 0 to 2 key columns), and three formulas that walk each group in the order of `order`, a number or datetime column: `percent_change` from the previous row, `cumsum` (the running total) and `index` (the group's first row = 100; the last two by owner decision of 2026-10-14). A text order column is a `type_mismatch` with a hint to `set_type` it, because it would sort alphabetically (`April` before `Januar`, `10` before `9`); repeated order values within a group are refused (aggregate first), and a row without an order value keeps an empty cell; `percent` multiplies a ratio or a share by 100, and a division by zero or a missing input leaves the cell empty, which the change line counts. It came from the owner's live test of 2026-10-14: "which year has the largest share of foreigners?" on a long table (year, origin, count), where the root could only ask the user to compute the column; now it derives the share, drops the other rows, sorts, and answers from the new profile. Since 2026-10-15 no operation loses precision without its change line saying so (owner rule): `set_type number` reads whole numbers exactly (never through a float64, which rounded `9007199254740993`) and counts the cells it can store only rounded; a `sum` aggregate, a row `sum` or `difference` and a `cumsum` of whole-number columns add in Python integers (pandas' `Int64` sum had wrapped around past 2^63 without a word), and a result past the int64 range is a float64 whose rounded values the change line counts; `pivot_longer` of whole numbers beside decimals counts the whole numbers past 2^53 that the shared float64 column rounds; a date-time keeps its fraction of a second and its UTC offset in `data.csv`; `set_type datetime` without a format reads a year-first cell (`2024-06-01T13:00:00Z`) year, month, day (pandas with `dayfirst` had read it as 6 January), and refuses a column with different UTC offsets as a `type_mismatch` rather than converting it to one time zone. Since 2026-10-15, `make_date` adds a date column built from whole-number part columns of the same row: a year (four digits), then a month (1 to 12) and optionally a day, or a quarter (1 to 4), or an ISO week (1 to 53), a part left out being its first (a year and a month give the first day of each month); a day without a month, a quarter or a week beside another part, a value out of its range and a row that is no calendar date (31 April) are refused, never moved; a text month column may hold month names and abbreviations in the eleven refusal languages (en, de, fr, es, it, pt, nl, pl, zh, vi, ru), compared without case and diacritics, an unknown name refusing it (owner decision of 2026-10-15); ISO week 1 can start in late December, which the change line says. `set_type` to datetime gained the formats `DD/MM/YYYY`, `YYYYMMDD` and `YYYYMM` (also on a number column), `YYYY-MM`, `MM/YYYY`, `MM.YYYY` and `YYYY`; the compact ones take exactly 8, 6 or 4 digits (`2024115` is refused, never guessed), `YYYY` on numbers says "read as years", and `MM.YYYY` on a column the parser read as numbers (`03.2020` as 3.202) pads four decimals only when every cell then is a month of a year 1000 to 2200, at most 300 years apart, so a column of ordinary decimals (`2.2`, `3.5`) is refused. Dates are read with Python's parser and stored at microseconds, so years outside pandas' nanosecond range (1677 to 2262) no longer crash the tool. A date column whose every day is at most 12 keeps its day and month order open from version to version (a warning carried by `with_date_notes`), and `set_type` swaps it in either direction: `MM/DD/YYYY` on a column read day first, `DD/MM/YYYY` or `DD.MM.YYYY` on one read month first. All this came from live test round 3 and its review: a MeteoSwiss table of `Year` and `Month` had no date for `line-timeseries-rolling`, the root tried `run_data_code`, which cannot read the user's table, and then asked the user to add the column by hand; the root prompt now names the operation and forbids handing data work back when an operation can do it. A date column the change made (a datetime column the version before did not have) fills an open required date role in `carry_bindings` (owner decision of 2026-10-15: code, not the model). No expression language (an injection surface), no joins, no model-guessed fills: the operations only move, type, aggregate, drop, label or compute from cells that are there, and `ToolSafety` validates the list against the same models before the tool runs. The server applies them with pandas to the stored canonical CSV, re-parses the result with `parse_dataset` (the dataset judge does not run again: the only new text is a rename's name or a `classify` label, which the model writes from a user message the scope judge already checked) and stores it as the next dataset version (`agents/anyplot/dataset_versions.py`): the previous version joins `anyplot.dataset_history`, which keeps the last five (older ones leave the store; the idle sweep takes a session's history only together with its current version), `anyplot.dataset_version` counts up from the paste's 1, the bindings are carried over (`carry_bindings`: the ones still valid stay, the roles left without one get the defaults), and the confirmation drops by itself because its fingerprint names the dataset id. The tool answers the change lines (one English line per operation with its counts, fenced as ``) and the new profile (fenced as ``), or a `PrepareError` code (`unknown_column`, `type_mismatch`, `empty_result`, `too_wide`, `invalid_operation`) with a message that names columns at most. The page learns about a changed dataset from the `done` event: after a turn that changed the dataset id it carries `dataset: {preview, profile, bindings, warnings, roles, changes, version, context_lines}`, the dataset route's shape plus the turn's change lines (`roles` since the formulas below), and `data_confirmed: false`; the BFF relays a `done` of up to 512 KiB whole (every other event stops at 64 KiB), which the largest dataset the parser accepts fits (`agents/stream.py`, `MAX_DONE_EVENT_CHARS`). `POST /v1/sessions/{sid}/dataset/undo` (the user's `.undo()` button; the model has no undo) restores the newest history entry as the current dataset, bindings carried over, unconfirmed again, and answers like the dataset route (`422 nothing_to_undo` for a paste that was never changed). The step-1 summary is the action `{"action": "summarize_data"}` on the messages route (judge skipped, like `create_plot`; since 2026-10-14 the turn also carries the notes pasted with the table, already judged at the paste, after a preamble and fenced as ``, and `get_dataset_profile` returns them under `user_notes` for later turns, with a fixed session-block line saying they exist, so the root uses them as context for the whole chat and acts on a clear request in them at once, the plot itself still waiting for the user's confirmation): the root reads the profile, which now carries `mean` and `median` for numeric columns, and answers in two to four sentences (rows, columns with types and units, missing values, values that look like percentages, the proposed bindings, one question when something is ambiguous), never claiming the data is right: the user confirms. The live eval `agents/evals/prep.py` scores the root's choices on `agents/evals/fixtures/prep/` by exact operation match, order-insensitive, with an 80 % gate (see [Run the prep eval](../../agents/README.md#run-the-prep-eval)). The page asks for the summary right after every parse that stores a dataset (a paste, or a transcription's `.use()`; never after an undo or a library switch), so it is the first turn in the thread, and the composer takes data requests in step 1. A `done` with `dataset` replaces the preview, the profile and the bindings, takes the page back to step 1, and shows a `# changed` block above the preview with the change lines and `.undo()`; after an undo the block names the restored version and keeps `.undo()` while the version is above 1, until the service answers `nothing_to_undo`. Built since 2026-10-15 (questions): a question never changes the data. In the live test of 2026-10-15 the root answered "which country had the highest per-capita emissions in 2023?" correctly, but by filtering the user's dataset to the rows of 2023 with `prepare_data`, which stores a version, and then asked the user to `.undo()` it. The root tool `query_data(operations, columns, limit)` (`agents/anyplot/data/query.py`) applies zero to five operations of the same closed set as `prepare_data` (no free expression) to a copy of the current version and returns the first `limit` rows (1 to 25, default 10) in at most 8 columns, fenced as ``, with the result's row count, a `truncated` flag and the change lines; the result is cut by rows to its 16 KB cap. Nothing is stored: the dataset, its version, the bindings and the confirmation stay as they are. The root prompt says that `prepare_data` is only for a change the user asked for, and that a question (a maximum in a year, a share per year, a trend) goes to `query_data`; a computed column the user then wants in the data is a `prepare_data` with the same operations. @@ -265,7 +265,7 @@ Schemas live in `agents/anyplot/schemas.py`: `ColumnProfile{name≤64, dtype, mi ### Parse -`agents/anyplot/data/parse.py` has no ADK import. Limits: UTF-8 at most 1 MB for a paste (200 KB until 2026-10-14) and 2 MB for a table the service computed (`MAX_COMPUTED_BYTES`, passed as `max_bytes` by every re-parse of the service's own CSV), a canonical CSV of at most 4 MiB, 50 columns, 20,000 rows and 200 characters per cell. It rejects NUL and control characters, strips the BOM, applies NFC, and strips zero-width and bidi control characters. Headers are canonicalised (NFC, control characters stripped, names over 64 characters renamed, duplicates suffixed, empty names become `column_N`, each rename a warning), and the canonical names are written into `data.csv`, which is the single source of column names. Formats: JSON records, a column map, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`, source format `json_pairs`; nesting depth at most 2, except the nested objects of records described below; a repeated key keeps its last value, with a warning), a JSON-stat 2.0 dataset (since 2026-10-14, source format `json_stat`; see below), or delimited text with the delimiter sniffed from `, ; \t |`. Since 2026-10-15 (live test round 3) two more shapes read. A table aligned with spaces (source format `whitespace`: a weather or statistics office's `.txt`, such as MeteoSwiss' homogenised monthly series) splits at runs of two or more spaces, so a header such as `Max Temp` stays one name; a tab is no separator there. Spaces are tried only when none of the four real delimiters finds a table (a CSV or TSV whose text fields hold spaces stays CSV or TSV), and they need stricter evidence, because they also separate the words of prose, the items of a list (`Hund 7`, which stays with the transcriber) and the keys and values of a metadata block: at least two data rows, a header line right above the first data row that holds no value (two or more years excepted, a wide table's `Land 2020 2021`), and no key-value block (a header whose first cell, or rows whose every first cell, ends with a colon). Owner principle of the review of 2026-10-15: a silent misread is the worst failure, so a refusal beats a guess. Single spaces split a table only of numbers and dates (every row at the header's width, no text cell, no two neighbouring fields that read as one value such as `1 234` or `2024-01-01 12:00`), with a warning on every such read: a single space also sits inside a two-word name (`Anna Lee`), so a table with a text column goes to the transcriber. A field in matching quotes loses them. A short row of a fixed-width layout (a blank cell) has its fields placed under the header's columns by their character position, never padded at the end, also below the last full row until a note, a source line or a summary row (`Total`, left out like other summary rows); a field that overlaps no column or more than one, columns whose own ranges overlap, and a row that does not line up, also a value row right below the table, are `unparseable`. The metadata block above such a table becomes notes like any other left-out lines. A line of the paste that only declares the missing-value mark (`Missing values: NA`, `nodata = -9999`; a sentinel number, negative or at least 999 in size, or one to three symbols or letters, never an answer such as `none`) counts as missing already while the table is found, so a row of marks stays a row; it blanks the cells of number and date columns only, a number matching by value (`-999.0`), with one warning that counts them, while a text category `M` or `NA` (Namibia) stays (owner decision of 2026-10-15). JSON records may nest objects, which become columns named with dots (`country.value`, `indicator.id`) down to three levels; an object nested deeper, or a field holding a list of objects or lists, is left out with one warning that names it (owner rule: canonical data is one flat CSV). A field holding a list of values expands to long format (owner decision of 2026-10-15, one measurement curve per record with `x: [...]` and `y: [...]`): the lists of one record expand together into one row per value, paired by position (so independent lists of one length, such as `tags` and `scores`, are paired too; the warning names every expanded field), the other fields repeated; an empty list leaves its cells empty, non-empty lists of different lengths in one record are `unparseable` with a message that names the record, the fields and their lengths, and so is a field that is a list in one record and a single value in another. When no text or whole-number field has a different value in every record, compared as `data.csv` will hold it (a unique measurement such as a temperature never counts), a first column `record_id` numbers the records, or `record_id_2` and so on when a field already has that name. Above the 20,000-row cap each record keeps evenly spaced values with its first and last one, the smallest thinning that fits, with a warning that gives the counts and the remedy (fewer points for one record, fewer records or points for several), which the root passes on. Records may also come wrapped as data APIs answer: an array of metadata objects and exactly one list of records (the World Bank's `[{page, pages, per_page, total, ...}, [records]]`), or an object with exactly one list of records beside other fields (`{"data": [...], "meta": {...}}`, also under `results`, `records`, `items` or any other key); the metadata's text, number and boolean fields go to the notes as `: `, and two lists of records, or metadata without records, are `unparseable`, since the parser takes no table by guess. A list beside the records counts as their schema only when its objects hold nothing but descriptor keys (`name`, `type`, `format`, `description` and the like) and their names cover every field of the records; descriptors beside an empty list of records are `unparseable`. A schema with rows of cells (CKAN, Socrata, Frictionless, pandas `orient="split"`: column names under `fields`, `columns`, `schema.fields` or `meta.view.columns`, rows as lists) reads as one record per row; rows without names that fit them are `unparseable`, so the schema is never taken for the data. BigQuery's own `rows: [{"f": [{"v": ...}]}]` shape is not read (its cells are a list of objects, which is left out, so the paste is refused). JSON-stat is still recognised first. The dataset pre-filter (`instruction_like`) reads the table with its constant columns, which come back into `data.csv` when the notes are dropped. The root's prompt lists every control of the chat page (`.parse()`, `.confirm()`, `.undo()`, the binding dropdowns, the language chip and so on) and forbids naming any other, since it once told a user to pick "a separator that recognises spaces", a setting that does not exist; `tests/unit/agents/test_prompt_controls.py` checks that each control the prompt names is a label in the page's code. JSON-stat is what statistics offices' APIs answer (the Swiss BFS PxWeb API, Eurostat, Statistik Austria, Statistics Norway): an object with `"class": "dataset"`, `id`, `size`, `dimension` and `value`; a JSON-stat 1.x bundle of one dataset is read too, a bundle of several, a collection or a lone dimension is `unparseable`. It is the one shape nested deeper than 2 (about five levels), and the 50-level bracket count still applies. The parser checks it strictly before it builds a row (as many sizes as dimensions, the product of the sizes at most 20,000 and equal to the number of values, `value` as an array or as a sparse object of positions, every dimension with as many categories as its size, `category.index` as an array or an object of positions), and refuses any mismatch as `unparseable` with a message that names no data. It flattens the dataset into a long table (owner rule: long format, no wide pivot; `prepare_data` can pivot later): one column per dimension with more than one category, named by the dimension's label (else its id), its cells the category labels (else the codes) kept as text, never typed and never missing (`NA` for Namibia stays `NA`, a year code stays text; a label two categories share gets each one's code in parentheses), then one column `value`, the only typed column, whatever the dataset's unit (a dimension named `value` in any case becomes `