feat(agents): prepare_data changes the data in step 1, with undo

agents/anyplot/agent.py

@@ -7,8 +7,9 @@
   version's theme, so a change keeps it), which ADK sends
   as a marked instruction block after the static prefix. Catalogue text such as the
   spec title started as a public issue, so it never enters that block: the root reads
-  it fenced as `<spec_text>` through `get_spec_brief`. Its tools are the four session
-  tools and the `plot_pipeline` NodeTool.
+  it fenced as `<spec_text>` through `get_spec_brief`. Its tools are the five session
+  tools (`prepare_data` among them, the step-1 data operations) and the `plot_pipeline`
+  NodeTool.
 * The adapters (one per enabled library) and the reviewer run only inside the
   pipeline through `ctx.run_node`, so a walk from the root would miss them;
   `ALL_AGENTS` lists every agent for the registry test.

agents/anyplot/briefs.py

@@ -8,6 +8,7 @@
 
 import json
 from collections.abc import Iterator
+from typing import Any
 
 from .data.roles import DataRole
 from .policy import fence
@@ -55,7 +56,12 @@ def profile_summary(profile: DatasetProfile) -> str:
 
 def profile_json(profile: DatasetProfile) -> str:
     """The profile as JSON, warnings left out (they name parse details, not data)."""
-    return profile.model_dump_json(exclude={"warnings"})
+    return _dump(profile)
+
+
+def _dump(profile: DatasetProfile, exclude: dict[str, Any] | None = None) -> str:
+    """The profile JSON a model reads: no warnings, and no field without a value (a text column's `min`, `mean`, ...)."""
+    return profile.model_dump_json(exclude={"warnings": True, **(exclude or {})}, exclude_none=True)
 
 
 def adapter_profile(profile: DatasetProfile, limit: int = MAX_ADAPTER_PROFILE_BYTES) -> tuple[str, bool]:
@@ -80,14 +86,15 @@ def trimmed_profiles(profile: DatasetProfile) -> Iterator[tuple[str, bool]]:
     """The profile JSON, then ever smaller versions of it, each with whether it was trimmed.
 
     The order of what goes: the sample rows one by one, then every column's top values,
-    then the min and max. Every version keeps every column name, its type and counts,
-    so a caller with a size cap takes the first version that fits.
+    then the ranges and the statistics (min, max, mean, median). Every version keeps
+    every column name, its type and counts, so a caller with a size cap takes the first
+    version that fits.
     """
     yield profile_json(profile), False
     for keep in range(len(profile.sample) - 1, -1, -1):
-        yield profile.model_copy(update={"sample": profile.sample[:keep]}).model_dump_json(exclude={"warnings"}), True
+        yield _dump(profile.model_copy(update={"sample": profile.sample[:keep]})), True
     bare = profile.model_copy(
         update={"sample": [], "columns": [column.model_copy(update={"top": []}) for column in profile.columns]}
     )
-    yield bare.model_dump_json(exclude={"warnings"}), True
-    yield bare.model_dump_json(exclude={"warnings": True, "columns": {"__all__": {"min", "max", "top"}}}), True
+    yield _dump(bare), True
+    yield _dump(bare, {"columns": {"__all__": {"min", "max", "mean", "median", "top"}}}), True

agents/anyplot/data/bindings.py

@@ -2,7 +2,9 @@
 
 No ADK import. `bindings.apply`, which writes validated bindings into session
 state and answers `409 run_active` during a run, belongs to the ADK session layer;
-it calls `check_bindings`.
+it calls `check_bindings`. `carry_bindings` is what a dataset version change
+(`prepare_data`, undo) does to the bindings: the ones still valid stay, the roles
+left without one get the defaults.
 
 Compatibility of a role kind with a column dtype (`ACCEPTS`), with the tier that
 ranks candidate columns for the default:
@@ -22,6 +24,7 @@
 `check_bindings` but is never bound by default.
 """
 
+import itertools
 import re
 from dataclasses import dataclass
 from difflib import SequenceMatcher
@@ -145,6 +148,44 @@ def default_bindings(roles: list[DataRole], profile: DatasetProfile) -> list[Bin
     return bindings
 
 
+def carry_bindings(bindings: list[Binding], roles: list[DataRole], profile: DatasetProfile) -> list[Binding]:
+    """The bindings after the dataset changed under them (`prepare_data`, undo): the valid ones stay.
+
+    A binding stays when its column is still in `profile` with a dtype its role accepts
+    (`check_bindings` on it alone finds no error). When every binding stayed, nothing
+    changes, so a role the user left unbound stays unbound. When one went (its column
+    dropped, renamed or re-typed), the roles without a binding are bound as
+    `default_bindings` would bind them on the columns no kept binding uses: the
+    defaults are computed on that reduced view, so a role whose first choice a kept
+    binding holds takes its next candidate rather than staying open. A family keeps
+    its members and gains the defaults' members under the next free numbers. The
+    result is in the order the defaults list them.
+    """
+    kept = [binding for binding in bindings if not check_bindings([binding], roles, profile).errors]
+    if len(kept) == len(bindings):
+        return list(bindings)
+    used = {binding.column for binding in kept}
+    taken = {binding.role for binding in kept}
+    open_roles = [role for role in roles if role.variadic or role.name not in taken]
+    free = profile.model_copy(update={"columns": [column for column in profile.columns if column.name not in used]})
+    for binding in default_bindings(open_roles, free):
+        role = resolve_role(binding.role, roles)
+        if role is not None and role.variadic:
+            number = next(n for n in itertools.count(1) if role.member(n) not in taken)
+            binding = Binding(role=role.member(number), column=binding.column)
+        taken.add(binding.role)
+        kept.append(binding)
+    # In the order the defaults list them: the spec's roles, a family's members by column position.
+    role_order = {role.name: position for position, role in enumerate(roles)}
+    column_order = {column.name: position for position, column in enumerate(profile.columns)}
+
+    def place(binding: Binding) -> tuple[int, int]:
+        role = resolve_role(binding.role, roles)
+        return role_order.get(role.name if role else "", len(roles)), column_order.get(binding.column, 0)
+
+    return sorted(kept, key=place)
+
+
 def check_bindings(bindings: list[Binding], roles: list[DataRole], profile: DatasetProfile) -> BindingCheck:
     """Validate bindings against the spec's roles and the dataset's columns.
 

agents/anyplot/data/parse.py

@@ -56,6 +56,11 @@
 and gets the typed frame back: `MISSING_VALUES` contains every token pandas reads
 as missing by default, so no such token survives into `data.csv` as text.
 
+`text_columns` names columns that skip step 6 and stay text whatever their cells
+hold: the `prepare_data` operation `set_type text` keeps a column of years or ids as
+a category through the re-parse of every later version (`prepare.py`), and the
+result records them in `ParsedDataset.text_columns`, so the store carries them on.
+
 Error messages never quote the user's data; at most they name a column.
 """
 
@@ -65,10 +70,11 @@
 import json
 import math
 import re
+import statistics
 import tokenize
 import unicodedata
 from collections import Counter
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Collection, Iterable, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Any, Literal
@@ -201,6 +207,8 @@
 )
 
 _INT64_MIN, _INT64_MAX = -(2**63), 2**63 - 1
+AMBIGUOUS_DATE_ORDER = "day and month order is ambiguous; read as DD/MM/YYYY"
+"""The note of a slash date column read day first by guess; `prepare.py` finds those columns by it."""
 _BOM = "\N{ZERO WIDTH NO-BREAK SPACE}"
 _NO_DELIMITER = "\x00"  # NUL never survives the control-character check, so it never splits a line
 
@@ -224,6 +232,8 @@ class ParsedDataset:
     column_dtypes: dict[str, ColumnDtype]
     parse_dates: list[str]
     warnings: list[str]
+    text_columns: list[str] = field(default_factory=list)
+    """The columns kept text by request (`parse_dataset(text_columns=...)`), in column order."""
 
 
 @dataclass
@@ -250,8 +260,12 @@ def pandas_dtypes(column_dtypes: Mapping[str, ColumnDtype]) -> dict[str, str]:
     return {name: PANDAS_DTYPES[dtype] for name, dtype in column_dtypes.items() if dtype != "datetime"}
 
 
-def parse_dataset(text: str) -> ParsedDataset:
-    """Parse pasted CSV, TSV, semicolon or pipe text, JSON or a Python literal into the canonical dataset."""
+def parse_dataset(text: str, *, text_columns: Collection[str] = ()) -> ParsedDataset:
+    """Parse pasted CSV, TSV, semicolon or pipe text, JSON or a Python literal into the canonical dataset.
+
+    `text_columns` names (canonical) columns that stay text instead of being typed by
+    their cells; a name that is no column is ignored.
+    """
     _check_size(text)
     text = text.removeprefix(_BOM)
     _check_controls(text)
@@ -272,7 +286,7 @@ def parse_dataset(text: str) -> ParsedDataset:
     if table is None:
         table = _read_delimited(text, warnings)
     headers, rows, source_format = table
-    return _build(headers, rows, source_format, warnings)
+    return _build(headers, rows, source_format, warnings, frozenset(text_columns))
 
 
 # --- Limits ------------------------------------------------------------------------------
@@ -586,6 +600,16 @@ def _clean_key(raw: str) -> tuple[str, bool]:
     return text.replace("\r\n", "\n").replace("\r", "\n").strip(), invisible
 
 
+def canonical_names(headers: Sequence[str]) -> list[str]:
+    """The column names `parse_dataset` gives these raw headers when no cell carries a unit.
+
+    The same cleaning and canonicalisation as the parse (`_clean_header`, then
+    `canonical_headers`), for a caller that writes a CSV and must name its columns as
+    the re-parse will: `prepare.py` keeps `text_columns` by those names.
+    """
+    return canonical_headers([_clean_header(raw)[0] for raw in headers])[0]
+
+
 def _clean_header(raw: str) -> tuple[str, bool, bool]:
     """The header text, whether invisible characters went, and whether tabs or line breaks became spaces."""
     text, invisible = _strip_invisible(raw)
@@ -812,7 +836,7 @@ def _as_dates(column: _Column, present: list[str]) -> bool:
             return False
         day_first = not second_big
         if not first_big and not second_big and any(parts[0] != parts[1] for parts in fields.values()):
-            column.notes.append("day and month order is ambiguous; read as DD/MM/YYYY")
+            column.notes.append(AMBIGUOUS_DATE_ORDER)
 
     parsed: dict[str, datetime] = {}
     for cell, parts in fields.items():
@@ -841,7 +865,11 @@ def _iso(value: datetime, with_time: bool) -> str:
 
 
 def _build(
-    raw_headers: list[str], raw_rows: list[list[str]], source_format: SourceFormat, warnings: list[str]
+    raw_headers: list[str],
+    raw_rows: list[list[str]],
+    source_format: SourceFormat,
+    warnings: list[str],
+    text_columns: frozenset[str] = frozenset(),
 ) -> ParsedDataset:
     headers: list[str] = []
     columns: list[_Column] = []
@@ -885,9 +913,16 @@ def _build(
                 "cell_too_long", f"a cell in column '{named}' has more than {MAX_INPUT_CELL_CHARS} characters"
             )
 
-    # Typing comes before naming: a column whose cells carried a unit is named after it.
-    for column in columns:
-        _type_column(column)
+    # Typing comes before naming: a column whose cells carried a unit is named after it. A
+    # column kept text by request is found by its name without a unit: it gets none, since it
+    # is never typed, and the only text that carries such a request is the canonical CSV of an
+    # earlier version, whose headers are unique already, so that name is its final name.
+    plain_names, _ = canonical_headers(headers) if text_columns else ([], [])
+    for index, column in enumerate(columns):
+        if text_columns and plain_names[index] in text_columns:
+            column.cells = list(column.source)  # kept text by request: the cells as cleaned, never typed
+        else:
+            _type_column(column)
     if any(column.decimal_comma for column in columns):
         for column in columns:
             if column.dtype == "number" and not column.decimal_comma:
@@ -942,16 +977,20 @@ def _build(
         column_dtypes={column.name: column.dtype for column in columns},
         parse_dates=[column.name for column in columns if column.dtype == "datetime"],
         warnings=capped,
+        text_columns=[column.name for column in columns if column.name in text_columns],
     )
 
 
 def _profile_column(column: _Column) -> ColumnProfile:
     present = [cell for cell in column.cells if cell]
     minimum: float | str | None = None
     maximum: float | str | None = None
+    mean: float | None = None
+    median: float | None = None
     top: list[str] = []
     if column.dtype in ("integer", "number") and column.numbers:
         minimum, maximum = min(column.numbers), max(column.numbers)
+        mean, median = _statistics(column.numbers)
     elif column.dtype == "datetime" and column.dates:
         minimum, maximum = _iso(min(column.dates), column.with_time), _iso(max(column.dates), column.with_time)
     elif column.dtype in ("text", "boolean"):
@@ -968,10 +1007,37 @@ def _profile_column(column: _Column) -> ColumnProfile:
         unique=len(set(present)),
         min=minimum,
         max=maximum,
+        mean=mean,
+        median=median,
         top=top,
     )
 
 
+def _statistics(numbers: Sequence[float]) -> tuple[float | None, float | None]:
+    """The mean and the median at six significant digits; None for one that no float holds.
+
+    `fsum` of two `1e308` values raises `OverflowError`, and the sum of the two middle
+    values can be infinite, so each is computed within the float range or left out
+    rather than failing the parse of a dataset whose values are all finite.
+    """
+    try:
+        mean: float | None = statistics.fmean(numbers)
+    except OverflowError:
+        mean = None
+    ordered = sorted(numbers)
+    middle = len(ordered) // 2
+    median = ordered[middle] if len(ordered) % 2 else ordered[middle - 1] / 2 + ordered[middle] / 2
+    return (
+        _rounded(mean) if mean is not None and math.isfinite(mean) else None,
+        _rounded(median) if math.isfinite(median) else None,
+    )
+
+
+def _rounded(value: float) -> float:
+    """A statistic at six significant digits, so a mean never carries float noise into the profile."""
+    return float(f"{value:.6g}")
+
+
 def _write_csv(names: list[str], rows: list[list[str]]) -> str:
     buffer = io.StringIO()
     writer = csv.writer(buffer, lineterminator="\n")