diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 6c26b7c9c7b..daa13ed8adc 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -23,7 +23,16 @@ (`json_pairs`: the columns `category` and `value`, one row per entry, in the pasted order; the keys stay text, the values are typed); nesting depth at most 2; `null`, `NaN` and `Infinity` are missing; a key repeated in an object - keeps its last value, with a warning. Text that starts with a bare `(` and is + keeps its last value, with a warning. A JSON-stat 2.0 dataset (`json_stat`, + since 2026-10-14: what the statistics offices' APIs return, BFS PxWeb, + Eurostat, Statistik Austria, Statistics Norway), or a JSON-stat 1.x bundle + of one dataset, is the one shape nested deeper than 2 (dimension, category, + index: about five levels): it is checked strictly and flattened into a long + table, one column per dimension with more than one category (named by its + label, the category labels as cells, kept text), then `value`, in the order of + `value`; see `_json_stat`. Its one-category dimensions, label, source and + notes become `context_lines`. The `MAX_NESTING` bracket count below applies + to it as to everything else. Text that starts with a bare `(` and is neither JSON nor a Python literal (a header such as `(in %) Land`, or prose in parentheses) is delimited text instead, like everything that starts otherwise, see `sniff_delimiter`. @@ -83,6 +92,7 @@ from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime +from itertools import product from typing import Any, Literal from ..schemas import ( @@ -186,6 +196,7 @@ _DECIMAL_COMMA = re.compile(r"[+-]?(?:[1-9][0-9]{0,2}(?:\.[0-9]{3})+|[0-9]+)(?:,[0-9]+)?") _THOUSANDS_DOT = re.compile(r"[+-]?(?:[1-9][0-9]{0,2}(?:\.[0-9]{3})+|[0-9]+)") _AMBIGUOUS_COMMA = re.compile(r"[+-]?[0-9]{1,3},[0-9]{3}") +_INTEGER_INDEX = re.compile(r"0|[1-9][0-9]{0,8}") # a position key of a sparse JSON-stat `value` object # A number with one unit. The number part only has to look like one here: the rules above # then type it. A trailing unit (`65 %`, `15,6 h`, `20°C`, `3 Std.`, `100 m²`, `12 CHF`, @@ -265,7 +276,8 @@ class ParsedDataset: context_lines: list[str] = field(default_factory=list) """The lines `find_table` left out above and below the table (a title, an export's metadata, a source line, a note), each as one cleaned text line, at most `MAX_CONTEXT_CHARS` in all: the user's notes about - the data. Summary rows are not among them. Empty for a literal and for `find_table=False`.""" + the data. Summary rows are not among them. Empty for `find_table=False` and for a literal, except a + JSON-stat dataset, whose label, source, notes and one-category dimensions are its lines (`_json_stat`).""" def without_context(parsed: ParsedDataset, warning: str) -> ParsedDataset: @@ -297,7 +309,7 @@ class _Column: source: list[str] # cleaned cells, as pasted cells: list[str] # canonical cells for data.csv, set by the typing step invisible: bool = False - text_only: bool = False # the `category` column of `json_pairs`: keys are names, never typed + text_only: bool = False # `json_pairs` category, `json_stat` dimension: keys are names, never typed dtype: ColumnDtype = "text" unit: str | None = None # the unit every cell carried, now part of the name bare: list[str] = field(default_factory=list) # the cells the number typing read: `source`, or the number parts @@ -336,7 +348,7 @@ def parse_dataset(text: str, *, text_columns: Collection[str] = (), find_table: literal = _LITERAL_START.match(stripped) if literal: assignment, body = literal.group(), stripped[literal.end() :] - table = _read_literal(body, warnings) + table = _read_literal(body, warnings, context) if table is None and (assignment or not body.startswith("(")): raise ParseError("unparseable", "the data starts like JSON or a Python literal but is neither") # A bare `(` that neither reader accepts is delimited text: `(in %) Land;Wert`, prose in parentheses. @@ -615,8 +627,14 @@ def _context_lines(records: Iterable[Sequence[str]], delimiter: str | None) -> t return lines, False -def _read_literal(text: str, warnings: list[str]) -> tuple[list[str], list[list[str]], SourceFormat] | None: - """JSON, or a Python literal when the JSON reader refuses the text, tabulated alike; None when it is neither.""" +def _read_literal( + text: str, warnings: list[str], context: list[str] +) -> tuple[list[str], list[list[str]], SourceFormat] | None: + """JSON, or a Python literal when the JSON reader refuses the text, tabulated alike; None when it is neither. + + A JSON-stat dataset (`_json_stat`) is flattened instead, its title, source, notes and + one-category dimensions going to `context`. + """ if _too_deep(text): raise ParseError("unparseable", f"the data is nested more than {MAX_NESTING} levels deep") decoded = _decode_literal(text) @@ -625,6 +643,9 @@ def _read_literal(text: str, warnings: list[str]) -> tuple[list[str], list[list[ data, repeated = decoded if repeated: warnings.append(f"{_count(repeated, 'key')} repeated; only the last value of a repeated key is kept") + dataset = _json_stat_dataset(data) + if dataset is not None: + return _json_stat(dataset, warnings, context) return _tabulate(data, warnings) @@ -805,6 +826,212 @@ def _json_cell(value: object, position: int) -> str: ) +# --- JSON-stat --------------------------------------------------------------------------- + + +def _json_stat_dataset(data: Any) -> dict[Any, Any] | None: + """The JSON-stat dataset in `data`, None when `data` is no JSON-stat document. + + A JSON-stat 2.0 dataset is an object with a `dimension` object and a `value`, marked by + `"class": "dataset"` or, as in some 1.x-style responses, by a top-level `id`. A JSON-stat 1.x + bundle (`{"": {"dimension": ..., "value": ...}}`) yields its dataset when it holds one. + A 2.0 collection or dimension, or a bundle of several datasets, holds no single table: + `unparseable`. Anything else (`{"class": "a"}` is an object of values) is None and goes to + `_tabulate` as before. + """ + if not isinstance(data, dict) or not data: + return None + kind = data.get("class") + if isinstance(data.get("dimension"), dict) and "value" in data and (kind == "dataset" or "id" in data): + return data + if kind == "collection" and isinstance(data.get("link"), dict): + raise ParseError("unparseable", "a JSON-stat collection lists datasets but holds no values; paste one dataset") + if kind == "dimension" and isinstance(data.get("category"), dict): + raise ParseError("unparseable", "a JSON-stat dimension holds no values; paste a dataset") + if all( + isinstance(entry, dict) and isinstance(entry.get("dimension"), dict) and "value" in entry + for entry in data.values() + ): + if len(data) > 1: + raise ParseError( + "unparseable", f"the JSON-stat bundle holds {len(data)} datasets; paste one dataset at a time" + ) + (only,) = data.values() + return dict(only) + return None + + +def _json_stat_refusal(reason: str) -> ParseError: + return ParseError("unparseable", f"the JSON-stat dataset is inconsistent: {reason}") + + +def _json_stat( + dataset: Mapping[Any, Any], warnings: list[str], context: list[str] +) -> tuple[list[str], list[list[str]], SourceFormat]: + """A JSON-stat dataset as a long table: one column per dimension with more than one category, then `value`. + + Every check comes before the rows are built: `id` and `size` name the dimensions in the + same order (a 1.x dataset keeps both inside `dimension`), the product of the sizes is at + most `MAX_ROWS` and equals the number of values (`value` as an array, or as a sparse + object of positions whose missing ones are empty), and every dimension lists as many + categories as its size (`category.index` as an array of codes or an object of code to + position; a dimension of one category may give only `category.label`). + + A dimension's column is named by its `label` (else its id), its cells are the category + labels (else the codes), cleaned like the keys of `json_pairs` and kept text: `NA` + (Namibia) or `2024` is a name, never a missing cell or a number, and only `value` is + typed. A label shared by two categories in that cleaned form gets its code in + parentheses, with a warning. The rows run row-major over `id`, the order of `value`. + The value column is `value` whatever the dataset's units, and a dimension named + `value` in any case is renamed `