diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 4fe04dd82cb..20137f58207 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -27,9 +27,13 @@ neither JSON nor a Python literal (a header such as `(in %) Land`, or prose in parentheses) is delimited text instead, like everything that starts otherwise, see `sniff_delimiter`. - Fully blank lines (also `;;;`-style lines of empty fields) are skipped; a row - with fewer fields than the header is padded with missing cells (one warning); - a row with more non-empty fields than the header is `unparseable`. + Fully blank lines (also `;;;`-style lines of empty fields) are skipped. Delimited + text is first narrowed to its table (`table_block`, since 2026-10-14): a title, an + export's metadata or a note above the header, a source line or a note below the + last row, and up to three summary rows (`Total`, `Summe`, `Mittelwert`…) at the end + are left out, each kind with one warning. A row with fewer fields than the header + is padded with missing cells (one warning); a row with more non-empty fields than + the header is `unparseable`. 4. **Clean every cell.** Remove invisible format characters (Unicode category `Cf`: zero-width characters, bidi controls, BOMs inside the text, tag characters), turn line and paragraph separators (U+2028, U+2029) into spaces, @@ -98,7 +102,21 @@ MAX_INPUT_BYTES = 200 * 1024 # "200 KB", counted like the schemas' 24 KiB: 204,800 UTF-8 bytes MAX_INPUT_CELL_CHARS = 200 # per header or cell, after cleaning; the profile's own cells stop at MAX_CELL_CHARS PREVIEW_ROWS = 20 -SNIFF_LINES = 10 +SNIFF_LINES = 50 # enough to see a table below an export's metadata or a few lines of notes +VALUE_SHARE = 0.5 +"""The share of filled cells that must read as values in a value column (`table_block`). Half, not a +large majority: in a short table, the header and a note split into the table's width at its commas +weigh as much as the few data rows.""" +MAX_SUMMARY_ROWS = 3 # summary rows at the end of a table that `table_block` leaves out +SUMMARY_LABELS: frozenset[str] = frozenset( + { + *("total", "totals", "grand total", "subtotal", "sum", "sum total", "average", "avg", "mean"), + *("median", "min", "max", "minimum", "maximum", "count", "std", "std dev", "result"), + *("summe", "gesamt", "gesamtsumme", "insgesamt", "zwischensumme", "durchschnitt", "mittel"), + *("mittelwert", "anzahl", "ergebnis"), + } +) +"""The first filled cell of a summary row, compared without case and without a trailing colon.""" MAX_NESTING = 50 # bracket levels a literal may open; counted before `ast.literal_eval` sees the text PAIR_HEADERS = ("category", "value") # the columns of an object of scalars or an array of pairs @@ -260,11 +278,14 @@ def pandas_dtypes(column_dtypes: Mapping[str, ColumnDtype]) -> dict[str, str]: return {name: PANDAS_DTYPES[dtype] for name, dtype in column_dtypes.items() if dtype != "datetime"} -def parse_dataset(text: str, *, text_columns: Collection[str] = ()) -> ParsedDataset: +def parse_dataset(text: str, *, text_columns: Collection[str] = (), find_table: bool = True) -> ParsedDataset: """Parse pasted CSV, TSV, semicolon or pipe text, JSON or a Python literal into the canonical dataset. `text_columns` names (canonical) columns that stay text instead of being typed by - their cells; a name that is no column is ignored. + their cells; a name that is no column is ignored. `find_table` narrows pasted text to + its table (`table_block`); a caller that re-reads a CSV the service wrote itself (a + prepared, generated or transcribed table) passes False, so every row it holds stays, + also a category named `Total` that a sort moved to the end. """ _check_size(text) text = text.removeprefix(_BOM) @@ -284,7 +305,7 @@ def parse_dataset(text: str, *, text_columns: Collection[str] = ()) -> ParsedDat raise ParseError("unparseable", "the data starts like JSON or a Python literal but is neither") # A bare `(` that neither reader accepts is delimited text: `(in %) Land;Wert`, prose in parentheses. if table is None: - table = _read_delimited(text, warnings) + table = _read_delimited(text, warnings, find_table=find_table) headers, rows, source_format = table return _build(headers, rows, source_format, warnings, frozenset(text_columns)) @@ -316,16 +337,46 @@ def _check_controls(text: str) -> None: def sniff_delimiter(text: str) -> tuple[str | None, SourceFormat]: """Pick the delimiter from the first `SNIFF_LINES` non-blank lines. - Each candidate (`,` `;` tab `|`) splits those lines with the csv module, quotes - honoured. A candidate counts only if it splits the header line into at least two - fields. Among those, the one under which the most sampled lines have exactly the - header's field count wins, then the one with more header fields, then the earlier - candidate in that order. When no candidate splits the header, the text is a single - column (returned as `None`, reported as `csv`). A semicolon file with decimal - commas therefore never sniffs as comma-separated: its header has no comma. + Pasted data often carries lines that are not part of the table: a title, an export's + metadata, a sentence explaining the columns (owner rule of 2026-10-14). So the table + decides, not the first line: each candidate (`,` `;` tab `|`) splits the sample with + the csv module, quotes honoured, and its table is the lines of its most common field + count (at least two fields). The candidate whose table holds the most cells that read + as a value (a number or a date, `_value_like`) wins, then the one with more such + lines, then more fields, then the earlier candidate in that order. A semicolon file + with decimal commas therefore never sniffs as comma-separated: split at its commas, + its cells are no numbers. A candidate's table counts only when the line above its + first data row has the table's width, so it can be the header: a single column of + decimal commas (`v`, `1,5`, `2,3`) is no two-column table without a header. + + When no candidate's table holds a value (text-only tables, prose), the header rule + decides as before (`_sniff_by_header`). """ lines = [line for line in _PHYSICAL_LINE.split(text) if line.strip()][:SNIFF_LINES] sample = "\n".join(lines) + best: tuple[int, int, int] | None = None + choice: tuple[str | None, SourceFormat] = (None, "csv") + for delimiter, source_format in DELIMITERS: + try: + rows = [row for row in csv.reader(io.StringIO(sample), delimiter=delimiter, skipinitialspace=True) if row] + except csv.Error: + continue + block = table_block(rows) + if not block.values or len(rows[block.start]) < block.width: + continue + count = sum(1 for row in rows if len(row) == block.width) + key = (block.values, count, block.width) + if best is None or key > best: + best, choice = key, (delimiter, source_format) + return choice if best is not None else _sniff_by_header(sample) + + +def _sniff_by_header(sample: str) -> tuple[str | None, SourceFormat]: + """The delimiter of a table without values: a candidate counts only if it splits the + header line into at least two fields. Among those, the one under which the most sampled + lines have exactly the header's field count wins, then the one with more header fields, + then the earlier candidate. When no candidate splits the header, the text is a single + column (returned as `None`, reported as `csv`).""" best: tuple[int, int] | None = None choice: tuple[str | None, SourceFormat] = (None, "csv") for delimiter, source_format in DELIMITERS: @@ -342,41 +393,155 @@ def sniff_delimiter(text: str) -> tuple[str | None, SourceFormat]: return choice -def _read_delimited(text: str, warnings: list[str]) -> tuple[list[str], list[list[str]], SourceFormat]: +def _read_delimited( + text: str, warnings: list[str], *, find_table: bool = True +) -> tuple[list[str], list[list[str]], SourceFormat]: delimiter, source_format = sniff_delimiter(text) reader = csv.reader( io.StringIO(text, newline=""), delimiter=delimiter or _NO_DELIMITER, skipinitialspace=True, strict=True ) - header: list[str] | None = None - rows: list[list[str]] = [] - padded = 0 + records: list[tuple[int, list[str]]] = [] try: for record in reader: - if not any(cell.strip() for cell in record): - continue - if header is None: - header = record - continue - if len(rows) == MAX_ROWS: - raise ParseError("too_many_rows", f"the data has more than {MAX_ROWS:,} rows") - width = len(header) - if len(record) < width: - record = record + [""] * (width - len(record)) - padded += 1 - elif len(record) > width: - if any(cell.strip() for cell in record[width:]): - raise ParseError("unparseable", f"line {reader.line_num} has more fields than the header") - record = record[:width] - rows.append(record) + if any(cell.strip() for cell in record): + records.append((reader.line_num, record)) except csv.Error as exc: raise ParseError("unparseable", f"the quoting is malformed near line {reader.line_num}") from exc - if header is None: + if not records: raise ParseError("unparseable", "the data is empty") + block = table_block([record for _, record in records]) if find_table else TableBlock(0, len(records)) + if block.above: + warnings.append(f"left out {_count(block.above, 'line')} above the table (a title, notes or metadata)") + if block.below: + warnings.append(f"left out {_count(block.below, 'line')} below the table (notes or a source line)") + if block.summary: + warnings.append( + f"left out {_count(block.summary, 'summary row')} at the end of the table (a total, sum, mean or similar)" + ) + header = records[block.start][1] + rows: list[list[str]] = [] + padded = 0 + for line_num, record in records[block.start + 1 : block.end]: + if len(rows) == MAX_ROWS: + raise ParseError("too_many_rows", f"the data has more than {MAX_ROWS:,} rows") + width = len(header) + if len(record) < width: + record = record + [""] * (width - len(record)) + padded += 1 + elif len(record) > width: + if any(cell.strip() for cell in record[width:]): + raise ParseError("unparseable", f"line {line_num} has more fields than the header") + record = record[:width] + rows.append(record) if padded: warnings.append(f"{_count(padded, 'row')} had fewer fields than the header; the missing cells are empty") return header, rows, source_format +@dataclass(frozen=True) +class TableBlock: + """Where the table sits among the pasted records: `records[start]` is the header, the rows + run to `end` (exclusive); `above`, `below` and `summary` count the records left out.""" + + start: int + end: int + above: int = 0 + below: int = 0 + summary: int = 0 + width: int = 0 + """The table's field count; 0 when no table with a value column was found (every record kept).""" + values: int = 0 + """The cells of the data rows that read as values.""" + + +def table_block(records: Sequence[Sequence[str]]) -> TableBlock: + """The table among pasted records that may carry other lines (owner rule of 2026-10-14). + + Real pastes are rarely a bare table: an export starts with metadata (Open-Meteo writes + `latitude,longitude,…` and its values above the header), a user adds a sentence that + explains the columns, and a source line, a note or a total row follows the table. The + table is found by its data: its width is the most common field count (at least two + fields, the larger one on a tie), its value columns are the columns where at least + `VALUE_SHARE` of the filled cells read as a value (`_value_like`) in the records of + that width, and a data row is a record of that width with a value in at least half of + the value columns. + + * The header is the record right above the first data row, past any wider record that + also holds the data (a ragged table's full rows, whose header is wider than its short + ones); everything above it is left out. When the first record already is a data row, + it is the header, as before. + * The rows end after the last data row and the records right below it that hold data at + the table's width or wider, or whose value cells are missing (`2024-01-11,,`); anything after that is + left out, such as a source line or a sentence, also one that its commas split into + the table's width. + * At most `MAX_SUMMARY_ROWS` rows at the end whose first filled cell is a summary label + (`SUMMARY_LABELS`: total, sum, mean, Summe, Gesamt…) are left out. + + A table without a value column (text only, prose) and a table of fewer than three records + of its width keep every record: the first is the header. + """ + whole = TableBlock(0, len(records)) + widths = Counter(len(record) for record in records if len(record) >= 2) + if not widths: + return whole + width = max(widths.items(), key=lambda item: (item[1], item[0]))[0] + sized = [record for record in records if len(record) == width] + if len(sized) < 3: + return whole + # The first record of the width is left out of the vote: it may be the header. + values = [j for j in range(width) if _mostly_values(row[j] for row in sized[1:])] + if not values: + return whole + + def data_row(record: Sequence[str]) -> bool: + return len(record) == width and 2 * sum(_value_like(record[j]) for j in values) >= len(values) + + def wide_data_row(record: Sequence[str]) -> bool: + # A data row with more fields than the most common width: a ragged table's full rows + # (`a,b,c` / `1,2,3` / `4,5` / `6,7`), whose header is wider than its short rows. + return len(record) >= width and 2 * sum(_value_like(record[j]) for j in values) >= len(values) + + def sparse_row(record: Sequence[str]) -> bool: + # A row of the table that lacks values (`2024-01-11,,`), not a sentence split at its commas. + return len(record) == width and all(_missing(record[j]) or _value_like(record[j]) for j in values) + + rows = [index for index, record in enumerate(records) if data_row(record)] + first, last = rows[0], rows[-1] + start = max(first - 1, 0) + while start > 0 and wide_data_row(records[start]): + start -= 1 + end = last + 1 + while end < len(records) and (sparse_row(records[end]) or wide_data_row(records[end])): + end += 1 + below = len(records) - end + summary = 0 + while summary < MAX_SUMMARY_ROWS and end - 1 > start and _summary_row(records[end - 1]): + end -= 1 + summary += 1 + found = sum(_value_like(records[index][j]) for index in rows for j in range(width)) + return TableBlock(start, end, above=start, below=below, summary=summary, width=width, values=found) + + +def _value_like(cell: str) -> bool: + """A cell that reads as a number or a date, also behind a unit (`65 %`, `€ 5`).""" + cell = cell.strip() + return bool(cell and (_looks_like_value(cell) or _TRAILING_UNIT.fullmatch(cell) or _LEADING_UNIT.fullmatch(cell))) + + +def _missing(cell: str) -> bool: + return cell.strip() in MISSING_VALUES + + +def _mostly_values(cells: Iterable[str]) -> bool: + present = [cell for cell in cells if not _missing(cell)] + return bool(present) and sum(_value_like(cell) for cell in present) >= VALUE_SHARE * len(present) + + +def _summary_row(record: Sequence[str]) -> bool: + label = next((cell.strip() for cell in record if not _missing(cell)), "") + return label.rstrip(":").strip().casefold() in SUMMARY_LABELS + + def _read_literal(text: str, warnings: list[str]) -> tuple[list[str], list[list[str]], SourceFormat] | None: """JSON, or a Python literal when the JSON reader refuses the text, tabulated alike; None when it is neither.""" if _too_deep(text): diff --git a/agents/anyplot/tools/session.py b/agents/anyplot/tools/session.py index ba52071aeb4..20f69f6b0ca 100644 --- a/agents/anyplot/tools/session.py +++ b/agents/anyplot/tools/session.py @@ -232,7 +232,8 @@ async def prepare_data(operations: list[dict[str, Any]], tool_context: ToolConte try: prepared = apply_operations(stored.as_parsed(), parsed_operations) # The columns `set_type text` made text stay text: the re-parse would type them by content again. - parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns) + # The CSV is the service's own: every row stays, also a `Total` category a sort moved to the end. + parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns, find_table=False) except PrepareError as exc: # The message names columns of the user's dataset at most, never a cell; fenced like every column name. return {"status": "error", "code": exc.code, "message": _fenced_lines([exc.message])} @@ -281,7 +282,7 @@ async def generate_data( return {"status": "error", "code": "invalid_arguments", "errors": _fenced_lines(validation_lines(exc))} try: generated = generate(spec) - parsed = parse_dataset(generated.csv) + parsed = parse_dataset(generated.csv, find_table=False) except GenerateError as exc: # The message names variables and series and quotes the model's own expression at most; fenced all the same. return {"status": "error", "code": exc.code, "message": _fenced_lines([exc.message])} diff --git a/agents/evals/fixtures/prep/en-hard-messy-export.json b/agents/evals/fixtures/prep/en-hard-messy-export.json index 5d35e0beca7..10130338f4e 100644 --- a/agents/evals/fixtures/prep/en-hard-messy-export.json +++ b/agents/evals/fixtures/prep/en-hard-messy-export.json @@ -2,11 +2,10 @@ "dataset_text": "Order Date,Customer,Item,Qty,Unit Price,Internal Code,Comment\n2024-03-02,Acme,Widget,4,12.5,X1,\n2024-03-02,Acme,Gadget,1,40,X2,\n2024-03-05,Globex,Widget,10,12.5,X1,rush\n2024-03-09,Initech,Gizmo,2,25,X3,\n2024-04-01,Globex,Gadget,3,40,X2,\n2024-04-11,Acme,Gizmo,5,25,X3,\n2024-04-20,Initech,Widget,7,12.5,X1,\nTOTAL,,,32,,,\n", "message": "This is a raw export. Remove the TOTAL line and the Internal Code and Comment columns, then give me the total quantity per item, largest first.", "expected_operations": [ - {"op": "drop_rows", "where": [{"column": "Order Date", "test": "eq", "value": "TOTAL"}]}, {"op": "drop_columns", "columns": ["Internal Code", "Comment"]}, {"op": "aggregate", "by": ["Item"], "column": "Qty", "how": "sum"}, {"op": "sort", "column": "Qty", "descending": true} ], "locale": "en", - "tags": ["hard", "chain", "messy", "drop_rows", "drop_columns", "aggregate", "sort"] + "tags": ["hard", "chain", "messy", "summary-row", "drop_columns", "aggregate", "sort"] } diff --git a/agents/main.py b/agents/main.py index 7216de4181b..3f25aa27286 100644 --- a/agents/main.py +++ b/agents/main.py @@ -12,7 +12,7 @@ | `GET /v1/eligibility?spec=&library=` | | `{eligible, status, reasons}` | | | `POST /v1/sessions` | `{user, spec_id, library, locale, snapshot}` | `{session_id, eligibility}` | `422 not_eligible` | | `POST /v1/sessions/{sid}/library` | `{library, snapshot}` | `{session_id, eligibility}` | `422 not_eligible`, `409 run_active` | -| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?}` | `{preview, profile, bindings, warnings, roles, suggestion, version}` (`suggestion`: the wide-table offer of `data/shape.py`, or null), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips only that fallback) | `413 too_long`, `422 unparseable`, `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `503 guard_unavailable` | +| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?}` | `{preview, profile, bindings, warnings, roles, suggestion, version}` (`suggestion`: the wide-table offer of `data/shape.py`, or null), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips that fallback and keeps every row) | `413 too_long`, `422 unparseable`, `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `503 guard_unavailable` | | `PUT /v1/sessions/{sid}/bindings` | `[{role, column}]` | `{bindings, complete, missing_roles, warnings}`: `warnings` names a role that needs values of 0 or more bound to a column with a negative value, or left open for that reason (`data/signs.py`); never a refusal | `409 run_active`, `422 invalid` (with `errors`, at most 20 lines) | | `PUT /v1/sessions/{sid}/language` | `{language}` (a short name or code, `^[A-Za-z][A-Za-z -]{0,39}$`) | `{language, source}`: the session language in its stored form (`English` is `en`), source `chosen`, which detection never overrides | `409 run_active`, `422` (the pattern) | | `POST /v1/sessions/{sid}/dataset/confirm` | | `{confirmed, rows, columns}`: step 1 of two, the user confirmed the dataset with its bindings; the pipeline runs only then | `409 run_active`, `422 no_dataset` | @@ -469,8 +469,9 @@ class DatasetBody(_Body): text: str = Field(min_length=1) confirmed: bool = False """The text is a table the user confirmed (the page's `.use()` on a transcription): the - route never hands it to the transcriber again, so a one-column reading can be stored; - the parse, the pre-filter, the judge and the store run as for any paste.""" + route never hands it to the transcriber again, so a one-column reading can be stored, + and the parse keeps every row of it (no `table_block` narrowing: the user has seen the + table row by row); the pre-filter, the judge and the store run as for any paste.""" class BindingBody(_Body): @@ -634,7 +635,8 @@ async def upload_dataset( raise AgentsError(413, "too_long") ledger = RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action") try: - parsed = parse_dataset(body.text) + # A confirmed transcription is the service's own CSV, which the user has seen row by row: no row is left out. + parsed = parse_dataset(body.text, find_table=not body.confirmed) except ParseError as exc: if exc.code == "too_long": raise AgentsError(413, "too_long") from None @@ -717,7 +719,7 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J ) csv_text = table_to_csv(reading.table) try: - parsed = parse_dataset(csv_text) + parsed = parse_dataset(csv_text, find_table=False) except ParseError: attribution("data_transcribe", ledger, outcome="unparseable", judge_tokens=reading.tokens) raise AgentsError(422, "unparseable") from None @@ -956,7 +958,7 @@ async def dataset_to_long(sid: SessionId, user: User, runtime: RuntimeDep) -> JS try: operations = OPERATIONS.validate_python([suggestion.operation()]) prepared = apply_operations(stored.as_parsed(), operations) - parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns) + parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns, find_table=False) except (PrepareError, ParseError, ValidationError): raise AgentsError(422, "no_suggestion") from None try: diff --git a/app/src/hooks/useAgentSession.test.ts b/app/src/hooks/useAgentSession.test.ts index f7237aa2e49..721cbfda1a0 100644 --- a/app/src/hooks/useAgentSession.test.ts +++ b/app/src/hooks/useAgentSession.test.ts @@ -467,7 +467,7 @@ describe('useAgentSession: data and bindings', () => { await act(() => result.current.confirmTranscription()); expect(calls.filter(call => call.path.endsWith('/dataset')).map(call => call.body)).toEqual([ { text: FREE_TEXT }, - { text: TRANSCRIBED.text, confirmed: true }, // the server skips only its free-text fallback + { text: TRANSCRIBED.text, confirmed: true }, // the server skips its free-text fallback and keeps every row ]); expect(result.current.state.transcription).toBeNull(); expect(result.current.state.dataset?.parsed).toEqual(DATASET); diff --git a/app/src/lib/agent.ts b/app/src/lib/agent.ts index d1820b9b993..9a31902e2cc 100644 --- a/app/src/lib/agent.ts +++ b/app/src/lib/agent.ts @@ -628,7 +628,8 @@ export const agentApi = { * A stored dataset, or a transcription of free text to confirm first (`isTranscribed`). * `confirmed` marks the table the user accepted with `.use()`: the server then never * hands the text to its transcriber again (a one-column table would otherwise be read - * again), while the parse, the pre-filter, the judge and the store run as usual. + * again) and keeps every row of it (no lines left out as notes or totals), while the + * pre-filter, the judge and the store run as usual. * Text that asks for a formula (`sin(x) von -pi bis pi`) answers `422 formula` and * stores nothing: the chat's `generate_data` computes such a table. */ diff --git a/changelog.d/parser-table-block.md b/changelog.d/parser-table-block.md new file mode 100644 index 00000000000..ed52cdf872a --- /dev/null +++ b/changelog.d/parser-table-block.md @@ -0,0 +1,9 @@ +### Changed + +- **Pasted data with notes, metadata or a total around the table parses.** The agent's + parser finds the table among the pasted lines by its data rows: a title, an export's + metadata (such as the two lines above an Open-Meteo download's header) or a sentence + that explains the columns above it, a source line or a note below it, and up to three + summary rows at the end (`Total`, `Summe`, `Mittelwert`…) are left out, each with a + warning in the preview. Before, the first line was always the header, so a raw + Open-Meteo download lost its data columns. (#12166) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 52918a3b17b..06b30a3e048 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -83,7 +83,7 @@ Deviations from the first sketch, each evidence-backed: there is no data-intake 1. The `.adapt()` overlay button in `SpecDetailView` does a full navigation to `/debug/agent?spec=&library=&language=`. 2. `POST /debug/agent/sessions {spec_id, library, locale}`. The BFF validates the spec and library against `core/constants.py` and the enabled libraries from anyplot-agents `GET /v1/status` (cached), loads a catalogue snapshot through `core/catalogue/queries.py` (title, description, data roles, notes, noqa-stripped code, library version), derives the user id, and forwards it. anyplot-agents normalises the code, runs the readiness scan and the SECURITY validator, and answers `422 not_eligible` for blocked pairs (the 13 map specs, validator failures, files without a `THEME` block or with a non-literal savefig target). anyplot-agents has no database access in phase 1. `GET /debug/agent/eligibility?spec=&library=` gives the button the same answer. `locale` is the browser's locale, recorded with a feedback case; it sets no language, and every session starts in English (see [Session language](#session-language)). -3. `POST .../dataset {text}` runs the deterministic parse and returns a 20-row preview, the `DatasetProfile`, default bindings and warnings. A deterministic pre-filter over every header and cell for text addressed to an AI, then the dataset judge with a data rubric, refuse instruction-like datasets. Both look for injection only: names, postal and e-mail addresses, phone numbers, web addresses and amounts are the user's data and plot like any other value. The judge sees every header and the same five sample rows and five top values the root and the adapter get, with cells in full, in parts of about 4,000 characters, one judge call per part (one part for every eval fixture). No other LLM call happens for a table. Text that is no table the parser can read (prose such as "a bar chart with dog 7, cat 3 and bird 4", a list typed one value per line; at most 4,000 characters, never a JSON array or object, which the parser reads or refuses itself) takes the transcriber path instead (owner rule of 2026-10-11): the pre-filter runs on the raw text, the judge model reads it under the `TranscribedTable` schema (`agents/anyplot/data/transcribe.py`, rubric `prompts/transcriber.md`: keep every value as written, invent and compute nothing, a note on anything ambiguous), the table's CSV is parsed like a paste, and the route answers `transcribed: true` with that CSV, the preview and the note, storing nothing. The page shows the table for the user to confirm or discard, because a wrong reading makes everything after it wrong; confirming posts the CSV through this same route with `confirmed: true`, which skips only the free-text fallback (a one-column table would otherwise be read again), while the deterministic parse, the pre-filter, the judge and the store run as for any paste. +3. `POST .../dataset {text}` runs the deterministic parse and returns a 20-row preview, the `DatasetProfile`, default bindings and warnings. A deterministic pre-filter over every header and cell for text addressed to an AI, then the dataset judge with a data rubric, refuse instruction-like datasets. Both look for injection only: names, postal and e-mail addresses, phone numbers, web addresses and amounts are the user's data and plot like any other value. The judge sees every header and the same five sample rows and five top values the root and the adapter get, with cells in full, in parts of about 4,000 characters, one judge call per part (one part for every eval fixture). No other LLM call happens for a table. Text that is no table the parser can read (prose such as "a bar chart with dog 7, cat 3 and bird 4", a list typed one value per line; at most 4,000 characters, never a JSON array or object, which the parser reads or refuses itself) takes the transcriber path instead (owner rule of 2026-10-11): the pre-filter runs on the raw text, the judge model reads it under the `TranscribedTable` schema (`agents/anyplot/data/transcribe.py`, rubric `prompts/transcriber.md`: keep every value as written, invent and compute nothing, a note on anything ambiguous), the table's CSV is parsed like a paste, and the route answers `transcribed: true` with that CSV, the preview and the note, storing nothing. The page shows the table for the user to confirm or discard, because a wrong reading makes everything after it wrong; confirming posts the CSV through this same route with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and the narrowing to the table (the user has seen every row, so no line is left out as a note or a total), while the deterministic parse, the pre-filter, the judge and the store run as for any paste. 4. The user adjusts the bindings in dropdowns (`PUT .../bindings`), confirms the data with its bindings (`POST .../dataset/confirm`, step 1 of two, see "Two steps: data first, then the plot" below) and clicks **Create plot**, which sends `POST .../messages {action: "create_plot"}`. The server sets the context variable `REQUEST_KIND=action`, so the judge is skipped, and sends a fixed message. Every turn first enters the run queue (see the bounds table below): the stream sends `ready`, then `status{step:"queued", position, waiting}` while the run waits, and the request deadline starts only when the run leaves the queue. 5. Root call 1 calls `plot_pipeline({})`. `PipelineArgs` has no spec, library or dataset field; those come from server-set session state, so neither the model nor an injection can redirect the pipeline. Its `theme` (`light` by default) is the one theme the run renders and the reviewer sees; the root sets `dark` only when the user asks for a dark plot, and keeps the latest version's theme, which its session block names, for a change. Without a dataset, with incomplete bindings or without the user's confirmation of the data the pipeline returns `PlotResult(status="not_ready", reason=...)` (`no_dataset`, `incomplete_bindings`, `data_unconfirmed`) before any LLM call. 6. The pipeline streams `adapting`, `checking`, `rendering`, `reviewing` and `repairing` and always ends with `Event(output=PlotResult)`. A `None` output would raise `NodeInterruptedError`. @@ -236,7 +236,7 @@ Schemas live in `agents/anyplot/schemas.py`: `ColumnProfile{name≤64, dtype, mi ### Parse -`agents/anyplot/data/parse.py` has no ADK import. Limits: UTF-8 at most 200 KB, 50 columns, 20,000 rows and 200 characters per cell. It rejects NUL and control characters, strips the BOM, applies NFC, and strips zero-width and bidi control characters. Headers are canonicalised (NFC, control characters stripped, names over 64 characters renamed, duplicates suffixed, empty names become `column_N`, each rename a warning), and the canonical names are written into `data.csv`, which is the single source of column names. Formats: JSON records, a column map, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`, source format `json_pairs`; nesting depth at most 2; a repeated key keeps its last value, with a warning), or delimited text with the delimiter sniffed from `, ; \t |`. A Python literal (`[`, `{` or `(`, also behind an assignment such as `cat_day = {...}`, which is stripped) is read like JSON with `ast.literal_eval`, which evaluates literals only; a name or a call is refused, and a bracket count that skips quoted strings and comments refuses more than 50 nesting levels before anything is evaluated. Text that starts with a bare `(` and is neither JSON nor a literal (a header such as `(in %) Land`, prose in parentheses) is delimited text as before. Everything is read as text first, then dates with explicit formats (an ambiguous day/month order is a warning), then numbers; a decimal comma is accepted only with a date guard. A column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, `CHF 1.200`, also with a no-break space between number and unit) is read as numbers, the unit moving into the column name (`Anteil (%)`) through the canonical-header path, with a warning, unless the header already names the unit; a column where only some cells carry a unit, or two units, stays text, as does one whose "unit" is a month, a boolean or an ordinal suffix (`5 Jan`, `1st`). The output is a canonical `data.csv` (comma, dot decimal, ISO dates) in a session-scoped dataset store (in memory, tied to the session's lifetime, the idle sweeper and the DELETE route; never a `user:` artifact, which would outlive the session and be listable by the model), plus the profile and default bindings (role dtype match, then name similarity, each column once; roles without typed bullets stay empty and the UI asks; a role that needs values of 0 or more never takes a column with a negative value, see `agents/anyplot/data/signs.py`: role names that are magnitudes in every spec, such as `radius`, `size`, `count`, `std`, `weight`, and `value` only in part-of-whole, area and flow plots such as pie, donut, treemap, sunburst, icicle, waffle, marimekko, sankey, rose and the basic funnel, never in signed ones such as waterfall or diverging bars; a binding of one by hand stays valid and becomes a warning in the session block, the `set_bindings` result, the PUT route's `warnings` and at the page's dropdown, through the role's `non_negative` flag). `bindings.apply(session, bindings, source)` validates roles against the snapshot's data roles and columns against the profile; the PUT route (through `session_service.append_event` with a `state_delta`) and the `set_bindings` tool both call it, and both answer `409` while a run is active. +`agents/anyplot/data/parse.py` has no ADK import. Limits: UTF-8 at most 200 KB, 50 columns, 20,000 rows and 200 characters per cell. It rejects NUL and control characters, strips the BOM, applies NFC, and strips zero-width and bidi control characters. Headers are canonicalised (NFC, control characters stripped, names over 64 characters renamed, duplicates suffixed, empty names become `column_N`, each rename a warning), and the canonical names are written into `data.csv`, which is the single source of column names. Formats: JSON records, a column map, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`, source format `json_pairs`; nesting depth at most 2; a repeated key keeps its last value, with a warning), or delimited text with the delimiter sniffed from `, ; \t |`. Pasted text is rarely a bare table, so since 2026-10-14 (owner rule) the delimited reader first finds the table among the lines (`table_block`): its data rows are the lines of the most common field count with numbers or dates in the same columns, the header is the line right above the first data row, and a title, an export's metadata (Open-Meteo writes two such lines above its header) or a sentence that explains the columns above it, a source line or a note below the last data row, and up to three summary rows at the end (`Total`, `Summe`, `Mittelwert`…) are left out, each kind with one warning in the preview; the delimiter is sniffed from the first 50 lines the same way, by the table with the most values. A table without a number or date column keeps every line, as before. A Python literal (`[`, `{` or `(`, also behind an assignment such as `cat_day = {...}`, which is stripped) is read like JSON with `ast.literal_eval`, which evaluates literals only; a name or a call is refused, and a bracket count that skips quoted strings and comments refuses more than 50 nesting levels before anything is evaluated. Text that starts with a bare `(` and is neither JSON nor a literal (a header such as `(in %) Land`, prose in parentheses) is delimited text as before. Everything is read as text first, then dates with explicit formats (an ambiguous day/month order is a warning), then numbers; a decimal comma is accepted only with a date guard. A column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, `CHF 1.200`, also with a no-break space between number and unit) is read as numbers, the unit moving into the column name (`Anteil (%)`) through the canonical-header path, with a warning, unless the header already names the unit; a column where only some cells carry a unit, or two units, stays text, as does one whose "unit" is a month, a boolean or an ordinal suffix (`5 Jan`, `1st`). The output is a canonical `data.csv` (comma, dot decimal, ISO dates) in a session-scoped dataset store (in memory, tied to the session's lifetime, the idle sweeper and the DELETE route; never a `user:` artifact, which would outlive the session and be listable by the model), plus the profile and default bindings (role dtype match, then name similarity, each column once; roles without typed bullets stay empty and the UI asks; a role that needs values of 0 or more never takes a column with a negative value, see `agents/anyplot/data/signs.py`: role names that are magnitudes in every spec, such as `radius`, `size`, `count`, `std`, `weight`, and `value` only in part-of-whole, area and flow plots such as pie, donut, treemap, sunburst, icicle, waffle, marimekko, sankey, rose and the basic funnel, never in signed ones such as waterfall or diverging bars; a binding of one by hand stays valid and becomes a warning in the session block, the `set_bindings` result, the PUT route's `warnings` and at the page's dropdown, through the role's `non_negative` flag). `bindings.apply(session, bindings, source)` validates roles against the snapshot's data roles and columns against the profile; the PUT route (through `session_service.append_event` with a `state_delta`) and the `set_bindings` tool both call it, and both answer `409` while a run is active. ### Adapt and validate diff --git a/docs/reference/api.md b/docs/reference/api.md index 284f73e7c9a..563f99f70f7 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -486,7 +486,7 @@ The routes mirror the agents service's `/v1` API. All paths below start with | `GET /eligibility?spec=&library=` | Spec id and library id | The agents service's answer, passed through | | `POST /sessions` | `{spec_id, library, locale}` | `{session_id, eligibility}`; `404 not_found` when the spec has no implementation for the library | | `POST /sessions/{sid}/library` | `{spec_id, library}` | Switches the library; the dataset and bindings stay | -| `POST /sessions/{sid}/dataset` | `{text, confirmed?}`, at most 200 KB (204,800 bytes) of UTF-8 | `{preview, profile, bindings, warnings, roles, suggestion, version}` for a table, stored as version 1 of the data (an earlier paste's undo history is dropped; `suggestion` is the wide-table offer or `null`, see [Data changes](#data-changes)): CSV, TSV, semicolon or pipe text; JSON as an array of records, an object of column lists, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`; a repeated key keeps its last value with a warning); the same shapes as a Python literal, also behind an assignment such as `cat_day = {...}`, with single quotes, trailing commas and tuples (text that starts with a bare `(` and is no literal is read as delimited text); a column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, also with a no-break space) is read as numbers and named ` ()` with a warning, unless the header already names the unit. For text that is no table (prose, a list typed one value per line) of at most 4,000 characters, `{transcribed: true, text, preview, profile, warnings, note}`, the table the judge model read, with its CSV in `text` and nothing stored: the page shows it for confirmation, and confirming posts that CSV through this route again with `confirmed: true`, which skips only the free-text fallback (a one-column table would otherwise be read again), never the parse, the pre-filter, the judge or the store. `413 too_long` above the limit, `422 unparseable` when neither the parser nor the model finds a table, `422 formula` when the text asks for a formula such as `sin(x) von -pi bis pi` or `y = x**2` (no model reads it; ask for the formula in the chat, where the assistant computes the table, see [Data changes](#data-changes)), `403 data_refused` when a header, a cell or the raw text addresses an AI (personal data such as names, addresses, e-mail addresses, phone numbers and web addresses is allowed), `503 guard_unavailable` when the judge model does not answer | +| `POST /sessions/{sid}/dataset` | `{text, confirmed?}`, at most 200 KB (204,800 bytes) of UTF-8 | `{preview, profile, bindings, warnings, roles, suggestion, version}` for a table, stored as version 1 of the data (an earlier paste's undo history is dropped; `suggestion` is the wide-table offer or `null`, see [Data changes](#data-changes)): CSV, TSV, semicolon or pipe text; JSON as an array of records, an object of column lists, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`; a repeated key keeps its last value with a warning); the same shapes as a Python literal, also behind an assignment such as `cat_day = {...}`, with single quotes, trailing commas and tuples (text that starts with a bare `(` and is no literal is read as delimited text); a column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, also with a no-break space) is read as numbers and named ` ()` with a warning, unless the header already names the unit. For text that is no table (prose, a list typed one value per line) of at most 4,000 characters, `{transcribed: true, text, preview, profile, warnings, note}`, the table the judge model read, with its CSV in `text` and nothing stored: the page shows it for confirmation, and confirming posts that CSV through this route again with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and keeps every row (no line is left out as a note or a total), never the parse, the pre-filter, the judge or the store. A pasted table may carry other lines: a title, metadata or notes above it, a source line or notes below it, and up to three summary rows at the end (`Total`, `Summe`…) are left out, each kind with one entry in `warnings`. `413 too_long` above the limit, `422 unparseable` when neither the parser nor the model finds a table, `422 formula` when the text asks for a formula such as `sin(x) von -pi bis pi` or `y = x**2` (no model reads it; ask for the formula in the chat, where the assistant computes the table, see [Data changes](#data-changes)), `403 data_refused` when a header, a cell or the raw text addresses an AI (personal data such as names, addresses, e-mail addresses, phone numbers and web addresses is allowed), `503 guard_unavailable` when the judge model does not answer | | `PUT /sessions/{sid}/bindings` | `[{role, column}]`, at most 50 | The agents service's `{bindings, complete, missing_roles, warnings}`; `422 no_dataset` before a parse, `422 invalid` with the check's `errors` lines for a binding the spec roles or the columns refuse, `409 run_active` while a turn or a render is in flight. A change of the bindings drops the data confirmation below | | `PUT /sessions/{sid}/language` | `{language}`: a short language name or code such as `de`, `Deutsch` or `Swedish`, 1 to 40 characters that match `^[A-Za-z][A-Za-z -]{0,39}$` | The agents service's `{language, source}`, the stored form (`English` comes back as `en`, `Swedish` as typed) with the source now `chosen`: the replies and the plot text of the next version follow it, and the language a typed message implies never replaces it (see [Session language](#session-language)). Versions made before keep their text. `422` for a value outside the pattern, `409 run_active` while a turn or a render is in flight | | `POST /sessions/{sid}/dataset/confirm` | none | Step 1 of two: you confirm that the stored dataset and its bindings fit; `{confirmed: true, rows, columns}`. The plot pipeline runs only on confirmed data (`create_plot` before it answers `plot{status: "not_ready", reason: "data_unconfirmed"}`); a new paste, a transcription's `.use()`, a binding change, a change of the data by the assistant or an undo drops the confirmation, a library switch keeps it. `422 no_dataset` before a parse, `409 run_active` while a turn or a render is in flight | diff --git a/tests/unit/agents/data/test_parse.py b/tests/unit/agents/data/test_parse.py index d63704bbc43..1b66cfd769c 100644 --- a/tests/unit/agents/data/test_parse.py +++ b/tests/unit/agents/data/test_parse.py @@ -503,6 +503,119 @@ def test_brackets_inside_strings_and_comments_are_not_nesting(self) -> None: """ +OPEN_METEO = """latitude,longitude,elevation,utc_offset_seconds,timezone,timezone_abbreviation +48.189804,16.377296,194.0,7200,Europe/Vienna,GMT+2 + +time,temperature_2m_max (°C),precipitation_sum (mm) +2025-01-01,-0.2,0.00 +2025-01-02,6.0,0.60 +2025-01-03,3.5,0.00 +2025-01-04,2.9,1.20 +""" +"""The shape of an Open-Meteo CSV download (owner's weather test of 2026-10-14): metadata above the header.""" + + +class TestTableAmongOtherLines: + """Pasted data with a title, metadata, notes or a total around the table (owner rule of 2026-10-14).""" + + def test_an_export_with_metadata_above_the_header(self) -> None: + parsed = parse_dataset(OPEN_METEO) + + assert list(parsed.column_dtypes) == ["time", "temperature_2m_max (°C)", "precipitation_sum (mm)"] + assert parsed.column_dtypes["time"] == "datetime" + assert parsed.profile.rows == 4 + assert parsed.profile.warnings == ["left out 2 lines above the table (a title, notes or metadata)"] + + def test_a_comment_above_and_below_also_with_commas(self) -> None: + text = ( + "Hier meine Wetterdaten, time ist das Datum, dann Temperatur und Regen:\n" + + OPEN_METEO + + "\nQuelle: Open-Meteo, ERA5, abgerufen am 14.10.2026\n" + ) + + parsed = parse_dataset(text) + + assert parsed.profile.rows == 4 + assert parsed.profile.warnings == [ + "left out 3 lines above the table (a title, notes or metadata)", + "left out 1 line below the table (notes or a source line)", + ] + + def test_a_sentence_below_the_table_that_its_commas_split_into_the_tables_width(self) -> None: + text = "Tag,Temp,Regen\nMo,3.5,0\nDi,4.1,1.2\nMi,2.0,0\nWerte gemessen in Wien, Temperatur in °C, Regen in mm\n" + + parsed = parse_dataset(text) + + assert parsed.profile.rows == 3 + assert parsed.profile.warnings == ["left out 1 line below the table (notes or a source line)"] + + def test_a_title_without_delimiter_above_a_semicolon_table_with_decimal_commas(self) -> None: + text = "Monatsmittel Wien 2025\nMonat;Temperatur;Niederschlag\nJan;1,2;30,5\nFeb;3,4;25,0\nMär;8,1;41,2\n" + + parsed = parse_dataset(text) + + assert parsed.profile.source_format == "semicolon" + assert parsed.column_dtypes == {"Monat": "text", "Temperatur": "number", "Niederschlag": "number"} + assert parsed.csv.splitlines()[1] == "Jan,1.2,30.5" + + @pytest.mark.parametrize("label", ["Summe", "TOTAL", "Gesamt:", "Mittelwert", "Average"]) + def test_a_summary_row_at_the_end_is_left_out(self, label: str) -> None: + text = f"Monat;Regen\nJan;30,5\nFeb;25,0\nMär;41,2\n{label};96,7\n" + + parsed = parse_dataset(text) + + assert parsed.profile.rows == 3 + assert parsed.profile.warnings == [ + "left out 1 summary row at the end of the table (a total, sum, mean or similar)" + ] + + def test_a_summary_row_in_a_date_column_and_a_source_line_below(self) -> None: + text = "Datum,Menge\n2024-03-02,4\n2024-03-05,10\n2024-03-09,2\nTOTAL,16\nExport: Shop-System v2\n" + + parsed = parse_dataset(text) + + assert parsed.column_dtypes == {"Datum": "datetime", "Menge": "integer"} + assert parsed.profile.rows == 3 + + def test_a_last_row_with_missing_values_stays(self) -> None: + parsed = parse_dataset("Datum,Temp,Regen\n2025-01-01,1.0,0\n2025-01-02,2.0,0\n2025-01-03,,\n") + + assert parsed.profile.rows == 3 + assert parsed.profile.warnings == [] + + def test_a_plain_table_and_a_text_table_are_unchanged(self) -> None: + plain = parse_dataset("x,y\n1,2\n3,4\n5,6\n") + text_only = parse_dataset("Name,Stadt\nAnna,Wien\nBen,Graz\nCem,Linz\n") + + assert (plain.csv, plain.profile.warnings) == ("x,y\n1,2\n3,4\n5,6\n", []) + assert text_only.profile.rows == 3 and text_only.profile.warnings == [] + + @pytest.mark.parametrize( + "text", ["a,b,c\n1,2,3\n4,5\n6,7\n8,9\n", "a,b,c\n4,5\n6,7\n1,2,3\n"], ids=["full-row-first", "full-row-last"] + ) + def test_a_ragged_table_keeps_its_wider_header_and_full_rows(self, text: str) -> None: + """Short rows are padded, as before: the most common width is no new header (#12166 review).""" + parsed = parse_dataset(text) + + assert list(parsed.column_dtypes) == ["a", "b", "c"] + assert parsed.profile.rows == text.count("\n") - 1 + assert parsed.profile.warnings[0].endswith("had fewer fields than the header; the missing cells are empty") + + def test_the_services_own_csv_keeps_every_row(self) -> None: + """A `Total` category a sort moved to the end stays in a re-parse of a prepared table (#12166 review).""" + sorted_csv = "Kategorie,Wert\nA,1\nB,2\nC,4\nTotal,3\n" + + assert parse_dataset(sorted_csv).profile.rows == 3 # a paste: the summary row is left out + assert parse_dataset(sorted_csv, find_table=False).profile.rows == 4 + assert parse_dataset(OPEN_METEO, find_table=False).profile.warnings[0].startswith("5 rows had fewer fields") + + def test_sniffing_reads_past_the_metadata(self) -> None: + assert sniff_delimiter("Wetterstation Hohe Warte\nDatum;Temp\n2025-01-01;1,5\n2025-01-02;2,5\n") == ( + ";", + "semicolon", + ) + + class TestPythonLiterals: def test_the_owners_dict_literal_with_its_assignment_and_trailing_comma(self) -> None: parsed = parse_dataset(OWNER_DICT_LITERAL) @@ -591,7 +704,9 @@ def test_a_bare_parenthesis_that_is_no_literal_is_delimited_text(self) -> None: assert header.column_dtypes == {"(in %) Land": "text", "Wert": "integer"} assert table.csv == "(a),b\n1,2\n" - assert stand.csv == "(Stand: 2024)\nLand;Wert\nA;1\nB;2\n" + # A title line above the table is left out since 2026-10-14 (`table_block`). + assert stand.csv == "Land,Wert\nA,1\nB,2\n" + assert stand.profile.warnings == ["left out 1 line above the table (a title, notes or metadata)"] assert error_code("(siehe Anhang) Die Werte sind 3, 4 und 5") == "no_table" assert parse_dataset("(x) Katze 3\n(y) Hund 7\n").csv == "(x) Katze 3\n(y) Hund 7\n"