feat(agents): find the table among pasted notes, metadata and totals

agents/anyplot/tools/session.py

@@ -232,7 +232,8 @@ async def prepare_data(operations: list[dict[str, Any]], tool_context: ToolConte
     try:
         prepared = apply_operations(stored.as_parsed(), parsed_operations)
         # The columns `set_type text` made text stay text: the re-parse would type them by content again.
-        parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns)
+        # The CSV is the service's own: every row stays, also a `Total` category a sort moved to the end.
+        parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns, find_table=False)
     except PrepareError as exc:
         # The message names columns of the user's dataset at most, never a cell; fenced like every column name.
         return {"status": "error", "code": exc.code, "message": _fenced_lines([exc.message])}
@@ -281,7 +282,7 @@ async def generate_data(
         return {"status": "error", "code": "invalid_arguments", "errors": _fenced_lines(validation_lines(exc))}
     try:
         generated = generate(spec)
-        parsed = parse_dataset(generated.csv)
+        parsed = parse_dataset(generated.csv, find_table=False)
     except GenerateError as exc:
         # The message names variables and series and quotes the model's own expression at most; fenced all the same.
         return {"status": "error", "code": exc.code, "message": _fenced_lines([exc.message])}

agents/evals/fixtures/prep/en-hard-messy-export.json

@@ -2,11 +2,10 @@
   "dataset_text": "Order Date,Customer,Item,Qty,Unit Price,Internal Code,Comment\n2024-03-02,Acme,Widget,4,12.5,X1,\n2024-03-02,Acme,Gadget,1,40,X2,\n2024-03-05,Globex,Widget,10,12.5,X1,rush\n2024-03-09,Initech,Gizmo,2,25,X3,\n2024-04-01,Globex,Gadget,3,40,X2,\n2024-04-11,Acme,Gizmo,5,25,X3,\n2024-04-20,Initech,Widget,7,12.5,X1,\nTOTAL,,,32,,,\n",
   "message": "This is a raw export. Remove the TOTAL line and the Internal Code and Comment columns, then give me the total quantity per item, largest first.",
   "expected_operations": [
-    {"op": "drop_rows", "where": [{"column": "Order Date", "test": "eq", "value": "TOTAL"}]},
     {"op": "drop_columns", "columns": ["Internal Code", "Comment"]},
     {"op": "aggregate", "by": ["Item"], "column": "Qty", "how": "sum"},
     {"op": "sort", "column": "Qty", "descending": true}
   ],
   "locale": "en",
-  "tags": ["hard", "chain", "messy", "drop_rows", "drop_columns", "aggregate", "sort"]
+  "tags": ["hard", "chain", "messy", "summary-row", "drop_columns", "aggregate", "sort"]
 }

agents/main.py

@@ -12,7 +12,7 @@
 | `GET /v1/eligibility?spec=&library=` | | `{eligible, status, reasons}` | |
 | `POST /v1/sessions` | `{user, spec_id, library, locale, snapshot}` | `{session_id, eligibility}` | `422 not_eligible` |
 | `POST /v1/sessions/{sid}/library` | `{library, snapshot}` | `{session_id, eligibility}` | `422 not_eligible`, `409 run_active` |
-| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?}` | `{preview, profile, bindings, warnings, roles, suggestion, version}` (`suggestion`: the wide-table offer of `data/shape.py`, or null), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips only that fallback) | `413 too_long`, `422 unparseable`, `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `503 guard_unavailable` |
+| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?}` | `{preview, profile, bindings, warnings, roles, suggestion, version}` (`suggestion`: the wide-table offer of `data/shape.py`, or null), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips that fallback and keeps every row) | `413 too_long`, `422 unparseable`, `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `503 guard_unavailable` |
 | `PUT /v1/sessions/{sid}/bindings` | `[{role, column}]` | `{bindings, complete, missing_roles, warnings}`: `warnings` names a role that needs values of 0 or more bound to a column with a negative value, or left open for that reason (`data/signs.py`); never a refusal | `409 run_active`, `422 invalid` (with `errors`, at most 20 lines) |
 | `PUT /v1/sessions/{sid}/language` | `{language}` (a short name or code, `^[A-Za-z][A-Za-z -]{0,39}$`) | `{language, source}`: the session language in its stored form (`English` is `en`), source `chosen`, which detection never overrides | `409 run_active`, `422` (the pattern) |
 | `POST /v1/sessions/{sid}/dataset/confirm` | | `{confirmed, rows, columns}`: step 1 of two, the user confirmed the dataset with its bindings; the pipeline runs only then | `409 run_active`, `422 no_dataset` |
@@ -469,8 +469,9 @@ class DatasetBody(_Body):
     text: str = Field(min_length=1)
     confirmed: bool = False
     """The text is a table the user confirmed (the page's `.use()` on a transcription): the
-    route never hands it to the transcriber again, so a one-column reading can be stored;
-    the parse, the pre-filter, the judge and the store run as for any paste."""
+    route never hands it to the transcriber again, so a one-column reading can be stored,
+    and the parse keeps every row of it (no `table_block` narrowing: the user has seen the
+    table row by row); the pre-filter, the judge and the store run as for any paste."""
 
 
 class BindingBody(_Body):
@@ -634,7 +635,8 @@ async def upload_dataset(
         raise AgentsError(413, "too_long")
     ledger = RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action")
     try:
-        parsed = parse_dataset(body.text)
+        # A confirmed transcription is the service's own CSV, which the user has seen row by row: no row is left out.
+        parsed = parse_dataset(body.text, find_table=not body.confirmed)
     except ParseError as exc:
         if exc.code == "too_long":
             raise AgentsError(413, "too_long") from None
@@ -717,7 +719,7 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J
     )
     csv_text = table_to_csv(reading.table)
     try:
-        parsed = parse_dataset(csv_text)
+        parsed = parse_dataset(csv_text, find_table=False)
     except ParseError:
         attribution("data_transcribe", ledger, outcome="unparseable", judge_tokens=reading.tokens)
         raise AgentsError(422, "unparseable") from None
@@ -956,7 +958,7 @@ async def dataset_to_long(sid: SessionId, user: User, runtime: RuntimeDep) -> JS
     try:
         operations = OPERATIONS.validate_python([suggestion.operation()])
         prepared = apply_operations(stored.as_parsed(), operations)
-        parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns)
+        parsed = parse_dataset(prepared.csv, text_columns=prepared.text_columns, find_table=False)
     except (PrepareError, ParseError, ValidationError):
         raise AgentsError(422, "no_suggestion") from None
     try:

app/src/hooks/useAgentSession.test.ts

@@ -467,7 +467,7 @@ describe('useAgentSession: data and bindings', () => {
     await act(() => result.current.confirmTranscription());
     expect(calls.filter(call => call.path.endsWith('/dataset')).map(call => call.body)).toEqual([
       { text: FREE_TEXT },
-      { text: TRANSCRIBED.text, confirmed: true }, // the server skips only its free-text fallback
+      { text: TRANSCRIBED.text, confirmed: true }, // the server skips its free-text fallback and keeps every row
     ]);
     expect(result.current.state.transcription).toBeNull();
     expect(result.current.state.dataset?.parsed).toEqual(DATASET);

app/src/lib/agent.ts

@@ -628,7 +628,8 @@ export const agentApi = {
    * A stored dataset, or a transcription of free text to confirm first (`isTranscribed`).
    * `confirmed` marks the table the user accepted with `.use()`: the server then never
    * hands the text to its transcriber again (a one-column table would otherwise be read
-   * again), while the parse, the pre-filter, the judge and the store run as usual.
+   * again) and keeps every row of it (no lines left out as notes or totals), while the
+   * pre-filter, the judge and the store run as usual.
    * Text that asks for a formula (`sin(x) von -pi bis pi`) answers `422 formula` and
    * stores nothing: the chat's `generate_data` computes such a table.
    */

changelog.d/parser-table-block.md

@@ -0,0 +1,9 @@
+### Changed
+
+- **Pasted data with notes, metadata or a total around the table parses.** The agent's
+  parser finds the table among the pasted lines by its data rows: a title, an export's
+  metadata (such as the two lines above an Open-Meteo download's header) or a sentence
+  that explains the columns above it, a source line or a note below it, and up to three
+  summary rows at the end (`Total`, `Summe`, `Mittelwert`…) are left out, each with a
+  warning in the preview. Before, the first line was always the header, so a raw
+  Open-Meteo download lost its data columns. (#12166)