diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index 950fd62aed3..f7c66214f5c 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -535,7 +535,7 @@ jobs: # What the image is built from: its Dockerfile, the service package, # the harness it copies, the dependency lock, README.md (copied next # to pyproject.toml) and the root .dockerignore (the build context). - RENDERER_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(agents/renderer/|agents/__init__\.py$|agents/anyplot/render/harness\.py$|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|\.github/workflows/ci-image\.yml$)' || true) + RENDERER_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(agents/renderer/|agents/__init__\.py$|agents/anyplot/render/harness\.py$|agents/anyplot/render/convert\.py$|tests/image/|tests/unit/agents/runtime/convert_fixtures\.py$|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|\.github/workflows/ci-image\.yml$)' || true) if [[ "$EVENT_NAME" == "workflow_dispatch" && "$FORCE_RUN" == "true" ]]; then echo "Manual trigger with force_run=true, will build the renderer image" @@ -610,6 +610,19 @@ jobs: EOF tail -n 1 harness.out + # The file conversion inside the image (harness.py --convert): every format the + # page sends, with the image's slimmed pyarrow (owner decision of 2026-10-15), + # its calamine and NumPy, the guard's preload, and the sandbox's environment. + # A file the Dockerfile's slimming removed but a reader needs fails here. + - name: Convert every data format through the harness + if: steps.check.outputs.should_build == 'true' + run: | + set -euo pipefail + docker run --rm --network none --read-only --tmpfs /tmp \ + --cap-drop ALL --security-opt no-new-privileges --user "$(id -u):$(id -g)" \ + -v "$PWD:/repo:ro" \ + anyplot-renderer:ci /app/.venv/bin/python -I /repo/tests/image/convert_smoke.py + # The fallback fonts: the harness appends them after the font the code chose, # so this fixture does what catalogue code does to the font setup (seaborn's # rcParams reset, a bold title, a monospace text) and draws seven of the @@ -689,6 +702,7 @@ jobs: run: | set -euo pipefail echo "image size: $(docker image inspect --format '{{.Size}}' anyplot-renderer:ci) bytes" + docker run --rm anyplot-renderer:ci du -sh /app/.venv/lib/python3.13/site-packages/pyarrow docker run --rm anyplot-renderer:ci sh -c 'grep PRETTY_NAME /etc/os-release; cat /usr/share/doc/fonts-noto/versions.txt; ls -l /usr/share/fonts/noto; du -sh /usr/share/fonts/noto; head -n 5 /usr/share/doc/fonts-noto/fonts-noto-core.copyright' # Also after a failed assert, so the PNGs show what went wrong. diff --git a/agents/README.md b/agents/README.md index 75c2aaca091..23ea41b9fef 100644 --- a/agents/README.md +++ b/agents/README.md @@ -92,6 +92,10 @@ Three rules hold for everything here: | `AGENT_SESSION_CLOSE_WARNING` | `true` | The counted refusal one before the closing one ends with the fixed `close_warning` sentence | | `AGENT_USER_PASTES_PER_MINUTE` | `10` | Dataset pastes per user within any 60 seconds, across sessions; one more is `429 rate_limited`. Only a paste that reaches the pre-filter or a judge counts | | `AGENT_USER_PASTES_PER_DAY` | `100` | Dataset pastes per user and UTC day; one more is `429 daily_limit` | +| `AGENT_USER_CONVERSIONS_PER_MINUTE` | `5` | File conversions (a dropped spreadsheet, Parquet, Arrow or NumPy file read in the renderer's sandbox) per user within any 60 seconds; one more is `429 rate_limited`. The paste that follows counts as one paste | +| `AGENT_USER_CONVERSIONS_PER_DAY` | `30` | File conversions per user and UTC day; one more is `429 daily_limit` | +| `AGENT_CONVERT_TIMEOUT_S` | `10` | Seconds one conversion may run in the sandbox, at most 15 | +| `AGENT_CONVERT_SLOT_WAIT_S` | `20` | Seconds a conversion waits for the render slot behind every pipeline render; then `503 capacity` | | `AGENT_MAX_ATTEMPTS` | `2` | Rounds per pipeline run, 1 to 3: the first and at most one repair round by default (a self-review adds an adapter call to a round, never a round); the eval harness sets 1 or 3 (`--max-attempts`) to measure no repair or a second repair round | | `AGENT_REVIEWER_MODE` | `catalogue` | The reviewer's rubric: `catalogue` (the offline scorer's rubric, verdict decided by code from the points) or `checklist` (the narrow checklist used before 2026-10-13) | | `AGENT_REVIEW_BAR`, `AGENT_STATUS_BAR` | `0.95`, `0.95` | `catalogue` reviews: the share of technical points (VQ, SC, DQ) below which a render gets the repair (1.0 before 2026-10-14; owner decision pending), and the ship bar at or above which it shows without the review's notes | diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 5d9900b909d..990742fe7a5 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -117,7 +117,7 @@ from collections import Counter from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace -from datetime import datetime +from datetime import datetime, timedelta, timezone from itertools import pairwise, product from typing import Any, Literal @@ -353,6 +353,12 @@ _DOT_DATE = re.compile(r"([0-9]{1,2})\.([0-9]{1,2})\.([0-9]{4})" + _LOCAL_TIME) _SLASH_DATE = re.compile(r"([0-9]{1,2})/([0-9]{1,2})/([0-9]{4})" + _LOCAL_TIME) +_ISO_STAMP = re.compile( + r"([0-9]{4})-([0-9]{2})-([0-9]{2})" + r"(?:[T ]([0-9]{2}):([0-9]{2})(?::([0-9]{2})(?:[.,]([0-9]{1,9}))?)?(Z|[+-][0-9]{2}(?::?[0-9]{2})?)?)?" +) +"""An ISO date, or a timestamp with seconds, a fraction of up to nine digits and an offset (`_as_iso_stamps`).""" + _DATE_FORMATS: tuple[tuple[re.Pattern[str], str], ...] = ( (_ISO_DATE, "iso"), (_DOT_DATE, "dot"), @@ -441,6 +447,8 @@ class _Column: bare: list[str] = field(default_factory=list) # the cells the number typing read: `source`, or the number parts numbers: list[float] = field(default_factory=list) dates: list[datetime] = field(default_factory=list) + date_texts: list[str] = field(default_factory=list) # the canonical text of each of `dates`, for exact min/max + date_rests: list[int] = field(default_factory=list) # nanoseconds past each of `dates`' microsecond, for order with_time: bool = False decimal_comma: bool = False notes: list[str] = field(default_factory=list) @@ -1948,7 +1956,8 @@ def _type_column(column: _Column) -> None: """Give the column its dtype and canonical cells, in this order of attempts. 1. **datetime**: every non-missing cell matches one format: ISO - `YYYY-MM-DD[(T| )HH:MM[:SS]]`, `DD.MM.YYYY`, or `DD/MM/YYYY` / `MM/DD/YYYY`, + `YYYY-MM-DD[(T| )HH:MM[:SS[.fraction]][offset]]` (`_as_iso_stamps`: up to nine + fraction digits and the offset kept as written), `DD.MM.YYYY`, or `DD/MM/YYYY` / `MM/DD/YYYY`, the last three with an optional ` HH:MM[:SS]` and one- or two-digit day and month. A slash column is day-first when a first part exceeds 12, month-first when a second part does, not a date when both happen, and day-first with a @@ -2199,7 +2208,85 @@ def _match_all(pattern: re.Pattern[str], cells: list[str]) -> dict[str, tuple[in return fields +def _as_iso_stamps(column: _Column, present: list[str]) -> bool | None: + """ISO dates and timestamps with every digit and the offset as written (owner rule of 2026-10-15). + + None when a cell is no ISO date, so the other formats are tried. A fraction of up to nine + digits stays (each cell padded to the column's longest fraction, which changes no value and + lets `pd.read_csv(parse_dates=...)` read the column as one format), and so does an offset + (`Z`, `+01`, `+0100`, `+01:00`, written `+01:00`; `Z` as `+00:00`). A column that mixes + cells with and without an offset is not a date column: no time is given a zone by guess. + A column with several offsets (summer and winter time) is written in UTC (`+00:00`) with a + note naming the offsets, because pandas reads one offset per column; the instants stay. + """ + matches = [] + for cell in present: + match = _ISO_STAMP.fullmatch(cell) + if match is None: + return None + matches.append((cell, match)) + offsets: dict[str, int | None] = {} + stamps: dict[str, tuple[datetime, str]] = {} + for cell, match in matches: + year, month, day = int(match.group(1)), int(match.group(2)), int(match.group(3)) + hour, minute, second = (int(match.group(index) or 0) for index in (4, 5, 6)) + fraction, offset = match.group(7) or "", match.group(8) + try: + stamp = datetime(year, month, day, hour, minute, second, int((fraction + "000000")[:6])) + except ValueError: + return False + minutes: int | None = None + if offset is not None: + digits = offset[1:].replace(":", "") if offset != "Z" else "0000" + minutes = int(digits[:2]) * 60 + int(digits[2:4] or 0) + if minutes >= 24 * 60: + return False + minutes = -minutes if offset.startswith("-") else minutes + stamp = stamp.replace(tzinfo=timezone(timedelta(minutes=minutes))) + offsets[cell] = minutes + stamps[cell] = (stamp, fraction) + zones = set(offsets.values()) + if None in zones and len(zones) > 1: + return False + mixed = len(zones) > 1 + if mixed: + shown = ", ".join(_offset_text(value) for value in sorted(zone for zone in zones if zone is not None)) + column.notes.append(f"times with the UTC offsets {shown} written in UTC (+00:00); the instants are the same") + digits = max((len(fraction) for _, fraction in stamps.values()), default=0) + column.with_time = any(":" in cell for cell in present) + texts: dict[str, str] = {} + for cell, (stamp, fraction) in stamps.items(): + if mixed: + stamp = stamp.astimezone(timezone.utc) + if not column.with_time: + texts[cell] = stamp.date().isoformat() + continue + text = stamp.replace(microsecond=0, tzinfo=None).isoformat(timespec="seconds") + if digits: + text += "." + fraction.ljust(digits, "0") + if stamp.tzinfo is not None: + text += _offset_text(0 if mixed else offsets[cell]) + texts[cell] = text + column.dtype = "datetime" + column.dates = [stamps[cell][0] for cell in column.source if cell] + column.date_texts = [texts[cell] for cell in column.source if cell] + column.date_rests = [int((stamps[cell][1] + "000000000")[6:9]) for cell in column.source if cell] + column.cells = [texts[cell] if cell else "" for cell in column.source] + return True + + +def _offset_text(minutes: int | None) -> str: + """An offset in minutes as `+HH:MM`.""" + value = minutes or 0 + sign = "-" if value < 0 else "+" + hours, rest = divmod(abs(value), 60) + return f"{sign}{hours:02d}:{rest:02d}" + + def _as_dates(column: _Column, present: list[str]) -> bool: + stamped = _as_iso_stamps(column, present) + if stamped is not None: + return stamped kind, fields = "", None for pattern, candidate in _DATE_FORMATS: fields = _match_all(pattern, present) @@ -2540,6 +2627,9 @@ def _profile_column(column: _Column) -> ColumnProfile: if column.dtype in ("integer", "number") and column.numbers: minimum, maximum = min(column.numbers), max(column.numbers) mean, median = _statistics(column.numbers) + elif column.dtype == "datetime" and column.dates and column.date_texts: + order = sorted(range(len(column.dates)), key=lambda index: (column.dates[index], column.date_rests[index])) + minimum, maximum = column.date_texts[order[0]], column.date_texts[order[-1]] elif column.dtype == "datetime" and column.dates: minimum, maximum = _iso(min(column.dates), column.with_time), _iso(max(column.dates), column.with_time) elif column.dtype in ("text", "boolean"): diff --git a/agents/anyplot/data/prepare.py b/agents/anyplot/data/prepare.py index 72118e7f7e9..9596d25b4ae 100644 --- a/agents/anyplot/data/prepare.py +++ b/agents/anyplot/data/prepare.py @@ -1157,7 +1157,9 @@ def _cell_text(value: object, with_time: bool) -> str | None: number = float(value) return None if math.isnan(number) else repr(number) if isinstance(value, pd.Timestamp): - return value.isoformat(timespec="seconds") if with_time else value.date().isoformat() + # Every digit and the offset as the value holds them (owner rule of 2026-10-15): a fraction is written + # only when there is one, an offset only for a zone-aware value. + return value.isoformat() if with_time else value.date().isoformat() return str(value) diff --git a/agents/anyplot/plugins/ledger.py b/agents/anyplot/plugins/ledger.py index f8e0253c6b5..9bb10a56e94 100644 --- a/agents/anyplot/plugins/ledger.py +++ b/agents/anyplot/plugins/ledger.py @@ -169,13 +169,14 @@ class _Day: user_data_runs: dict[str, int] = field(default_factory=dict) user_messages: dict[str, int] = field(default_factory=dict) user_pastes: dict[str, int] = field(default_factory=dict) + user_conversions: dict[str, int] = field(default_factory=dict) MESSAGE_WINDOW_S = 60.0 """The sliding window of `AGENT_USER_MESSAGES_PER_MINUTE` and `AGENT_USER_PASTES_PER_MINUTE`.""" MessageLimit = Literal["rate_limited", "daily_limit"] """Why `message_limit` or `paste_limit` refuses: the minute's count, or the day's.""" -WindowKind = Literal["message", "paste"] +WindowKind = Literal["message", "paste", "convert"] class UsageBook: @@ -291,6 +292,27 @@ def add_paste(self, user_id: str) -> None: day.user_pastes[user_id] = day.user_pastes.get(user_id, 0) + 1 self._stamp("paste", user_id) + def user_conversions(self, user_id: str) -> int: + """The user's file conversions today that passed `convert_limit`.""" + return self._current().user_conversions.get(user_id, 0) + + def convert_limit(self, user_id: str, settings: AgentSettings) -> MessageLimit | None: + """Why the user may not convert another file now, or None: `AGENT_USER_CONVERSIONS_PER_MINUTE` and + `_PER_DAY`, across sessions, checked before the file reaches the sandbox.""" + return self._limit( + "convert", + user_id, + self.user_conversions(user_id), + settings.user_conversions_per_minute, + settings.user_conversions_per_day, + ) + + def add_conversion(self, user_id: str) -> None: + """Count one file conversion against the user's minute and day.""" + day = self._current() + day.user_conversions[user_id] = day.user_conversions.get(user_id, 0) + 1 + self._stamp("convert", user_id) + def _limit( self, kind: WindowKind, user_id: str, today_count: int, per_minute: int, per_day: int ) -> MessageLimit | None: diff --git a/agents/anyplot/render/backends/fake.py b/agents/anyplot/render/backends/fake.py index af5cb94372a..ccc7a39e68d 100644 --- a/agents/anyplot/render/backends/fake.py +++ b/agents/anyplot/render/backends/fake.py @@ -14,9 +14,10 @@ from PIL import Image, ImageDraw +from agents.renderer.wire import ConvertMeta from core.palette import IMPRINT -from ..contract import DataJob, DataOutput, RenderJob, RenderResult, Theme, ThemeOutput +from ..contract import ConvertJob, ConvertOutput, DataJob, DataOutput, RenderJob, RenderResult, Theme, ThemeOutput BACKGROUNDS: dict[Theme, str] = {"light": "#FAF8F1", "dark": "#1A1A17"} @@ -57,13 +58,15 @@ class FakeOutcome: FIXTURE_CSV = "x,y\n0,0\n1,1\n2,4\n3,9\n" """What a fake data run returns by default: the table a snippet computing `y = x**2` for 0 to 3 writes.""" DataScript = Callable[[DataJob], DataOutput] +ConvertScript = Callable[[ConvertJob], ConvertOutput] @dataclass class FakeBackend: """Fixture renders; `script(job, theme)` decides the outcome (default: a clean render). - A data run returns `FIXTURE_CSV` with exit code 0 unless `data_script(job)` decides; + A data run returns `FIXTURE_CSV` with exit code 0 unless `data_script(job)` decides, + and so does a conversion (as a one-sheet workbook) unless `convert_script(job)` does; no code runs either way, so the fake is for tests and offline development only. """ @@ -72,6 +75,15 @@ class FakeBackend: jobs: list[RenderJob] = field(default_factory=list) data_script: DataScript | None = None data_jobs: list[DataJob] = field(default_factory=list) + convert_script: ConvertScript | None = None + convert_jobs: list[ConvertJob] = field(default_factory=list) + + async def convert(self, job: ConvertJob) -> ConvertOutput: + self.convert_jobs.append(job) + if self.convert_script is not None: + return self.convert_script(job) + meta = ConvertMeta(kind="xlsx", rows=4, columns=2, sheet="Sheet1", sheets=["Sheet1"], sheet_count=1) + return ConvertOutput(exit_code=0, text=FIXTURE_CSV, meta=meta, wall_s=0.1) async def run_data(self, job: DataJob) -> DataOutput: self.data_jobs.append(job) diff --git a/agents/anyplot/render/backends/local.py b/agents/anyplot/render/backends/local.py index 0ea96943d84..ab55557abad 100644 --- a/agents/anyplot/render/backends/local.py +++ b/agents/anyplot/render/backends/local.py @@ -23,6 +23,9 @@ (`harness.py --data snippet.py`), no theme and no plotting settings, and reads the harness's `status.json` (its exit code and a refusal's measured value and cap, as the renderer does) and the `out.csv` it wrote after exit code 0, under the computed-data cap. +A conversion (`convert`) does the same with the harness in convert mode on `input.bin` +(the converter mounted read-only next to the harness), and reads `out.csv` under the paste +cap and `meta.json`. The backend refuses `ENVIRONMENT=production` and fails when Docker is missing: there is no bare-subprocess fallback, because that would run model-written code on @@ -30,16 +33,29 @@ """ import asyncio +import json import os import shutil import tempfile import time from pathlib import Path -from agents.renderer.wire import DATA_STATUS_FILE, MAX_DATA_STATUS_BYTES, parse_data_status +from agents.renderer.wire import ( + CONVERT_META_FILE, + CONVERT_REQUEST_FILE, + DATA_STATUS_FILE, + MAX_CONVERT_META_BYTES, + MAX_CONVERT_TEXT_BYTES, + MAX_DATA_STATUS_BYTES, + ConvertMeta, + convert_refusal, + parse_data_status, +) from ...data.parse import MAX_COMPUTED_BYTES from ..contract import ( + ConvertJob, + ConvertOutput, DataJob, DataOutput, RendererUnavailable, @@ -51,7 +67,14 @@ ) from ..harness import DATA_OUT_FILE from ..png import PngRejected, read_output -from ..runtimes.python import DATA_FILE_NAME, HARNESS_SOURCE, IMAGE_HARNESS +from ..runtimes.python import ( + CONVERT_INPUT_NAME, + CONVERTER_SOURCE, + DATA_FILE_NAME, + HARNESS_SOURCE, + IMAGE_CONVERTER, + IMAGE_HARNESS, +) MEMORY_LIMIT = "2g" @@ -134,6 +157,8 @@ def _isolation(self, name: str, run_dir: Path) -> list[str]: f"{run_dir}:/work", "-v", f"{HARNESS_SOURCE}:{IMAGE_HARNESS}:ro", + "-v", + f"{CONVERTER_SOURCE}:{IMAGE_CONVERTER}:ro", "-w", "/work", ] @@ -228,6 +253,66 @@ async def run_data(self, job: DataJob) -> DataOutput: finally: shutil.rmtree(run_dir, ignore_errors=True) + def convert_argv(self, job: ConvertJob, run_dir: Path) -> list[str]: + """The docker command of a conversion: the render's isolation, the harness in convert mode.""" + argv = self._isolation(f"r-{job.job_id}-convert", run_dir) + for key, value in self.runtime.convert_env().items(): + argv += ["-e", f"{key}={value}"] + argv += [self.image, *self.runtime.convert_command()] + if argv[argv.index("--network") + 1] != "none" or any("--network=" in part for part in argv): + raise AssertionError("a conversion container must never have network access") + return argv + + async def convert(self, job: ConvertJob) -> ConvertOutput: + """Convert a file in a throwaway container; read the text and `meta.json` after exit code 0.""" + run_dir = Path(tempfile.mkdtemp(prefix=f"anyplot-convert-{job.job_id}-")) + name = f"r-{job.job_id}-convert" + try: + os.chmod(run_dir, 0o700) + (run_dir / CONVERT_INPUT_NAME).write_bytes(job.data) + (run_dir / CONVERT_REQUEST_FILE).write_text(json.dumps({"sheet": job.sheet}), encoding="utf-8") + started = time.monotonic() + process = await asyncio.create_subprocess_exec( + *self.convert_argv(job, run_dir), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + try: + await asyncio.wait_for(_drain_and_wait(process), timeout=job.timeout_s) + except TimeoutError: + await self._kill(name, process) + return ConvertOutput( + exit_code=None, timed_out=True, reason="timeout", wall_s=time.monotonic() - started + ) + except asyncio.CancelledError: + await asyncio.shield(self._kill(name, process)) + raise + wall = time.monotonic() - started + try: + status = parse_data_status(read_output(run_dir / DATA_STATUS_FILE, MAX_DATA_STATUS_BYTES)) + except PngRejected: + status = None + exit_code = status.exit if status is not None else process.returncode + if exit_code != 0: + measured, limit = (status.measured, status.limit) if status is not None else (None, None) + return ConvertOutput( + exit_code=exit_code, refusal=convert_refusal(exit_code), measured=measured, limit=limit, wall_s=wall + ) + try: + raw = read_output(run_dir / DATA_OUT_FILE, MAX_CONVERT_TEXT_BYTES) + meta_raw = read_output(run_dir / CONVERT_META_FILE, MAX_CONVERT_META_BYTES) + if raw is None or meta_raw is None: + raise ValueError("missing output") + text = raw.decode("utf-8") + meta = ConvertMeta.model_validate_json(meta_raw, strict=True) + except (PngRejected, ValueError): + return ConvertOutput(exit_code=0, reason="output_rejected", wall_s=wall) + return ConvertOutput(exit_code=0, text=text, meta=meta, wall_s=wall) + finally: + shutil.rmtree(run_dir, ignore_errors=True) + async def render(self, job: RenderJob) -> RenderResult: run_dir = Path(tempfile.mkdtemp(prefix=f"anyplot-run-{job.job_id}-")) try: diff --git a/agents/anyplot/render/backends/remote.py b/agents/anyplot/render/backends/remote.py index b484181dd3a..04f31e52936 100644 --- a/agents/anyplot/render/backends/remote.py +++ b/agents/anyplot/render/backends/remote.py @@ -53,6 +53,10 @@ before the route; it raises `DataRunUnsupported`, which the tool answers as `unavailable` instead of failing the turn. +**Conversions.** `convert` posts a dropped binary file to `AGENT_RENDER_URL/convert` (base64 +in JSON) the same way; a `404` raises `DataRunUnsupported`, which the dataset route answers +`503 convert_unavailable`. + No semaphore here: `SerialRenderer` (`render/serial.py`) already hands this backend one theme at a time, and the renderer keeps its own single render slot for the callers it serves together. @@ -77,9 +81,19 @@ from google.oauth2 import id_token as google_id_token from pydantic import ValidationError -from agents.renderer.wire import MAX_PNG_BYTES, DataRunRequest, DataRunResponse, RenderRequest, RenderResponse +from agents.renderer.wire import ( + MAX_PNG_BYTES, + ConvertRequest, + ConvertResponse, + DataRunRequest, + DataRunResponse, + RenderRequest, + RenderResponse, +) from ..contract import ( + ConvertJob, + ConvertOutput, DataJob, DataOutput, DataRunUnsupported, @@ -88,6 +102,7 @@ RenderResult, ThemeOutput, ) +from ..serial import RenderBusy logger = logging.getLogger(__name__) @@ -106,6 +121,9 @@ CANCEL_TIMEOUT_S = 5.0 FRONT_END_RETRY = frozenset({429, 500, 502, 503, 504}) _DETAIL = re.compile(r"^[a-z_]{1,32}$") +BUSY_DETAILS = frozenset({"busy", "low_memory"}) +"""The renderer's `503` details that mean a busy instance, not a missing one: a conversion answers them as +`RenderBusy`, so the route says `capacity` (try again) rather than `convert_unavailable`.""" _MAX_PNG_BASE64 = 4 * ((MAX_PNG_BYTES + 2) // 3) @@ -329,6 +347,68 @@ def _data_result(self, job: DataJob, response: httpx.Response) -> DataOutput: limit=answer.limit, ) + async def convert(self, job: ConvertJob) -> ConvertOutput: + """POST the file to `/convert` as base64; a renderer without the route (404) raises `DataRunUnsupported`.""" + try: + body = ConvertRequest.model_validate( + { + "job_id": job.job_id, + "data_base64": base64.b64encode(job.data).decode("ascii"), + "sheet": job.sheet, + "timeout_s": job.timeout_s, + } + ).model_dump(mode="json") + except ValidationError as exc: + fields = ",".join(sorted({str(error["loc"][0]) for error in exc.errors() if error["loc"]})) + raise _unavailable(f"the conversion does not fit the renderer contract ({fields})") from None + timeout = httpx.Timeout(job.timeout_s + RESPONSE_MARGIN_S, connect=CONNECT_TIMEOUT_S) + try: + response = await self._post(body, timeout, path="/convert") + except asyncio.CancelledError: + await self._cancel_remote(job.job_id) + raise + except httpx.TimeoutException as exc: + await self._cancel_remote(job.job_id) + raise _unavailable(f"the renderer did not answer the conversion in time ({type(exc).__name__})") from None + status = response.status_code + if status == 404: + logger.warning("remote renderer: no /convert route (HTTP 404); deploy the renderer first") + raise DataRunUnsupported("the renderer has no /convert route yet (HTTP 404)") + if status != 200: + detail = _detail(response) + if status == 503 and detail in BUSY_DETAILS: + # The renderer's own slot or memory floor: a busy instance, which the route answers `capacity`. + raise RenderBusy(f"the renderer is busy ({detail})") + suffix = f", {detail}" if detail else "" + raise _unavailable(f"the renderer answered the conversion with HTTP {status}{suffix}") + try: + answer = ConvertResponse.model_validate_json(response.content) + except ValidationError: + raise _unavailable("the renderer's conversion answer does not match the contract") from None + if answer.job_id != job.job_id: + raise _unavailable("the renderer answered for another job") + logger.info( + "remote conversion %s: exit %s, refusal %s, reason %s, %.2f s", + job.job_id, + answer.exit_code, + answer.refusal, + answer.reason, + answer.wall_s, + ) + if answer.reason == "launcher": + raise _unavailable("the renderer's sandbox launcher failed for the conversion") + return ConvertOutput( + exit_code=answer.exit_code, + timed_out=answer.timed_out, + refusal=answer.refusal, + measured=answer.measured, + limit=answer.limit, + text=answer.text, + meta=answer.meta, + reason=answer.reason, + wall_s=answer.wall_s, + ) + async def _post(self, body: dict[str, object], timeout: httpx.Timeout, *, path: str = "/render") -> httpx.Response: """POST the job, retrying connection errors and front-end 429/5xx with backoff inside the budget.""" delays = iter(self.retry_delays_s) diff --git a/agents/anyplot/render/backends/sandbox.py b/agents/anyplot/render/backends/sandbox.py index 95ab6c0cb2e..acb042a6787 100644 --- a/agents/anyplot/render/backends/sandbox.py +++ b/agents/anyplot/render/backends/sandbox.py @@ -14,7 +14,16 @@ `Services.backend` puts in front of every backend, hands it one theme at a time. """ -from ..contract import DataJob, DataOutput, DataRunUnsupported, RenderJob, RenderResult, RuntimeAdapter +from ..contract import ( + ConvertJob, + ConvertOutput, + DataJob, + DataOutput, + DataRunUnsupported, + RenderJob, + RenderResult, + RuntimeAdapter, +) SANDBOX_BINARY = "/usr/local/gcp/bin/sandbox" @@ -38,3 +47,7 @@ async def render(self, job: RenderJob) -> RenderResult: async def run_data(self, job: DataJob) -> DataOutput: """Data runs go through the renderer's `POST /run-data` too; this variant has none.""" raise DataRunUnsupported("the in-process sandbox backend runs no data snippets; use remote, local or fake") + + async def convert(self, job: ConvertJob) -> ConvertOutput: + """Conversions go through the renderer's `POST /convert`; this variant has none.""" + raise DataRunUnsupported("the in-process sandbox backend converts no files; use remote, local or fake") diff --git a/agents/anyplot/render/contract.py b/agents/anyplot/render/contract.py index c41cf688b90..068d67bf8c6 100644 --- a/agents/anyplot/render/contract.py +++ b/agents/anyplot/render/contract.py @@ -11,6 +11,11 @@ (`DataOutput`). A backend that cannot (a renderer deployed before `POST /run-data`) raises `DataRunUnsupported`. +A `ConvertJob` is a binary data file (Excel, OpenDocument, Parquet, Feather, Arrow, +NumPy) the page dropped; `convert` reads it in the same isolation with the harness in +convert mode and returns the CSV text (or JSON records) for the paste box +(`ConvertOutput`). A backend without conversions raises `DataRunUnsupported` too. + A `RuntimeAdapter` describes one language runtime: the file name the code is written to, the command (the CI command, with the probe harness in front for Python), the environment per theme, and the code transforms the pipeline applies before a render @@ -26,6 +31,8 @@ from pathlib import Path from typing import Any, Protocol +from agents.renderer.wire import ConvertMeta + from ..schemas import RENDER_ID_PATTERN, Theme @@ -141,8 +148,47 @@ def __post_init__(self) -> None: self.stderr_tail = self.stderr_tail[-MAX_STDERR_CHARS:] +MAX_CONVERT_TIMEOUT_S = 15.0 +"""A conversion's longest time limit; the renderer's `wire.MAX_CONVERT_TIMEOUT_S` is the same (pinned by a test).""" + + +@dataclass(frozen=True) +class ConvertJob: + """One binary data file to convert to text in isolation (the dataset route's `/convert`); `sheet` names the + sheet or `.npz` array the page chose, None for the first.""" + + job_id: str + data: bytes = field(repr=False) + sheet: str | None = None + timeout_s: float = 10.0 + + def __post_init__(self) -> None: + if not _RENDER_ID.fullmatch(self.job_id): + raise ValueError("job_id must match the render id pattern") + if not 0 < self.timeout_s <= MAX_CONVERT_TIMEOUT_S: + raise ValueError("timeout_s must be positive and at most the conversion limit") + + +@dataclass +class ConvertOutput: + """What a conversion left: the harness's exit code (`harness.CONVERT_EXIT`), the refusal it stands for with + the measured value and the cap, or the text and its `meta` after a 0. `reason` is the renderer's run reason + (`timeout`, `memory`, ...) when the host stopped the run.""" + + exit_code: int | None + timed_out: bool = False + refusal: str | None = None + measured: int | None = None + limit: int | None = None + text: str | None = field(default=None, repr=False) + meta: ConvertMeta | None = None + reason: str | None = None + wall_s: float = 0.0 + + class RenderBackend(Protocol): - """Runs a job's code once per theme in isolation and collects the raw outputs; runs data snippets the same way.""" + """Runs a job's code once per theme in isolation and collects the raw outputs; runs data snippets and file + conversions the same way.""" name: str @@ -150,6 +196,8 @@ async def render(self, job: RenderJob) -> RenderResult: ... async def run_data(self, job: DataJob) -> DataOutput: ... + async def convert(self, job: ConvertJob) -> ConvertOutput: ... + class RuntimeAdapter(Protocol): """One language runtime: how its code is written, run, prepared and collected.""" @@ -173,6 +221,14 @@ def data_env(self) -> dict[str, str]: """The complete environment of a data run: no theme, no plotting settings.""" ... + def convert_command(self) -> list[str]: + """The argv that converts the file `input.bin` in the work directory with the harness in convert mode.""" + ... + + def convert_env(self) -> dict[str, str]: + """The complete environment of a conversion.""" + ... + def normalise(self, code: str, library: str) -> str: """The canvas and catalogue-header normaliser of this runtime.""" ... diff --git a/agents/anyplot/render/convert.py b/agents/anyplot/render/convert.py new file mode 100644 index 00000000000..69b437bf6d4 --- /dev/null +++ b/agents/anyplot/render/convert.py @@ -0,0 +1,1095 @@ +"""File conversion for the renderer's convert mode: a binary data file becomes CSV text for the paste route. + +The harness (`harness.py --convert input.bin`) loads this module by its path, calls +`preload()` before it arms its audit-hook guard (every import and every `open` is refused +after that), and then `convert(data, sheet)` on the bytes it read, with the guard armed. +Like the harness it is a standalone module with no anyplot import, because it runs inside +the sandbox with the isolated interpreter; the renderer image copies it next to the +harness (`/opt/anyplot/convert.py`). The tests import it directly. + +The format comes from the first bytes, never from the file's name or type: + +* **Excel and OpenDocument** (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.ods`) through + calamine, which reads the values the file stored and nothing else: no formula is + evaluated, no external link followed, no macro or DDE run. A formula cell the file + saved without its value is empty, and the cells are counted (`formulas_without_value`, + xlsx and xlsm only: the other formats do not say); a sheet whose cells are all such + formulas is refused (`formulas_only`, owner decision of 2026-10-15). Before calamine + loads an xlsx sheet, one linear pass over its part (`scan_sheet_xml`) finds the span of + its cells and refuses a sheet past the caps, because calamine allocates the whole + bounding box (one cell at XFD1048576 is 549 GB). An encrypted workbook (an OLE + container with an `EncryptedPackage` stream) is refused with "remove the password". + The sheet goes out as its raw cell grid (`csv.writer`): dates ISO, numbers in their + shortest form, booleans `TRUE`/`FALSE`, empty for nothing, so the parser's + `table_block` finds a header below row 1 and keeps a title or a footnote as a note; + the rows of a table block keep their empty trailing fields (`pad_table_rows`). + Merged cells keep their value in the top-left cell only, which is how calamine reads + them. Hidden sheets are skipped and counted, very hidden ones are never named, chart + and macro sheets are skipped. Without a chosen sheet the first visible sheet with a + cell in it is read; the answer names the visible sheets with cells (at most + `MAX_SHEETS_LISTED`, whole, and only names that are printable and no path) so the page + can offer the others. +* **Parquet, Feather (v1 and v2) and Arrow IPC** (file and stream) through pyarrow. The + row count comes from the Parquet row groups (which must add up to the footer's total) + before any data page is read, and at most `MAX_ROWS` + 1 rows are ever read, batch by + batch, whatever a footer claims. Struct columns are flattened to dotted names (a + repeated name gets `_2` and is counted, `renamed_columns`), dictionary columns decoded. + Data keeps the precision it came with (owner rule of 2026-10-15): timestamps and times + carry every stored digit (nanoseconds included) and a zone-aware timestamp its offset in + its own zone (UTC with `Z`, counted as `utc_timestamps`, only where the zone cannot be + read); decimals and 64-bit integers are written exactly, float32 in its shortest text. + Map, binary, union, duration and interval columns are dropped, counted and named + (`dropped_columns`, `Conversion.dropped`). A table with a list column goes out as JSON records, one + per line, instead of CSV: the paste route's parser expands list fields into long format + (one row per value, a `record_id` column, thinning to its row cap), the owner's list + rule of 2026-10-15 (#12191), so there is one implementation of it (`list_columns`). +* **NumPy** `.npy` and `.npz` with `allow_pickle=False`: an object array (a pickle + inside) is refused, and so is any array that is neither 2-D nor a structured 1-D + array. The header's shape is checked against the bytes that are there before an + array is allocated. An `.npz` is a zip of `.npy` members, each a "sheet". +* **Never**: pickle and joblib (loading one runs code) and HDF5; anything else is + `unsupported`. + +Every zip (xlsx, xlsm, xlsb, ods, npz) passes `check_archive` first: at most +`MAX_ZIP_ENTRIES` entries, `MAX_ZIP_UNCOMPRESSED` bytes declared uncompressed in all, at +most `MAX_ZIP_RATIO` times the archive's own size, no encrypted member, and every member +inflated (bounded by its declared size) to prove that size, because calamine's reader +trusts none. Every pattern over file content is linear. A reader that runs out of the +sandbox's address space is `too_complex`. + +The caps are the computed-data caps of the harness's data mode for rows and columns +(`MAX_ROWS`, `MAX_COLUMNS`), with `ROW_SLACK` more raw rows for a spreadsheet's titles +and notes, and the paste cap (`MAX_CSV_BYTES`, 1 MiB) for the text: the text goes into +the paste box, and a paste over it is refused. A file over a cap is refused with the +measured value and the cap, never cut. +""" + +import csv +import datetime +import decimal +import io +import math +import re +import struct +import zipfile +import zlib + + +MAX_INPUT_BYTES = 5 * 1024 * 1024 +"""The largest file converted: the BFF's and the agents service's raw cap (base64 in JSON, about 6.7 MB).""" +MAX_ROWS = 20_000 # agents/anyplot/schemas.py MAX_ROWS (the harness's DATA_MAX_ROWS) +MAX_COLUMNS = 50 # agents/anyplot/schemas.py MAX_COLUMNS +ROW_SLACK = 200 +"""Raw spreadsheet rows past `MAX_ROWS`: a title, notes and a header above the table, totals and footnotes below.""" +MAX_GRID_ROWS = MAX_ROWS + ROW_SLACK +MAX_CSV_BYTES = 1024 * 1024 # agents/anyplot/data/parse.py MAX_INPUT_BYTES: the text goes through the paste route +MAX_ZIP_ENTRIES = 2_000 +MAX_ZIP_UNCOMPRESSED = 100 * 1024 * 1024 +MAX_ZIP_RATIO = 250 +"""Declared uncompressed bytes per byte of archive: an xlsx of repeated rows packs about 20 to 1, a zip bomb 1,000 to 1.""" +MAX_SHEETS_LISTED = 10 +MAX_SHEET_NAME_CHARS = 255 # wire.MAX_SHEET_NAME_CHARS: a listed name comes back as the choice, whole +NPY_MAX_HEADER = 10_000 # numpy's own `_MAX_HEADER_SIZE` + +REFUSALS = ( + "unsupported", + "encrypted", + "pickle", + "hdf5", + "corrupt", + "archive_limits", + "too_many_rows", + "too_many_columns", + "too_large", + "empty", + "sheet_not_found", + "array_shape", + "object_array", + "formulas_only", + "too_complex", +) +"""Why a file is not converted. The harness exits with `REFUSAL_EXIT_BASE` plus the index (`harness.CONVERT_EXIT`), +and the renderer maps the code back (`wire.CONVERT_REASONS`); a test pins the three together.""" +REFUSAL_EXIT_BASE = 10 + +OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +ZIP_MAGICS = (b"PK\x03\x04", b"PK\x05\x06") +ODS_MIMETYPE = b"application/vnd.oasis.opendocument.spreadsheet" +SPREADSHEET_KINDS = ("xlsx", "xlsm", "xlsb", "xls", "ods") +_CONTROLS = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]") + + +class Refused(Exception): + """A file this module does not convert: `reason` is one of `REFUSALS`, with the measured value and the cap.""" + + def __init__(self, reason, measured=None, limit=None): + super().__init__(reason) + self.reason = reason + self.measured = measured + self.limit = limit + + +class Conversion: + """The CSV text of one table and what the page shows next to it.""" + + def __init__(self, csv_text, kind, rows, columns, sheet=None, sheets=(), sheet_count=0, warnings=None): + self.csv_text = csv_text + self.kind = kind + self.rows = rows + self.columns = columns + self.sheet = sheet + self.sheets = list(sheets) + self.sheet_count = sheet_count + self.warnings = dict(warnings or {}) + self.dropped = [name if len(name) <= 100 else name[:99] + "…" for name in self.warnings.pop("_dropped", [])][ + :MAX_DROPPED_NAMED + ] + self.text_format = "csv" + + def meta(self): + """`meta.json` for the host (`wire.ConvertMeta`).""" + return { + "kind": self.kind, + "format": self.text_format, + "rows": self.rows, + "columns": self.columns, + "sheet": self.sheet, + "sheets": self.sheets, + "sheet_count": self.sheet_count, + "warnings": {key: value for key, value in self.warnings.items() if value}, + "dropped": self.dropped, + } + + +def preload(): + """Import everything a conversion needs, before the harness's guard refuses every import and `open`. + + pyarrow and numpy import some of their modules on first use (`pyarrow.compute` for a + cast, `numpy.lib.format` for a load); a module missing here (a machine without the + `convert` extra) is skipped, and a conversion that needs it fails as `error`. + """ + import importlib + + for name in ( + "numpy", + "numpy.lib.format", + "pyarrow", + "pyarrow.compute", + "pyarrow.ipc", + "pyarrow.feather", + "pyarrow.parquet", + "python_calamine", + "ast", + "html", + ): + try: + importlib.import_module(name) + except ImportError: + continue + # `zipfile` decodes member names as cp437, which the codec registry imports on first use; a round trip through + # an in-memory zip loads that and the deflate path once. + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("warm.npy", b"warm") + with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive: + archive.read("warm.npy") + "warm".encode("cp437").decode("cp437") + # The OLE sniffer matches stream names in UTF-16LE: its codec loads on first use as well (found by the image + # smoke, where the guard refused that import and an encrypted workbook came back `blocked`). + "warm".encode("utf-16-le").decode("utf-16-le") + + +# --- Sniffing ------------------------------------------------------------------------------- + + +def sniff(data): + """The format the first bytes name: one of the converted kinds, or a `Refused` for anything else.""" + head = bytes(data[:16]) + if head.startswith(ZIP_MAGICS): + return _zip_kind(data) + if head.startswith(OLE_MAGIC): + return _ole_kind(data) + if head.startswith(b"PAR1"): + return "parquet" + if head.startswith((b"ARROW1", b"FEA1")): + return "feather" + if head.startswith(b"\xff\xff\xff\xff"): + return "arrow" + if head.startswith(b"\x93NUMPY"): + return "npy" + if head.startswith(b"\x89HDF\r\n\x1a\n"): + raise Refused("hdf5") + if len(head) >= 2 and head[0] == 0x80 and 2 <= head[1] <= 5: + raise Refused("pickle") + raise Refused("unsupported") + + +def _open_zip(data): + try: + return zipfile.ZipFile(io.BytesIO(data)) + except (zipfile.BadZipFile, ValueError, OSError, EOFError): + raise Refused("corrupt") from None + + +def check_archive(archive, data): + """Refuse a zip whose central directory announces too much (entries, uncompressed bytes, ratio, encryption), + then inflate every member to prove its declared size (`_inflated_size`). + + calamine's own zip reader trusts no declared size: a 1.79 MB xlsx that declared 1 KB for a 322 MB part + passed the directory check (security review of #12194). Each member is inflated with a bound of its declared + size plus one byte, so the whole check inflates at most `MAX_ZIP_UNCOMPRESSED`; a member that inflates to + another size than it declares is `corrupt`. + """ + infos = archive.infolist() + size = len(data) + if len(infos) > MAX_ZIP_ENTRIES: + raise Refused("archive_limits", len(infos), MAX_ZIP_ENTRIES) + if any(info.flag_bits & 0x1 for info in infos): + raise Refused("encrypted") + total = sum(info.file_size for info in infos) + if total > MAX_ZIP_UNCOMPRESSED: + raise Refused("archive_limits", total, MAX_ZIP_UNCOMPRESSED) + if total > MAX_ZIP_RATIO * max(size, 1): + raise Refused("archive_limits", total // max(size, 1), MAX_ZIP_RATIO) + for info in infos: + if _inflated_size(data, info) != info.file_size: + raise Refused("corrupt") + + +INFLATE_CHUNK = 1024 * 1024 + + +def _inflated_size(data, info): + """The bytes a member's stored data inflates to, counted up to its declared size plus one, linearly. + + Read from the member's local header in `data` (the name and extra lengths there, then `compress_size` bytes), + never through `zipfile`, which stops at the declared size. A method other than stored or deflated is refused. + """ + offset = info.header_offset + header = data[offset : offset + 30] + if len(header) < 30 or header[:4] != b"PK\x03\x04": + raise Refused("corrupt") + name_length, extra_length = struct.unpack(" MAX_CSV_BYTES: + raise Refused("too_large", size, MAX_CSV_BYTES) + return text + + +def _table(names, columns, kind, warnings, sheet=None, sheets=(), sheet_count=0): + """Named columns of cell texts, all the same length, as a `Conversion` with a header row.""" + if not names: + raise Refused("empty") + if len(names) > MAX_COLUMNS: + raise Refused("too_many_columns", len(names), MAX_COLUMNS) + count = len(columns[0]) if columns else 0 + if count > MAX_ROWS: + raise Refused("too_many_rows", count, MAX_ROWS) + rows = [[cell_text(name) for name in names]] + rows.extend([column[index] for column in columns] for index in range(count)) + return Conversion(grid_csv(rows), kind, count, len(names), sheet, sheets, sheet_count, warnings) + + +# --- Spreadsheets --------------------------------------------------------------------------- + + +def read_grid(sheet): + """The sheet's cells row by row: trailing empty cells and rows trimmed, the caps checked while it is read. + + Only rows up to the last one with a cell count, so formatted empty rows below a table + do not; a cell past `MAX_GRID_ROWS` rows or `MAX_COLUMNS` columns refuses the sheet, + whatever dimension the file declares. + """ + rows = [] + blank = 0 + width = 0 + for index, row in enumerate(sheet.iter_rows()): + cells = [cell_text(value) for value in row] + while cells and cells[-1] == "": + cells.pop() + if not cells: + blank += 1 + continue + if index >= MAX_GRID_ROWS: + raise Refused("too_many_rows", max(index + 1, sheet.height), MAX_ROWS) + if len(cells) > MAX_COLUMNS: + raise Refused("too_many_columns", max(len(cells), sheet.width), MAX_COLUMNS) + if rows: + rows.extend([] for _ in range(blank)) + blank = 0 + width = max(width, len(cells)) + rows.append(cells) + return pad_table_rows(rows), width + + +def pad_table_rows(rows): + """Give the rows of each table block their missing trailing fields back. + + Trimming trailing empty cells keeps a title or a footnote a one-field line, which the + parser's `table_block` leaves out as a note; a data row whose last cells are empty + (`C,` in a `category,value` table) must keep its fields, or it reads as a note too + (Copilot review of #12194). A block starts at a row with two or more cells and runs + over the rows that follow it without a blank row between; its rows are padded to its + widest row. A one-field row right under a table without a blank line therefore counts + as a data row; one after a blank line stays a note. + """ + padded = [] + block = [] + + def flush(): + if block: + size = max(len(row) for row in block) + padded.extend(row + [""] * (size - len(row)) for row in block) + block.clear() + + for row in rows: + if not row: + flush() + padded.append(row) + elif block or len(row) >= 2: + block.append(row) + else: + padded.append(row) + flush() + return padded + + +def _visible_sheets(workbook, warnings): + from python_calamine import SheetTypeEnum, SheetVisibleEnum + + visible = [] + hidden = 0 + for meta in workbook.sheets_metadata: + if meta.typ != SheetTypeEnum.WorkSheet: + continue + if meta.visible == SheetVisibleEnum.Visible: + visible.append(meta.name) + elif meta.visible == SheetVisibleEnum.Hidden: + hidden += 1 + if hidden: + warnings["hidden_sheets"] = hidden + return visible + + +def read_workbook(data, kind, sheet_name, warnings): + """The chosen sheet, or the first visible one with a cell, as a raw-grid `Conversion`.""" + from python_calamine import CalamineWorkbook, PasswordError + + try: + workbook = CalamineWorkbook.from_filelike(io.BytesIO(data)) + except PasswordError: + raise Refused("encrypted") from None + except Exception: # calamine's errors (zip, XML, a broken record) and anything its reader raises + raise Refused("corrupt") from None + with workbook: + visible = _visible_sheets(workbook, warnings) + if sheet_name is not None and sheet_name not in visible: + raise Refused("sheet_not_found") + scans = {} + + def scan(name): + """The xlsx sheet part's linear scan (`scan_sheet_xml`), or None for the other formats.""" + if kind not in ("xlsx", "xlsm"): + return None + if name not in scans: + scans[name] = sheet_scan(data, name) + return scans[name] + + def load(name): + found = scan(name) + if found is not None: + found.check_caps() # before calamine allocates the sheet's bounding box + try: + sheet = workbook.get_sheet_by_name(name) + except MemoryError: + raise Refused("too_complex") from None + except Exception: + raise Refused("corrupt") from None + rows, width = read_grid(sheet) + return rows, width, found.unsaved if found is not None else 0 + + chosen = None + for name in [sheet_name] if sheet_name is not None else visible: + rows, width, unsaved = load(name) + if rows or unsaved: + chosen = name + break + if chosen is None: + raise Refused("empty") + if not rows: + # Owner decision of 2026-10-15: a sheet whose cells are all formulas without saved values is refused. + raise Refused("formulas_only", unsaved) + if unsaved: + warnings["formulas_without_value"] = unsaved + + def has_cells(name): + if name == chosen: + return True + found = scan(name) + if found is not None: + return found.cells > 0 + try: + return bool(load(name)[0]) + except Refused: + return True # it has cells, too many of them: offered, and refused when chosen + + # Owner decision of 2026-10-15: the picker offers no empty sheet. The first `MAX_LISTING_CHECKS` are looked + # at; the rest count as sheets with cells. + checked = [name for name in visible[:MAX_LISTING_CHECKS] if has_cells(name)] + count = len(checked) + max(len(visible) - MAX_LISTING_CHECKS, 0) + return Conversion(grid_csv(rows), kind, len(rows), width, _named(chosen), _listed(checked), count, warnings) + + +def _listable(name): + """A name the page may offer and send back whole: at most `MAX_SHEET_NAME_CHARS`, printable, no path parts.""" + return ( + 0 < len(name) <= MAX_SHEET_NAME_CHARS + and not _UNSAFE_NAME.search(name) + and "/" not in name + and "\\" not in name + and name not in (".", "..") + ) + + +def _listed(names): + """The sheet names the page may offer, whole: each is sent back as the choice, so none is shortened. A name + that is not `_listable` (too long for the wire, a control or bidi character, a path) is left out and counted.""" + return [name for name in names if _listable(name)][:MAX_SHEETS_LISTED] + + +def _named(name): + """The name of the sheet read, or None for one the page may not show or send back.""" + return name if _listable(name) else None + + +# Every pattern over file content is linear: a tag's attributes are `[^<>]*`, which stops at the next `<`, so a scan +# never runs past it, and there is no lazy `.*?` across cells. A 30 MB run of `` (security review of +# #12194: the earlier `(.*?)` pattern ran to the 10 s kill on it) costs one pass. +_SHEET_TAG = re.compile(rb"<(?:\w+:)?sheet\b([^<>]*)>") +_RELATIONSHIP_TAG = re.compile(rb"<(?:\w+:)?Relationship\b([^<>]*)>") +_ATTRIBUTE = re.compile(rb"([\w:]+)\s*=\s*(\"[^\"<]*\"|'[^'<]*')") +_SHEET_TOKEN = re.compile(rb"<(/?)([A-Za-z_][\w.:-]{0,40}+)([^<>]*+)>") +"""A tag: whether it closes, its qualified name, its attributes. Possessive, so a run of `` costs one +step per `<` (30 MB in about 0.4 s); `scan_sheet_xml` keeps the tags it reads by their local name.""" +_SHEET_TAGS = frozenset({b"row", b"c", b"f", b"v", b"is"}) +_REF = re.compile(rb"""\br\s*=\s*["']([A-Za-z]{0,3})([0-9]{0,7})["']""") +_UNSAFE_NAME = re.compile(r"[\x00-\x1f\x7f-\x9f​-‏‪-‮⁦-⁩]") +MAX_LISTING_CHECKS = 50 +"""Visible sheets looked at for cells before the picker lists them; the rest count as sheets with cells.""" + + +class SheetScan: + """What one pass over an xlsx sheet part found: the span of the cells with content, and the formula cells the + file saved without a value (calamine reads those as empty).""" + + def __init__(self): + self.cells = 0 + self.unsaved = 0 + self.first_row = self.last_row = self.first_col = self.last_col = None + + def place(self, row, col): + self.cells += 1 + self.first_row = row if self.first_row is None else min(self.first_row, row) + self.last_row = row if self.last_row is None else max(self.last_row, row) + self.first_col = col if self.first_col is None else min(self.first_col, col) + self.last_col = col if self.last_col is None else max(self.last_col, col) + self.check_caps() # past a cap the scan stops: the rest of a huge sheet is never walked + + def check_caps(self): + """Refuse a sheet whose cells span more rows or columns than the caps, before calamine allocates its + bounding box: one cell at XFD1048576 made it allocate 549 GB and abort (security review of #12194).""" + if self.first_row is None: + return + rows = self.last_row - self.first_row + 1 + columns = self.last_col - self.first_col + 1 + if rows > MAX_GRID_ROWS: + raise Refused("too_many_rows", rows, MAX_ROWS) + if columns > MAX_COLUMNS: + raise Refused("too_many_columns", columns, MAX_COLUMNS) + + +def _column_index(letters): + index = 0 + for letter in letters.upper(): + index = index * 26 + (letter - 64) + return index + + +def scan_sheet_xml(xml): + """One linear pass over a sheet part's tags (`_SHEET_TOKEN`): rows and cells by their `r` (or one past the + previous one when it is missing), a cell with a non-empty ``, an `` or an `` counts as content, and a + cell with an `` and no non-empty `` as a formula saved without its value.""" + found = SheetScan() + row = 0 + col = 0 + in_cell = False + formula = value = content = False + for match in _SHEET_TOKEN.finditer(xml): + closing, name, attributes = match.group(1), match.group(2).rpartition(b":")[2], match.group(3) + if name not in _SHEET_TAGS: + continue + if name == b"row": + if not closing: + ref = _REF.search(attributes) + row = int(ref.group(2)) if ref and ref.group(2) else row + 1 + col = 0 + continue + if name == b"c": + if closing: + if in_cell: + if content: + found.place(row, col) + if formula and not value: + found.unsaved += 1 + in_cell = False + continue + ref = _REF.search(attributes) + if ref and ref.group(1): + col = _column_index(ref.group(1)) + if ref.group(2): + row = int(ref.group(2)) + else: + col += 1 + in_cell = not attributes.rstrip().endswith(b"/") + formula = value = content = False + continue + if not in_cell or closing: + continue + if name == b"f": + formula = content = True + elif name == b"is": + content = True + elif name == b"v" and not attributes.rstrip().endswith(b"/") and xml[match.end() : match.end() + 1] != b"<": + value = content = True + return found + + +def _attributes(raw): + import html + + return { + key.decode("ascii", "replace"): html.unescape(value[1:-1].decode("utf-8", "replace")) + for key, value in _ATTRIBUTE.findall(raw) + } + + +def sheet_scan(data, sheet_name): + """`scan_sheet_xml` of an xlsx sheet, found from the workbook's own parts with plain patterns, no XML parser: + the sheet's relationship id in `xl/workbook.xml`, its part in `xl/_rels/workbook.xml.rels`. None when a part + is missing (calamine then reads the sheet without the pre-check).""" + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + workbook = archive.read("xl/workbook.xml") + relations = archive.read("xl/_rels/workbook.xml.rels") + target = None + for raw in _SHEET_TAG.findall(workbook): + attributes = _attributes(raw) + if attributes.get("name") == sheet_name: + target = next((value for key, value in attributes.items() if key.endswith(":id")), None) + break + if target is None: + return None + part = None + for raw in _RELATIONSHIP_TAG.findall(relations): + attributes = _attributes(raw) + if attributes.get("Id") == target: + part = attributes.get("Target") + break + if not part: + return None + path = part.lstrip("/") if part.startswith("/") else "xl/" + part + xml = archive.read(path) # its size was checked against the archive caps by inflating it + except (KeyError, zipfile.BadZipFile, OSError, ValueError, EOFError): + return None + return scan_sheet_xml(xml) + + +# --- Parquet, Feather, Arrow ---------------------------------------------------------------- + + +def read_arrow(data, kind, warnings): + """A Parquet, Feather or Arrow IPC table, at most `MAX_ROWS` + 1 rows read, as a `Conversion`.""" + import pyarrow as pa + + try: + table = _arrow_table(pa, data, kind) + names, columns, lists = _arrow_columns(pa, table, warnings) + except Refused: + raise + except MemoryError: # pyarrow's ArrowMemoryError among them: a decompression bomb past the address-space limit + raise Refused("too_complex") from None + except Exception: # pyarrow's ArrowInvalid, OSError and kin for a broken or hostile file + raise Refused("corrupt") from None + if lists: + return _records(names, columns, kind, warnings) + return _table(names, columns, kind, warnings) + + +def json_value(value): + """One value for a JSON record: numbers, text, booleans and null as they are, lists and objects element by + element, everything else (dates, times, decimals) as its cell text.""" + if value is None or isinstance(value, bool | int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, str): + return _CONTROLS.sub(" ", value) + if isinstance(value, list | tuple): + return [json_value(item) for item in value] + if isinstance(value, dict): + return {str(key): json_value(item) for key, item in value.items()} + return cell_text(value) + + +def _records(names, columns, kind, warnings): + """A table with list columns as JSON records, one per line: the paste route's parser expands list fields into + long format (one row per value, `record_id`, thinning to its row cap: `data/parse.py` `_records`), the owner's + list rule of 2026-10-15, so the conversion does not expand them a second way.""" + import json + + if len(names) > MAX_COLUMNS: + raise Refused("too_many_columns", len(names), MAX_COLUMNS) + count = len(columns[0]) if columns else 0 + lines = [ + json.dumps({name: column[index] for name, column in zip(names, columns, strict=True)}, ensure_ascii=False) + for index in range(count) + ] + text = "[\n" + ",\n".join(lines) + "\n]\n" + size = len(text.encode("utf-8", errors="surrogatepass")) + if size > MAX_CSV_BYTES: + raise Refused("too_large", size, MAX_CSV_BYTES) + conversion = Conversion(text, kind, count, len(names), warnings=warnings) + conversion.text_format = "json" + return conversion + + +def _arrow_table(pa, data, kind): + """Every read is single-threaded (`use_threads=False`): the sandbox's process limit counts threads, and a + decompression pool that cannot start its workers fails the read.""" + batches = [] + count = 0 + + def take(batch): + nonlocal count + count += batch.num_rows + if count > MAX_ROWS: + raise Refused("too_many_rows", count, MAX_ROWS) + batches.append(batch) + + source = pa.BufferReader(pa.py_buffer(data)) + if kind == "parquet": + import pyarrow.parquet as pq + + parquet = pq.ParquetFile(source) + metadata = parquet.metadata + # The reader goes by row groups, so their counts are checked, and a footer total that disagrees with them + # is a forged footer (security review of #12194). + grouped = sum(metadata.row_group(index).num_rows for index in range(metadata.num_row_groups)) + if grouped != metadata.num_rows: + raise Refused("corrupt") + if grouped > MAX_ROWS: + raise Refused("too_many_rows", grouped, MAX_ROWS) + schema = parquet.schema_arrow + _leaf_count(pa, schema) + for batch in parquet.iter_batches(batch_size=4096, use_threads=False): + take(batch) + elif data[:4] == b"FEA1": + import pyarrow.feather as feather + + table = feather.read_table(source, use_threads=False) + if table.num_rows > MAX_ROWS: + raise Refused("too_many_rows", table.num_rows, MAX_ROWS) + return table + elif kind == "feather": + reader = pa.ipc.open_file(source, options=pa.ipc.IpcReadOptions(use_threads=False)) + schema = reader.schema + _leaf_count(pa, schema) + for index in range(reader.num_record_batches): + take(reader.get_batch(index)) + else: + stream = pa.ipc.open_stream(source, options=pa.ipc.IpcReadOptions(use_threads=False)) + schema = stream.schema + _leaf_count(pa, schema) + for batch in stream: + take(batch) + return pa.Table.from_batches(batches, schema=schema) + + +def _leaf_count(pa, schema): + """Refuse a schema with more than `MAX_COLUMNS` columns once its structs are flattened, before any row is read.""" + count = 0 + pending = [field.type for field in schema] + while pending: + kind = pending.pop() + if pa.types.is_struct(kind): + pending.extend(kind.field(index).type for index in range(kind.num_fields)) + else: + count += 1 + if count > 10 * MAX_COLUMNS: + break + if count > MAX_COLUMNS: + raise Refused("too_many_columns", count, MAX_COLUMNS) + + +def _arrow_columns(pa, table, warnings): + """The table's columns: structs flattened, unsupported types dropped and counted. + + Scalar columns become cell texts. List columns (`list`, `large_list`, `fixed_size_list`) are + kept as lists of JSON values, and the third value says there is one: the table then goes + out as JSON records (`_records`), whose list fields the parser expands. + """ + import pyarrow.compute as pc + + while any(pa.types.is_struct(field.type) for field in table.schema): + table = table.flatten() + names = [] + columns = [] + lists = False + list_types = (pa.types.is_list, pa.types.is_large_list, pa.types.is_fixed_size_list) + lists_ahead = any( + any(check(field.type.value_type if pa.types.is_dictionary(field.type) else field.type) for check in list_types) + for field in table.schema + ) + for name, column in zip(table.column_names, table.columns, strict=True): + kind = column.type + if pa.types.is_dictionary(kind): + column = pa.chunked_array([chunk.dictionary_decode() for chunk in column.chunks], type=kind.value_type) + kind = column.type + if pa.types.is_list(kind) or pa.types.is_large_list(kind) or pa.types.is_fixed_size_list(kind): + lists = True + warnings["list_columns"] = warnings.get("list_columns", 0) + 1 + names.append(str(name)) + columns.append([json_value(value) for value in column.to_pylist()]) + continue + if ( + pa.types.is_map(kind) + or pa.types.is_binary(kind) + or pa.types.is_large_binary(kind) + or pa.types.is_fixed_size_binary(kind) + or pa.types.is_union(kind) + or pa.types.is_dictionary(kind) + or pa.types.is_duration(kind) + or pa.types.is_interval(kind) + ): + # A duration or an interval has no form the parser reads as one value; named, never written as repr text. + dropped(warnings, str(name)) + continue + if pa.types.is_timestamp(kind) or pa.types.is_time(kind): + texts = _exact_times(pa, pc, column, kind, warnings) + names.append(str(name)) + columns.append(texts) + continue + if pa.types.is_floating(kind) and kind.bit_width < 64: + # The shortest text that reads back as the same float32 or float16 (`0.1`, not its float64 digits). + import numpy as np + + narrow = np.float16 if kind.bit_width == 16 else np.float32 + texts = ["" if value is None else _float_text(narrow(value)) for value in column.to_pylist()] + names.append(str(name)) + columns.append(texts) + continue + values = column.to_pylist() + texts = [cell_text(value) for value in values] + names.append(str(name)) + # A JSON record keeps a number a number and a missing value null; anything else is its cell text. + columns.append( + [ + json_value(value) if value is None or isinstance(value, bool | int | float) else text + for value, text in zip(values, texts, strict=True) + ] + if lists_ahead + else texts + ) + return distinct_names(names, warnings), columns, lists + + +MAX_DROPPED_NAMED = 20 +"""Dropped columns named in the answer (`meta.dropped`); more are counted only.""" +_TRAILING_FRACTION = re.compile(r"(\.\d*?)0+(?=$|[Z+-])") + + +def dropped(warnings, name): + """Count a column left out (`dropped_columns`) and name it for the page (owner rule of 2026-10-15: a step that + changes the data says so). The names travel in `warnings["_dropped"]`, which `Conversion` moves to `dropped`.""" + warnings["dropped_columns"] = warnings.get("dropped_columns", 0) + 1 + warnings.setdefault("_dropped", []).append(name) + + +def _exact_times(pa, pc, column, kind, warnings): + """Timestamps and times as ISO text with every digit the file stored (nanoseconds included) and, for a + zone-aware timestamp, the offset in its own zone (`+0100`): owner rule of 2026-10-15, data keeps its precision + and its time zone as written. Trailing zeros of a fraction go, which changes no value. + + pyarrow formats the zone from the system's time-zone database; where that zone cannot be read, the times are + written in UTC (`Z`) and counted (`utc_timestamps`), so the change is said, never silent.""" + try: + texts = pc.cast(column, pa.string()).to_pylist() + except Exception: + if not (pa.types.is_timestamp(kind) and kind.tz is not None): + raise + warnings["utc_timestamps"] = warnings.get("utc_timestamps", 0) + 1 + texts = pc.cast(pc.cast(column, pa.timestamp(kind.unit, "UTC")), pa.string()).to_pylist() + return ["" if text is None else _TRAILING_FRACTION.sub(lambda m: m.group(1).rstrip("."), text) for text in texts] + + +def _float_text(value): + """A float32 or float16 in the shortest text that reads back as the same value; empty for NaN and infinity.""" + import numpy as np + + return str(value) if np.isfinite(value) else "" + + +def distinct_names(names, warnings): + """The column names with every repeat renamed `_2`, `_3` and counted (`renamed_columns`). + + A flattened struct can meet a column of the same dotted name (`point.x` beside a struct + `point` with a field `x`); a JSON record would keep only one of them (Copilot review of #12194). + """ + taken = set() + result = [] + renamed = 0 + for name in names: + candidate = name + suffix = 2 + while candidate in taken: + candidate = f"{name}_{suffix}" + suffix += 1 + if candidate != name: + renamed += 1 + taken.add(candidate) + result.append(candidate) + if renamed: + warnings["renamed_columns"] = warnings.get("renamed_columns", 0) + renamed + return result + + +# --- NumPy ---------------------------------------------------------------------------------- + + +def _npy_array(raw): + """One `.npy` image as an array, its header checked before the data is allocated.""" + import numpy as np + + format_ = np.lib.format + handle = io.BytesIO(raw) + try: + version = format_.read_magic(handle) + if version == (1, 0): + shape, fortran, dtype = format_.read_array_header_1_0(handle, max_header_size=NPY_MAX_HEADER) + else: + shape, fortran, dtype = format_.read_array_header_2_0(handle, max_header_size=NPY_MAX_HEADER) + except MemoryError: + raise Refused("too_complex") from None + except Exception: # ValueError, SyntaxError, tokenize's TokenError of a broken header literal, and kin + raise Refused("corrupt") from None + if dtype.hasobject: + raise Refused("object_array") + count = math.prod(shape) if shape else 1 + if count * max(dtype.itemsize, 1) > len(raw): + raise Refused("corrupt") + handle.seek(0) + try: + return format_.read_array(handle, allow_pickle=False, max_header_size=NPY_MAX_HEADER) + except MemoryError: + raise Refused("too_complex") from None + except Exception: + raise Refused("corrupt") from None + + +def _array_shape_ok(array): + return array.ndim == 2 and array.dtype.names is None or array.ndim == 1 and array.dtype.names is not None + + +def _array_columns(array, warnings): + """A 2-D array's columns (`column_1`, ...) or a structured array's fields, as cell texts.""" + import numpy as np + + if array.dtype.names is None: + if array.shape[0] > MAX_ROWS: + raise Refused("too_many_rows", array.shape[0], MAX_ROWS) + if array.shape[1] > MAX_COLUMNS: + raise Refused("too_many_columns", array.shape[1], MAX_COLUMNS) + fields = [(f"column_{index + 1}", array[:, index]) for index in range(array.shape[1])] + else: + if array.shape[0] > MAX_ROWS: + raise Refused("too_many_rows", array.shape[0], MAX_ROWS) + fields = [] + for name in array.dtype.names: + field = array[name] + if field.ndim != 1 or field.dtype.names is not None or field.dtype.kind == "m": + dropped(warnings, name) # a sub-array, a nested record or a duration: no single cell holds it + continue + fields.append((name, field)) + names = [] + columns = [] + for name, values in fields: + if values.dtype.kind == "M": + texts = ["" if np.isnat(value) else str(np.datetime_as_string(value)) for value in values] + elif values.dtype.kind == "m": + texts = ["" if np.isnat(value) else str(value) for value in values] # a 2-D array of durations + elif values.dtype.kind == "f" and values.dtype.itemsize < 8: + texts = [_float_text(value) for value in values] + else: + texts = [cell_text(value) for value in values.tolist()] + names.append(name) + columns.append(texts) + return names, columns + + +def read_numpy(data, kind, sheet_name, warnings): + """An `.npy` array, or one array of an `.npz` (the chosen one, else the first 2-D or structured one).""" + if kind == "npy": + array = _npy_array(data) + if not _array_shape_ok(array): + raise Refused("array_shape", array.ndim, 2) + names, columns = _array_columns(array, warnings) + return _table(names, columns, kind, warnings) + with _open_zip(data) as archive: + members = [info.filename for info in archive.infolist()] + labels = [member[: -len(".npy")] for member in members] + if sheet_name is not None: + if sheet_name not in labels: + raise Refused("sheet_not_found") + candidates = [sheet_name] + else: + candidates = labels + refused = None + for label in candidates: + try: + raw = archive.read(label + ".npy") + except (zipfile.BadZipFile, OSError, ValueError, EOFError, KeyError): + raise Refused("corrupt") from None + try: + array = _npy_array(raw) + except Refused as exc: + if exc.reason != "object_array" or sheet_name is not None: + raise + refused = exc # never read: the first array that is no pickle is + continue + if not _array_shape_ok(array): + refused = refused or Refused("array_shape", array.ndim, 2) + continue + names, columns = _array_columns(array, warnings) + return _table(names, columns, kind, warnings, _named(label), _listed(labels), len(labels)) + raise refused or Refused("empty") + + +# --- Entry ---------------------------------------------------------------------------------- + + +def convert(data, sheet=None): + """The CSV text of the file in `data` (bytes), or `Refused`; `sheet` names a sheet or an `.npz` array.""" + data = bytes(data) + if len(data) > MAX_INPUT_BYTES: + raise Refused("too_large", len(data), MAX_INPUT_BYTES) + if not data: + raise Refused("empty") + try: + kind = sniff(data) + warnings = {} + if kind in SPREADSHEET_KINDS: + return read_workbook(data, kind, sheet, warnings) + if kind in ("parquet", "feather", "arrow"): + return read_arrow(data, kind, warnings) + return read_numpy(data, kind, sheet, warnings) + except MemoryError: # past the sandbox's address-space limit anywhere in a reader + raise Refused("too_complex") from None diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 0428d2df7c5..595df657aa1 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -218,6 +218,48 @@ """Exit codes of data mode: `error` is an exception in the snippet (its traceback on stderr), `no_frame` no `df` or one that is no DataFrame, `frame_shape` MultiIndex columns or an index that cannot become columns, `blocked` an operation `DataGuard` refused.""" +CONVERT_FLAG = "--convert" +CONVERT_MODULE_FILE = "convert.py" +"""The converter (`agents/anyplot/render/convert.py`), next to the harness in the image and in the checkout.""" +CONVERT_REQUEST_FILE = "request.json" +"""`{"sheet": }`: the sheet (or `.npz` array) the page chose; missing means the first one.""" +CONVERT_META_FILE = "meta.json" +"""What the conversion found (`convert.Conversion.meta`): the kind, the sheet and the others, the warning counts.""" +CONVERT_MAX_INPUT_BYTES = 5 * 1024 * 1024 # convert.MAX_INPUT_BYTES +CONVERT_MAX_REQUEST_BYTES = 4096 +CONVERT_REFUSALS = ( + "unsupported", + "encrypted", + "pickle", + "hdf5", + "corrupt", + "archive_limits", + "too_many_rows", + "too_many_columns", + "too_large", + "empty", + "sheet_not_found", + "array_shape", + "object_array", + "formulas_only", + "too_complex", +) +"""`convert.REFUSALS`, in its order (pinned by a test): the harness exits with 10 plus the index.""" +CONVERT_EXIT = { + "ok": 0, + "error": 1, + "usage": 2, + "blocked": 8, + **{reason: 10 + index for index, reason in enumerate(CONVERT_REFUSALS)}, +} +"""Exit codes of convert mode: `error` an exception in the reader (its class on stderr), `blocked` an operation the +guard refused (a reader that tried to import, open or run code), the rest a refused file.""" +CONVERT_AST_READERS = ( + ("ast.py", "parse"), + ("ast.py", "literal_eval"), + ("numpy/lib/_format_impl.py", "_read_array_header"), +) +"""The one `compile` convert mode lets through: NumPy's `.npy` header reader parsing its dict literal to a tree.""" DATA_BLOCKED_PREFIXES = ( "ctypes.", "os.system", @@ -1684,7 +1726,7 @@ def frame_csv(frame, measure=None): return text, None -def write_status(handle, code, measure): +def write_status(handle, code, measure, exits=DATA_EXIT): """`status.json`: the run's `DATA_EXIT` code and, for a broken cap, the measured value and the cap. The host reads it whatever the launcher's exit code says: Cloud Run's `sandbox do` @@ -1695,7 +1737,7 @@ def write_status(handle, code, measure): """ if handle is None: return - reason = next((name for name, value in DATA_EXIT.items() if value == code), None) + reason = next((name for name, value in exits.items() if value == code), None) report = {"exit": code, "reason": reason} for key in ("measured", "limit"): if isinstance(measure.get(key), int): @@ -1852,9 +1894,122 @@ def run_plot(path, guard): exec(code, namespace) +def chain_matches(frame, chain): + """Whether `frame` and the frames above it run the `(file suffix, function)` pairs of `chain`, in order.""" + for function in chain: + if not is_one_of(frame, (function,)): + return False + frame = frame.f_back + return True + + +class ConvertGuard(DataGuard): + """The audit hook of convert mode: `DataGuard`, with no code to run at all, but for one `compile`. + + NumPy reads an `.npy` header, a dict literal, with `ast.literal_eval`, which compiles the + text to a syntax tree (`compile` with `PyCF_ONLY_AST`, no code object): `CONVERT_AST_READERS`. + Every `exec` is refused, as are every `open` and every `import` after the preload. + """ + + def lets_code_run(self, event, args): + """Only NumPy's header compile: none of data mode's `CODE_BUILDERS` (`namedtuple`, dataclasses), which a + reader of data files never needs (Copilot review of #12194), and no entry code object to run.""" + return event == "compile" and chain_matches(self.caller(), CONVERT_AST_READERS) + + +def load_converter(): + """`convert.py` next to this file (the image's `/opt/anyplot/convert.py`), loaded by path: `-I` puts no script + directory on `sys.path`. Registered in `sys.modules` like an import.""" + import importlib.util + + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), CONVERT_MODULE_FILE) + spec = importlib.util.spec_from_file_location("anyplot_convert", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main_convert(path, guard=None): + """Convert mode (module docstring): read the file, preload, arm `ConvertGuard`, write `out.csv`, `meta.json` + and `status.json`.""" + _report("start") + set_limits() + sys.dont_write_bytecode = True + try: + status = open(DATA_STATUS_FILE, "w", encoding="utf-8") # before the guard, which refuses every open + except OSError: + status = None + measure = {} + code = CONVERT_EXIT["error"] + try: + code = run_conversion(path, guard, measure) + finally: + write_status(status, code, measure, CONVERT_EXIT) + return code + + +def run_conversion(path, guard, measure): + """`main_convert` after the limits. Only the exception's class reaches stderr: its message may quote the file.""" + try: + with open(path, "rb") as handle: + data = handle.read(CONVERT_MAX_INPUT_BYTES + 1) + request = {} + if os.path.exists(CONVERT_REQUEST_FILE): + with open(CONVERT_REQUEST_FILE, encoding="utf-8") as handle: + request = json.loads(handle.read(CONVERT_MAX_REQUEST_BYTES)) + except (OSError, ValueError): + return CONVERT_EXIT["usage"] + sheet = request.get("sheet") if isinstance(request, dict) else None + if sheet is not None and not isinstance(sheet, str): + return CONVERT_EXIT["usage"] + if len(data) > CONVERT_MAX_INPUT_BYTES: + measure.update(measured=len(data), limit=CONVERT_MAX_INPUT_BYTES) + return CONVERT_EXIT["too_large"] + converter = load_converter() + converter.preload() + guard = guard if guard is not None else ConvertGuard() + with ( + open(DATA_OUT_FILE, "w", encoding="utf-8", newline="") as out, + open(CONVERT_META_FILE, "w", encoding="utf-8") as meta, + ): + guard.arm() + try: + result = converter.convert(data, sheet) + text = result.csv_text + # Not ASCII-escaped: ten 255-character CJK sheet names would pass the host's 16 KiB cap as escapes. + report = json.dumps(result.meta(), ensure_ascii=False) + except converter.Refused as exc: + if guard.blocked: + return CONVERT_EXIT["blocked"] + for key in ("measured", "limit"): + value = getattr(exc, key, None) + if isinstance(value, int) and not isinstance(value, bool): + measure[key] = value + return CONVERT_EXIT.get(exc.reason, CONVERT_EXIT["error"]) + except BaseException as exc: # MemoryError and SystemExit too: the code is the harness's own + try: + print(f"conversion failed: {type(exc).__name__}", file=sys.stderr, flush=True) + except Exception: + pass + return CONVERT_EXIT["blocked"] if guard.blocked else CONVERT_EXIT["error"] + finally: + try: + _report("end", **_usage()) + except Exception: + pass + if guard.blocked: + return CONVERT_EXIT["blocked"] + out.write(text) + meta.write(report) + return CONVERT_EXIT["ok"] + + def main(argv): if len(argv) == 3 and argv[1] == DATA_FLAG: return main_data(argv[2]) + if len(argv) == 3 and argv[1] == CONVERT_FLAG: + return main_convert(argv[2]) _report("start") if len(argv) != 2: print("usage: harness.py ", file=sys.stderr) diff --git a/agents/anyplot/render/runtimes/python.py b/agents/anyplot/render/runtimes/python.py index a088f5d5a80..c22e9bb1649 100644 --- a/agents/anyplot/render/runtimes/python.py +++ b/agents/anyplot/render/runtimes/python.py @@ -18,7 +18,7 @@ from ...code.normalise import normalise from ...code.validate import validate_security from ..contract import Theme -from ..harness import DATA_FLAG +from ..harness import CONVERT_FLAG, DATA_FLAG from ..png import MAX_PNG_BYTES, MAX_PROBE_BYTES, read_output @@ -26,6 +26,10 @@ IMAGE_HARNESS = "/opt/anyplot/harness.py" MPL_SEED = "/opt/mplconfig" HARNESS_SOURCE = Path(__file__).resolve().parents[1] / "harness.py" +IMAGE_CONVERTER = "/opt/anyplot/convert.py" +CONVERTER_SOURCE = HARNESS_SOURCE.with_name("convert.py") +CONVERT_INPUT_NAME = "input.bin" +"""A conversion's file (the renderer's `wire.CONVERT_INPUT_FILE`).""" DATA_FILE_NAME = "snippet.py" """A data run's code file (the renderer's `executor.SNIPPET_FILE`); `gates.error_summary` finds its frames by it.""" DATA_RLIMIT_AS_MB = 1024 @@ -86,6 +90,14 @@ def data_env(self) -> dict[str, str]: "ANYPLOT_RLIMIT_NPROC": str(DATA_RLIMIT_NPROC), } + def convert_command(self) -> list[str]: + """A conversion: the harness in convert mode on the file written as `input.bin`.""" + return [self.python, "-I", self.harness, CONVERT_FLAG, CONVERT_INPUT_NAME] + + def convert_env(self) -> dict[str, str]: + """A data run's environment, with Arrow's system allocator (the renderer's `sandbox_env` for a conversion).""" + return {**self.data_env(), "ARROW_DEFAULT_MEMORY_POOL": "system"} + def normalise(self, code: str, library: str) -> str: return normalise(code, library=library) diff --git a/agents/anyplot/render/serial.py b/agents/anyplot/render/serial.py index 504d73f8e41..bd0ac08960a 100644 --- a/agents/anyplot/render/serial.py +++ b/agents/anyplot/render/serial.py @@ -18,7 +18,9 @@ The render's own `job.timeout_s` starts when the backend runs it, after the wait. A data run (`run_data`, the `run_data_code` tool) takes one slot of the `run` lane -like one theme of a render: it is a sandbox too. No ADK import. +like one theme of a render: it is a sandbox too. A file conversion (`convert`, the dataset +route's `/convert`) takes one slot of the `toggle` lane, so it waits behind every pipeline +render and never delays one. No ADK import. """ import asyncio @@ -26,7 +28,7 @@ from dataclasses import replace from typing import Literal -from .contract import DataJob, DataOutput, RenderBackend, RenderJob, RenderResult +from .contract import ConvertJob, ConvertOutput, DataJob, DataOutput, RenderBackend, RenderJob, RenderResult RenderLane = Literal["run", "toggle"] @@ -117,3 +119,15 @@ async def run_data(self, job: DataJob, *, wait_s: float | None = None) -> DataOu return await self.backend.run_data(job) finally: self.slots.release() + + async def convert(self, job: ConvertJob, *, wait_s: float | None = None) -> ConvertOutput: + """Convert a file in one slot of the `toggle` lane, behind every waiting pipeline render, so a conversion + never delays a plot; raises `RenderBusy` when the slot wait ran out.""" + try: + await self.slots.acquire("toggle", wait_s) + except TimeoutError: + raise RenderBusy(f"no render slot came free within {wait_s} s") from None + try: + return await self.backend.convert(job) + finally: + self.slots.release() diff --git a/agents/anyplot/settings.py b/agents/anyplot/settings.py index 72cfad5998b..226ea154e4a 100644 --- a/agents/anyplot/settings.py +++ b/agents/anyplot/settings.py @@ -46,6 +46,10 @@ | `AGENT_SESSION_CLOSE_WARNING` | `true` | The counted refusal one before the closing one carries the fixed `close_warning` suffix | | `AGENT_USER_PASTES_PER_MINUTE` | `10` | Dataset pastes per user within any 60 seconds, across sessions; more get `429 rate_limited` before any parser or model reads the paste | | `AGENT_USER_PASTES_PER_DAY` | `100` | Dataset pastes per user and UTC day; more get `429 daily_limit` | +| `AGENT_USER_CONVERSIONS_PER_MINUTE` | `5` | File conversions (a dropped spreadsheet, Parquet, Arrow or NumPy file read in the sandbox) per user within any 60 seconds, across sessions; more get `429 rate_limited`. The paste that follows a conversion counts as one paste | +| `AGENT_USER_CONVERSIONS_PER_DAY` | `30` | File conversions per user and UTC day; more get `429 daily_limit` | +| `AGENT_CONVERT_TIMEOUT_S` | `10` | Seconds one conversion may run in the sandbox, at most 15 | +| `AGENT_CONVERT_SLOT_WAIT_S` | `20` | Seconds a conversion waits for the render slot, behind every pipeline render; then `503 capacity` | | `AGENT_MAX_ATTEMPTS` | `2` | Adapter attempts per pipeline run, 1 to 3: the first and at most one repair round by default; the eval harness sets 1 or 3 to measure no repair or a second repair round | | `AGENT_ROOT_THINKING_BUDGET` | `1024` | Claude only: adaptive thinking of the root, with that many tokens added to its output cap; 0 turns it off (the 2026-10-11 measurement: with thinking on root and adapter every run passed the gates and 51.6 % ended `ok`, against 91.0 % and 28.7 % without) | | `AGENT_ADAPTER_THINKING_BUDGET` | `8192` | The same for the adapter (2,048 before 2026-10-14, when effort `high` thought past it); the eval harness overrides both with `--thinking-budget ROOT,ADAPTER` | @@ -360,6 +364,23 @@ class AgentSettings(BaseSettings): """Dataset pastes per user and UTC day (`AGENT_USER_PASTES_PER_DAY`), checked like the minute's; more get `429 daily_limit`. Owner decision of 2026-10-14.""" + user_conversions_per_minute: PositiveInt = 5 + """File conversions per user within any 60 seconds (`AGENT_USER_CONVERSIONS_PER_MINUTE`), across sessions: a + dropped spreadsheet, Parquet, Arrow or NumPy file read in the renderer's sandbox (`POST .../dataset/convert`). + Its own counter, apart from the pastes: a conversion spends sandbox time, no tokens, and the paste that + follows it counts as one paste. More get `429 rate_limited`. Owner decision of 2026-10-15.""" + + user_conversions_per_day: PositiveInt = 30 + """File conversions per user and UTC day (`AGENT_USER_CONVERSIONS_PER_DAY`); more get `429 daily_limit`.""" + + convert_timeout_s: float = Field(default=10.0, gt=0, le=15) + """Seconds one conversion may run in the sandbox (`AGENT_CONVERT_TIMEOUT_S`), host-enforced; at most the + renderer's 15 s (`wire.MAX_CONVERT_TIMEOUT_S`). Short, so a conversion never holds the render slot for long.""" + + convert_slot_wait_s: float = Field(default=20.0, gt=0, le=60) + """Seconds a conversion waits for the instance's render slot (`AGENT_CONVERT_SLOT_WAIT_S`), behind every + pipeline render (the `toggle` lane); past it the route answers `503 capacity`.""" + max_attempts: int = Field(default=2, ge=1, le=MAX_ATTEMPTS) """Adapter attempts per pipeline run (`AGENT_MAX_ATTEMPTS`), 1 to 3, enforced by the loop in `pipeline.py`. The default 2 is the design's "at most one repair round"; the eval harness diff --git a/agents/evals/matrix.py b/agents/evals/matrix.py index 0807c0000d4..d0a56314aa0 100644 --- a/agents/evals/matrix.py +++ b/agents/evals/matrix.py @@ -442,6 +442,10 @@ async def run_data(self, job: Any) -> Any: """A data run (`run_data_code`) is no plot attempt: passed through unrecorded.""" return await self.backend.run_data(job) + async def convert(self, job: Any) -> Any: + """A file conversion is no plot attempt either: passed through unrecorded.""" + return await self.backend.convert(job) + async def aclose(self) -> None: close = getattr(self.backend, "aclose", None) if close is not None: diff --git a/agents/main.py b/agents/main.py index 4f3d1579fc0..a02b2402a83 100644 --- a/agents/main.py +++ b/agents/main.py @@ -13,6 +13,7 @@ | `POST /v1/sessions` | `{user, spec_id, library, locale, snapshot}` | `{session_id, eligibility}` | `422 not_eligible` | | `POST /v1/sessions/{sid}/library` | `{library, snapshot}` | `{session_id, eligibility}` | `422 not_eligible`, `409 run_active` | | `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?, source?}` (`source`: the name of the file the page read the text from, at most 1,024 characters; screened on its own, never a note) | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version, language, file_name?}` (`file_name`: the dropped file's name as the session keeps it, `(name withheld)` when the screen refused it; `suggestion`: the wide-table offer of `data/shape.py`, or null; `context_lines`: the notes pasted above or below the table that passed the pre-filter and the scope judge, which the root gets with the next `summarize_data` turn; `language`: the session language as `{language, source}`, which a paste before any typed message or choice sets from the notes' or the column names' language, source `pasted`), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips that fallback and keeps every row) | `413 too_long`, `422 unparseable`, `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `403 session_closed` (the scope judge closed the session), `409 run_active` (a turn of the user in any session, or another paste of the user in flight), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_PASTES_PER_MINUTE` / `_PER_DAY`, or the daily token budgets for `daily_limit`), `429 locked_out` (the day's `attack` strikes), `503 guard_unavailable` | +| `POST /v1/sessions/{sid}/dataset/convert` | `{data_base64, sheet?}` (a dropped Excel, OpenDocument, Parquet, Feather, Arrow or NumPy file, at most 5 MiB; `sheet` a sheet or `.npz` array the previous answer named) | `{text, format, kind, rows, columns, sheet, sheets, sheet_count, warnings, dropped}`: the file read in the renderer's sandbox (`harness.py --convert`) into the text the paste box takes (CSV, or JSON records for a table with list columns, `format`), every value with the precision it came with, the sheet read and at most 10 visible ones with cells (`sheet_count` all of them), `warnings` counts (`formulas_without_value`, `hidden_sheets`, `dropped_columns`, `list_columns`, `utc_timestamps`, `renamed_columns`), `dropped` the names of the columns left out; nothing stored, no model call | `413 too_long`, `422 invalid`, `422 file_` (`wire.CONVERT_REASONS`, with `measured` and `limit` for a broken cap) and `file_timeout`, `file_too_complex` (also a reader that died without its `status.json`), `file_too_large`, `file_unreadable`, `403 session_closed`, `409 run_active` (another conversion of the user in flight), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_CONVERSIONS_PER_MINUTE` / `_PER_DAY`, or the daily token budgets), `429 locked_out`, `503 capacity` (no render slot within `AGENT_CONVERT_SLOT_WAIT_S`), `503 convert_unavailable` (no renderer, or one without `/convert`) | | `PUT /v1/sessions/{sid}/bindings` | `[{role, column}]` | `{bindings, complete, missing_roles, warnings}`: `warnings` names a role that needs values of 0 or more bound to a column with a negative value, or left open for that reason (`data/signs.py`); never a refusal | `409 run_active`, `422 invalid` (with `errors`, at most 20 lines) | | `PUT /v1/sessions/{sid}/language` | `{language}` (a short name or code, `^[A-Za-z][A-Za-z -]{0,39}$`) | `{language, source}`: the session language in its stored form (`English` is `en`), source `chosen`, which detection never overrides | `409 run_active`, `422` (the pattern) | | `POST /v1/sessions/{sid}/dataset/confirm` | | `{confirmed, rows, columns}`: step 1 of two, the user confirmed the dataset with its bindings; the pipeline runs only then | `409 run_active`, `422 no_dataset` | @@ -169,6 +170,7 @@ from agents.anyplot.plugins.ledger import CURRENT_LEDGER, RequestLedger, attribution, budget_allows, judge_budget_tokens from agents.anyplot.plugins.scope_guard import LOCKED_OUT, SESSION_CLOSED, scope_refusals from agents.anyplot.policy import fence, file_name_block, refusal, scope_rubric, transcribe_rubric, user_notes_block +from agents.anyplot.render.contract import ConvertJob, RendererUnavailable from agents.anyplot.render.serial import RenderBusy from agents.anyplot.render.store import RenderStoreFull from agents.anyplot.render.watermark import add_footer @@ -197,6 +199,12 @@ ) from agents.anyplot.settings import AgentSettings, get_settings from agents.anyplot.theme_render import RenderGone, render_theme +from agents.renderer.wire import ( + CONVERT_HARNESS_EXITS, + MAX_CONVERT_BASE64_CHARS, + MAX_CONVERT_INPUT_BYTES, + MAX_SHEET_NAME_CHARS, +) from agents.stream import MAX_DONE_ROLES, Translator from core.constants import SUPPORTED_LIBRARIES @@ -419,6 +427,8 @@ class Runtime: artifact_service: InMemoryArtifactService = field(default_factory=InMemoryArtifactService) active: dict[str, ActiveRun] = field(default_factory=dict) pasting: set[str] = field(default_factory=set) + converting: set[str] = field(default_factory=set) + """The users with a file conversion in flight: one per user at a time (`409 run_active` for a second).""" eligibility_cache: dict[tuple[str, str], Eligibility] = field(default_factory=dict) runner: Runner | None = None queue: RunQueue | None = None @@ -634,6 +644,14 @@ class DatasetBody(_Body): note; the text itself is the file's, unchanged. Never logged.""" +class ConvertBody(_Body): + """A dropped binary data file as base64 (the BFF's CSRF guard takes JSON only), and the sheet the page chose.""" + + data_base64: str = Field(min_length=4, max_length=MAX_CONVERT_BASE64_CHARS) + sheet: str | None = Field(default=None, min_length=1, max_length=MAX_SHEET_NAME_CHARS) + """A sheet of a workbook or an array of an `.npz`, as the previous answer named it; None reads the first.""" + + class BindingBody(_Body): role: str = Field(min_length=1, max_length=32) column: str | None = Field(default=None, max_length=64) @@ -820,6 +838,130 @@ async def upload_dataset( runtime.pasting.discard(user) +CONVERT_FAILURES: dict[str, str] = { + "timeout": "file_timeout", + "memory": "file_too_complex", + "disk_budget": "file_too_complex", + "file_budget": "file_too_complex", + "output_rejected": "file_too_large", +} +"""A conversion the renderer stopped, by its run reason, as the route's refusal code; any other failure (the reader +raised, the guard refused an operation) is `file_unreadable`.""" + + +@app.post("/v1/sessions/{sid}/dataset/convert", dependencies=v1_dependencies) +async def convert_dataset_file( + sid: SessionId, body: ConvertBody, user: User, runtime: RuntimeDep, rid: RequestId +) -> JSONResponse: + """A dropped binary data file read in the renderer's sandbox into the text the paste box takes. + + Nothing is stored and no model reads anything: the page puts the text into the box and + the user presses `.parse()`, which goes through the ordinary paste route (its limits, the + pre-filter, the judges) with the file's name and sheet as `source`. The conversion has its + own counter (`AGENT_USER_CONVERSIONS_PER_MINUTE` / `_PER_DAY`, counted when the file goes to + the sandbox), checked after the paste route's daily budget and strikes, so a user who may + not paste spends no sandbox time; the paste that follows counts as one paste (owner + decision of 2026-10-15). One conversion per user at a time (`Runtime.converting`), across + sessions; it waits for the render slot behind every pipeline render. The attribution line + carries the kind, the outcome, the byte count and the time, never the file, its text or its + sheet names. + """ + services = get_services() + settings = get_settings() + session = await runtime.session(user, sid) + if scope_refusals(session.events) >= settings.session_refusals: + raise AgentsError(403, SESSION_CLOSED) + if user in runtime.converting: + raise AgentsError(409, "run_active") + try: + data = base64.b64decode(body.data_base64, validate=True) + except (ValueError, binascii.Error): + raise AgentsError(422, "invalid") from None + if not data: + raise AgentsError(422, "invalid") + if len(data) > MAX_CONVERT_INPUT_BYTES: + raise AgentsError(413, "too_long") + _paste_budget(services, user, settings) + ledger = RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action") + limit = services.usage.convert_limit(user, settings) + if limit is not None: + attribution("convert_limit", ledger, verdict=limit) + raise AgentsError(429, limit) + runtime.converting.add(user) # no await since the check above, so no second conversion slipped in between + try: + services.usage.add_conversion(user) + job = ConvertJob( + job_id=f"c{secrets.token_hex(8)}", data=data, sheet=body.sheet, timeout_s=settings.convert_timeout_s + ) + try: + output = await services.backend.convert(job, wait_s=settings.convert_slot_wait_s) + except RenderBusy: + attribution("data_convert", ledger, outcome="capacity", bytes=len(data)) + raise AgentsError(503, "capacity") from None + except (RendererUnavailable, NotImplementedError): + attribution("data_convert", ledger, outcome="unavailable", bytes=len(data)) + raise AgentsError(503, "convert_unavailable") from None + finally: + runtime.converting.discard(user) + kind = output.meta.kind if output.meta is not None else None + if output.exit_code == 0 and output.text is not None and output.meta is not None: + meta = output.meta + attribution( + "data_convert", + ledger, + outcome="ok", + kind=kind, + format=meta.format, + rows=meta.rows, + columns=meta.columns, + sheets=meta.sheet_count, + bytes=len(data), + wall_s=round(output.wall_s, 3), + ) + return JSONResponse( + { + "text": output.text, + "format": meta.format, + "kind": meta.kind, + "rows": meta.rows, + "columns": meta.columns, + "sheet": meta.sheet, + "sheets": meta.sheets, + "sheet_count": meta.sheet_count, + "warnings": dict(meta.warnings), + "dropped": list(meta.dropped), + }, + headers=_NO_STORE, + ) + if output.refusal is not None: + code = f"file_{output.refusal}" + elif output.timed_out: + code = "file_timeout" + elif output.reason is not None: + code = CONVERT_FAILURES.get(output.reason, "file_unreadable") + elif output.exit_code is None or output.exit_code not in CONVERT_HARNESS_EXITS: + # A process that died without its `status.json` (a native reader's SIGABRT on an allocation it could not + # make; security review of #12194): the file was too much for the sandbox, not unreadable. + code = "file_too_complex" + else: + code = "file_unreadable" + attribution( + "data_convert", + ledger, + outcome=code, + exit_code=output.exit_code, + reason=output.reason, + bytes=len(data), + wall_s=round(output.wall_s, 3), + ) + content: dict[str, Any] = {"detail": code} + if output.refusal is not None: + content.update( + {key: value for key, value in (("measured", output.measured), ("limit", output.limit)) if value is not None} + ) + return JSONResponse(status_code=422, content=content, headers=_NO_STORE) + + def _paste_budget(services: Services, user: str, settings: AgentSettings) -> None: """`429 locked_out` past the day's `attack` strikes, `429 daily_limit` past the daily token budgets.""" if services.usage.daily_ok(user, settings): diff --git a/agents/renderer/Dockerfile b/agents/renderer/Dockerfile index b0dbcfb1ef0..3aa687211c3 100644 --- a/agents/renderer/Dockerfile +++ b/agents/renderer/Dockerfile @@ -36,7 +36,32 @@ COPY pyproject.toml uv.lock README.md ./ # imports it. `--no-install-project` keeps the project itself out of the venv, so a # sandbox's interpreter finds no anyplot package on its path; the service imports # `agents.renderer` from the working directory, which uvicorn puts on `sys.path`. -RUN uv venv && uv sync --frozen --no-install-project --extra lib-matplotlib --extra lib-seaborn +# The `convert` extra (python-calamine, pyarrow) is the file conversion's readers +# (`harness.py --convert`, agents/anyplot/render/convert.py). +RUN uv venv && uv sync --frozen --no-install-project --extra lib-matplotlib --extra lib-seaborn --extra convert + +# pyarrow ships far more than the converter reads (owner decision of 2026-10-15: slim it). +# The conversion needs the core, compute, IPC, Feather and Parquet; it never touches +# Flight, the cloud and HDFS filesystems, ORC, Substrait's Python module, Parquet +# encryption or the C++ test module, nor the headers, Cython sources and tests a build +# against pyarrow would use. libarrow_substrait itself stays: `lib` links it. Of the +# three identical copies of libarrow_python only the one the modules load by its soname +# stays. 163 MB installed become 99 MB; the CI job converts every format inside the +# built image (`tests/image/convert_smoke.py`), so a removed file that is needed fails it. +RUN p=.venv/lib/python3.13/site-packages/pyarrow \ + && du -sh "$p" \ + && rm -rf "$p/include" "$p/includes" "$p/src" "$p/tests" \ + && find "$p" \( -name '*.pxd' -o -name '*.pyx' -o -name '*.pxi' \) -delete \ + && rm -f "$p"/_flight.cpython-*.so "$p"/libarrow_flight.so.* "$p"/libarrow_python_flight.so* \ + "$p"/_s3fs.cpython-*.so "$p"/libarrow_s3.so.* \ + "$p"/_azurefs.cpython-*.so "$p"/_gcsfs.cpython-*.so "$p"/_hdfs.cpython-*.so \ + "$p"/_orc.cpython-*.so "$p"/_dataset_orc.cpython-*.so \ + "$p"/_substrait.cpython-*.so "$p"/_pyarrow_cpp_tests.cpython-*.so \ + "$p"/_parquet_encryption.cpython-*.so "$p"/_dataset_parquet_encryption.cpython-*.so \ + "$p"/libarrow_python_parquet_encryption.so* \ + "$p"/libarrow_python.so "$p"/libarrow_python.so.*.*.* \ + && du -sh "$p" \ + && .venv/bin/python -c "import pyarrow.parquet, pyarrow.feather, pyarrow.ipc, pyarrow.compute" # The image does not install the project, so its version is written to a file # that `GET /status` reports. @@ -96,6 +121,7 @@ COPY agents/__init__.py ./agents/__init__.py COPY agents/renderer/ ./agents/renderer/ COPY --from=builder /app/VERSION ./agents/renderer/VERSION COPY agents/anyplot/render/harness.py /opt/anyplot/harness.py +COPY agents/anyplot/render/convert.py /opt/anyplot/convert.py # The fallback fonts and their OFL notices. matplotlib scans /usr/share/fonts # recursively without fontconfig (font_manager.findSystemFonts), so one diff --git a/agents/renderer/executor.py b/agents/renderer/executor.py index 6c2d6452dc6..9ef924ee969 100644 --- a/agents/renderer/executor.py +++ b/agents/renderer/executor.py @@ -65,8 +65,16 @@ every failed render and data run since 2026-10-11. The attempt log carries both, with the signal as `launcher_signal`. +**Conversions** (`run_convert`, the service's `POST /convert`) take the same path again: the +file's bytes as `input.bin` (`_prepare` writes bytes as they are), the chosen sheet in +`request.json`, the harness in convert mode (`harness.py --convert input.bin`) and Arrow's +system allocator in the environment. After exit code 0 the host reads `out.csv` under the +paste cap (`MAX_CONVERT_TEXT_BYTES`, strict UTF-8) and `meta.json` (`wire.ConvertMeta`); a +refusal's exit code becomes its name (`wire.convert_refusal`), with the measured value and +the cap from `status.json`. + Strictly serial: the service holds one render slot (`main.py`), and the themes of a -job run one after the other; a data run takes the same slot. No ADK import. +job run one after the other; a data run and a conversion take the same slot. No ADK import. """ import asyncio @@ -88,12 +96,20 @@ from .settings import RendererSettings from .wire import ( + CONVERT_INPUT_FILE, + CONVERT_META_FILE, + CONVERT_REQUEST_FILE, DATA_STATUS_FILE, + MAX_CONVERT_META_BYTES, + MAX_CONVERT_TEXT_BYTES, MAX_CSV_BYTES, MAX_DATA_STATUS_BYTES, MAX_PNG_BYTES, MAX_PROBE_BYTES, MAX_STDERR_CHARS, + ConvertMeta, + ConvertRequest, + ConvertRun, DataRun, DataRunRequest, RenderRequest, @@ -101,6 +117,7 @@ Theme, ThemeRun, clean_probe, + convert_refusal, parse_data_status, ) @@ -120,6 +137,10 @@ """The harness's data-mode flag (`harness.DATA_FLAG`).""" DATA_LABEL = "data" """The sandbox name suffix and log label of a data run, where a render has its theme.""" +CONVERT_FLAG = "--convert" +"""The harness's convert-mode flag (`harness.CONVERT_FLAG`).""" +CONVERT_LABEL = "convert" +"""The sandbox name suffix and log label of a conversion.""" LAUNCHER_FAILURE = "failed to exec in container" HARNESS_START = 'HARNESS {"event": "start"}' HARNESS_END = 'HARNESS {"event": "end"' @@ -375,10 +396,14 @@ def read_regular(path: Path, max_bytes: int) -> bytes | None: return bytes(chunks) -def sandbox_env(config: ExecutorConfig, theme: Theme | None, cpu_s: int) -> dict[str, str]: +def sandbox_env(config: ExecutorConfig, theme: Theme | None, cpu_s: int, *, convert: bool = False) -> dict[str, str]: """The complete environment inside the sandbox: nothing of the host's reaches it. - A data run (`theme` None) gets no theme and no matplotlib settings: it imports no plotting library. + A data run or a conversion (`theme` None) gets no theme and no matplotlib settings: it imports no plotting + library. A conversion also gets Arrow's system allocator (`ARROW_DEFAULT_MEMORY_POOL=system`), so pyarrow's + buffers come from the same `malloc` as everything else under the address-space limit; with the bundled + jemalloc pool, a test run on a developer machine logged a failed background-thread start under the process + limit. """ plot: dict[str, str] = {} if theme is not None: @@ -400,6 +425,8 @@ def sandbox_env(config: ExecutorConfig, theme: Theme | None, cpu_s: int) -> dict "ANYPLOT_RLIMIT_CPU_S": str(cpu_s), "ANYPLOT_RLIMIT_FSIZE_MB": str(config.rlimit_fsize_mb), } + if convert: + env["ARROW_DEFAULT_MEMORY_POOL"] = "system" if config.rlimit_as_mb: env["ANYPLOT_RLIMIT_AS_MB"] = str(config.rlimit_as_mb) if config.rlimit_nproc: @@ -407,9 +434,11 @@ def sandbox_env(config: ExecutorConfig, theme: Theme | None, cpu_s: int) -> dict return env -def build_argv(config: ExecutorConfig, name: str, run_dir: Path, theme: Theme | None, cpu_s: int) -> list[str]: - """The `sandbox do` command of one theme, or of a data run (`theme` None, the harness in data mode); - asserted to carry neither `--write` nor `--allow-egress`.""" +def build_argv( + config: ExecutorConfig, name: str, run_dir: Path, theme: Theme | None, cpu_s: int, *, convert: bool = False +) -> list[str]: + """The `sandbox do` command of one theme, of a data run (`theme` None, the harness in data mode) or of a + conversion (`convert`, the harness in convert mode); asserted to carry neither `--write` nor `--allow-egress`.""" argv = [ config.sandbox_binary, "do", @@ -421,11 +450,14 @@ def build_argv(config: ExecutorConfig, name: str, run_dir: Path, theme: Theme | if config.bind_tmp: argv += ["--mount", f"type=bind,source={run_dir / 'tmp'},destination=/tmp"] argv += ["-w", WORKDIR] - for key, value in sandbox_env(config, theme, cpu_s).items(): + for key, value in sandbox_env(config, theme, cpu_s, convert=convert).items(): argv += ["--env", f"{key}={value}"] flags = list(argv) argv += ["--", config.python, "-I", config.harness] - argv += [CODE_FILE] if theme is not None else [DATA_FLAG, SNIPPET_FILE] + if convert: + argv += [CONVERT_FLAG, CONVERT_INPUT_FILE] + else: + argv += [CODE_FILE] if theme is not None else [DATA_FLAG, SNIPPET_FILE] if any(part.split("=", 1)[0] in FORBIDDEN_FLAGS for part in flags): raise AssertionError("a sandbox must never get --write or --allow-egress") return argv @@ -452,6 +484,8 @@ class Attempt: """A data run's exit code as the harness wrote it to `status.json`; None without a readable one.""" status_measured: int | None = None status_limit: int | None = None + convert_meta: ConvertMeta | None = None + """A conversion's `meta.json`, read with its `out.csv` after exit code 0.""" @property def data_exit(self) -> int | None: @@ -535,6 +569,22 @@ def to_data_wire(self, attempts: int) -> DataRun: attempts=attempts, ) + def to_convert_wire(self, attempts: int) -> ConvertRun: + from_status = self.reason is None and self.exit_code is not None + exit_code = self.data_exit + return ConvertRun( + exit_code=exit_code, + timed_out=self.timed_out, + reason=self.reason, + refusal=convert_refusal(exit_code) if from_status else None, + measured=self.status_measured if from_status else self.measured, + limit=self.status_limit if from_status else self.limit, + text=self.csv_text if self.convert_meta is not None else None, + meta=self.convert_meta if self.csv_text is not None else None, + wall_s=round(self.wall_s, 3), + attempts=attempts, + ) + def _b64(data: bytes) -> str: return base64.b64encode(data).decode("ascii") @@ -561,6 +611,8 @@ async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: ... async def run_data(self, job: DataRunRequest) -> DataRun: ... + async def run_convert(self, job: ConvertRequest) -> ConvertRun: ... + class SandboxExecutor: """Runs one theme of a job per call in a fresh Cloud Run sandbox.""" @@ -589,43 +641,60 @@ def available(self) -> bool: async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: """Render one theme; a launcher failure before the harness started is retried once.""" - files = {CODE_FILE: job.source, DATA_FILE: job.data_csv} + files: dict[str, str | bytes] = {CODE_FILE: job.source, DATA_FILE: job.data_csv} attempt, attempts = await self._launch(files, theme, job.timeout_s) return attempt.to_wire(theme, attempts) async def run_data(self, job: DataRunRequest) -> DataRun: """Run one data snippet with the harness in data mode; the same retry, watchdog and kill path as a theme.""" - attempt, attempts = await self._launch({SNIPPET_FILE: job.source}, None, job.timeout_s) + files: dict[str, str | bytes] = {SNIPPET_FILE: job.source} + attempt, attempts = await self._launch(files, None, job.timeout_s) return attempt.to_data_wire(attempts) - async def _launch(self, files: dict[str, str], theme: Theme | None, timeout_s: float) -> tuple[Attempt, int]: - """One run (a theme, or a data run when `theme` is None) and its launches; one retry for a launcher failure.""" + async def run_convert(self, job: ConvertRequest) -> ConvertRun: + """Convert one file with the harness in convert mode: the file's bytes as `input.bin`, the sheet the page + chose in `request.json`; the same retry, watchdog and kill path as a data run.""" + files: dict[str, str | bytes] = { + CONVERT_INPUT_FILE: job.data(), + CONVERT_REQUEST_FILE: json.dumps({"sheet": job.sheet}), + } + attempt, attempts = await self._launch(files, None, job.timeout_s, convert=True) + return attempt.to_convert_wire(attempts) + + async def _launch( + self, files: dict[str, str | bytes], theme: Theme | None, timeout_s: float, *, convert: bool = False + ) -> tuple[Attempt, int]: + """One run (a theme, a data run when `theme` is None, a conversion with `convert`) and its launches; one + retry for a launcher failure.""" if not self.available: raise Unavailable("sandbox_unavailable") if self.stuck: raise Unavailable("stuck") - label = theme if theme is not None else DATA_LABEL + label = theme if theme is not None else CONVERT_LABEL if convert else DATA_LABEL cpu_s = max(1, min(self.config.rlimit_cpu_s, math.ceil(timeout_s))) started = self.clock() - attempt = await self._attempt(files, theme, cpu_s, timeout_s) + attempt = await self._attempt(files, theme, cpu_s, timeout_s, convert=convert) attempts = 1 if attempt.launcher_failed: remaining = timeout_s - (self.clock() - started) _log_attempt(attempt, label, retry=remaining >= MIN_RETRY_S) if remaining >= MIN_RETRY_S: - attempt = await self._attempt(files, theme, cpu_s, remaining) + attempt = await self._attempt(files, theme, cpu_s, remaining, convert=convert) attempts = 2 if attempt.launcher_failed: attempt.reason = "launcher" _log_attempt(attempt, label, retry=False) return attempt, attempts - async def _attempt(self, files: dict[str, str], theme: Theme | None, cpu_s: int, timeout_s: float) -> Attempt: - name = f"r-{secrets.token_hex(6)}-{theme if theme is not None else DATA_LABEL}" + async def _attempt( + self, files: dict[str, str | bytes], theme: Theme | None, cpu_s: int, timeout_s: float, *, convert: bool = False + ) -> Attempt: + label = theme if theme is not None else CONVERT_LABEL if convert else DATA_LABEL + name = f"r-{secrets.token_hex(6)}-{label}" run_dir = self.config.runs_dir / name try: self._prepare(run_dir, files) - return await self._run(name, run_dir, theme, cpu_s, timeout_s) + return await self._run(name, run_dir, theme, cpu_s, timeout_s, convert=convert) except OSError as exc: # A full volume, a failed fork: the host could not run the job, which is # not the code's fault. `_run` has stopped any sandbox it started. @@ -634,8 +703,9 @@ async def _attempt(self, files: dict[str, str], theme: Theme | None, cpu_s: int, finally: shutil.rmtree(run_dir, ignore_errors=True) - def _prepare(self, run_dir: Path, files: dict[str, str]) -> None: - """The run directory with the code (and a render's data); the sandbox mounts its `work` directory.""" + def _prepare(self, run_dir: Path, files: dict[str, str | bytes]) -> None: + """The run directory with the code (and a render's data, or a conversion's file as bytes); the sandbox + mounts its `work` directory.""" work = run_dir / "work" work.mkdir(parents=True) # The sandbox is uid 0 behind gVisor's gofer, and the spikes ran with these @@ -646,11 +716,16 @@ def _prepare(self, run_dir: Path, files: dict[str, str]) -> None: tmp = run_dir / "tmp" tmp.mkdir() os.chmod(tmp, 0o1777) - for file_name, text in files.items(): - (work / file_name).write_text(text, encoding="utf-8") + for file_name, content in files.items(): + if isinstance(content, bytes): + (work / file_name).write_bytes(content) + else: + (work / file_name).write_text(content, encoding="utf-8") - async def _run(self, name: str, run_dir: Path, theme: Theme | None, cpu_s: int, timeout_s: float) -> Attempt: - argv = build_argv(self.config, name, run_dir, theme, cpu_s) + async def _run( + self, name: str, run_dir: Path, theme: Theme | None, cpu_s: int, timeout_s: float, *, convert: bool = False + ) -> Attempt: + argv = build_argv(self.config, name, run_dir, theme, cpu_s, convert=convert) started = self.clock() process = await asyncio.create_subprocess_exec( *argv, @@ -706,7 +781,9 @@ async def _run(self, name: str, run_dir: Path, theme: Theme | None, cpu_s: int, if theme is None: if stop is None: self._read_status(attempt, run_dir / "work") - if attempt.data_exit == 0: + if attempt.data_exit == 0 and convert: + self._collect_convert(attempt, run_dir / "work") + elif attempt.data_exit == 0: self._collect_data(attempt, run_dir / "work") elif stop is None or stop.reason == "timeout": self._collect(attempt, run_dir / "work", theme) @@ -740,6 +817,27 @@ def _collect_data(self, attempt: Attempt, work: Path) -> None: except UnicodeDecodeError: attempt.reason = "output_rejected" + def _collect_convert(self, attempt: Attempt, work: Path) -> None: + """Read the text (`out.csv`, at most the paste cap, strict UTF-8) and `meta.json` a conversion left; a text + without a readable meta, or a meta without a text, is `output_rejected`.""" + try: + raw = read_regular(work / OUT_FILE, MAX_CONVERT_TEXT_BYTES) + meta = read_regular(work / CONVERT_META_FILE, MAX_CONVERT_META_BYTES) + except OutputRejected as exc: + attempt.reason = "output_rejected" + attempt.measured = exc.size + attempt.limit = exc.limit + return + try: + if raw is None or meta is None: + raise ValueError("missing output") + attempt.csv_text = raw.decode("utf-8") + attempt.convert_meta = ConvertMeta.model_validate_json(meta, strict=True) + except ValueError: # UnicodeDecodeError and pydantic's ValidationError are ValueErrors + attempt.csv_text = None + attempt.convert_meta = None + attempt.reason = "output_rejected" + def _collect(self, attempt: Attempt, work: Path, theme: Theme) -> None: """Read the PNG and the probe the run left; a refused PNG is reported, a refused probe is simply absent.""" try: diff --git a/agents/renderer/main.py b/agents/renderer/main.py index 91374825f3a..ac143626683 100644 --- a/agents/renderer/main.py +++ b/agents/renderer/main.py @@ -13,7 +13,8 @@ |---|---|---|---| | `POST /render` | `RenderRequest` (`wire.py`): `{job_id, language, library, source, data_csv, themes, timeout_s}` | `RenderResponse`: per theme the exit code, a timeout flag, the stderr tail, the probe, the PNG as base64, the attempts, a reason with the measured value and the limit; the slot wait and total time; `MemAvailable` | `401 unauthenticated`, `403 forbidden`, `409 job_conflict` (another payload for a `job_id` and themes in flight or just finished), `422 invalid_job`, `499 cancelled` or `499 client_gone` (nobody reads it), `500 internal`, `503 busy` (no slot within `RENDERER_SLOT_WAIT_S`), `503 low_memory`, `503 stuck`, `503 sandbox_unavailable`, `503 volume_missing`, `503 io` | | `POST /run-data` | `DataRunRequest`: `{job_id, source, timeout_s}` (`timeout_s` at most 30) | `DataRunResponse`: the exit code (`harness.DATA_EXIT`), a timeout flag, a reason with the measured value and the limit, `csv_text` (the `out.csv` the harness wrote, only after exit code 0), the stderr tail, the attempts, the timings, the version, `MemAvailable` | the errors of `POST /render` | -| `POST /render/{job_id}/cancel` | | `{"cancelled": n}`: the renders and data runs of that job it stopped | `401`, `403`, `422 invalid_job` | +| `POST /convert` | `ConvertRequest`: `{job_id, data_base64, sheet?, timeout_s}` (the file at most 5 MiB, `timeout_s` at most 15) | `ConvertResponse`: the exit code (`harness.CONVERT_EXIT`), a timeout flag, a run reason, `refusal` (`wire.CONVERT_REASONS`) with the measured value and the cap, `text` (CSV, or JSON records for a table with list columns, at most 1 MiB) and `meta` (`{kind, format, rows, columns, sheet, sheets, sheet_count, warnings}`) after exit code 0, the attempts, the timings, the version, `MemAvailable` | the errors of `POST /render` | +| `POST /render/{job_id}/cancel` | | `{"cancelled": n}`: the renders, data runs and conversions of that job it stopped | `401`, `403`, `422 invalid_job` | | `GET /status` | | `RendererStatus`: name, version, revision, in flight, waiting, stuck launchers, `MemAvailable`, whether the launcher exists, whether the run volume is in place | `401`, `403` | All routes sit behind the caller check (`auth.py`) and Cloud Run IAM. There is no @@ -35,6 +36,12 @@ idempotency rules (its key is the `job_id` and `data`). A renderer deployed before the route answers it `404`, which the agents side reads as `unavailable`. +**Conversions.** `POST /convert` (drag and drop of binary data files, owner decision of +2026-10-15) reads one file with the harness in convert mode (`harness.py --convert`, +`agents/anyplot/render/convert.py`) under the same guard, slot, memory floor, watchdog, kill +path and idempotency rules (key: the `job_id` and `convert`). Nothing of the file, its text or +its sheet names is logged; the log line carries the kind, the exit code and the refusal. + **`job_id` and the themes are the idempotency key.** The `remote` backend and the deploy smoke replay a `POST /render` after an ambiguous failure (a connection error, a front-end 429 or 5xx), and the first request may have reached this service all the @@ -80,7 +87,7 @@ class and answered `500 internal`, because its message can hold the job's conten import signal import time from collections import OrderedDict -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Coroutine from dataclasses import dataclass from functools import lru_cache from pathlib import Path @@ -96,6 +103,8 @@ class and answered `500 internal`, because its message can hold the job's conten from .settings import RendererSettings, get_settings from .wire import ( JOB_ID_PATTERN, + ConvertRequest, + ConvertResponse, DataRunRequest, DataRunResponse, RendererStatus, @@ -125,16 +134,20 @@ class JobConflict(Exception): """A `POST /render` whose key is in flight or just finished with another payload; answered `409 job_conflict`.""" -type Job = RenderRequest | DataRunRequest -type Answer = RenderResponse | DataRunResponse +type Job = RenderRequest | DataRunRequest | ConvertRequest +type Answer = RenderResponse | DataRunResponse | ConvertResponse type JobKey = tuple[str, tuple[str, ...]] -"""The idempotency key of a request: its `job_id` and its themes, in order, or `("data",)` for a data run.""" +"""The idempotency key of a request: its `job_id` and its themes, in order, `("data",)` for a data run and +`("convert",)` for a conversion.""" DATA_KEY = ("data",) +CONVERT_KEY = ("convert",) def job_key(job: Job) -> JobKey: if isinstance(job, DataRunRequest): return (job.job_id, DATA_KEY) + if isinstance(job, ConvertRequest): + return (job.job_id, CONVERT_KEY) return (job.job_id, tuple(job.themes)) @@ -241,9 +254,14 @@ def join(self, job: Job) -> "asyncio.Future[Answer]": answer: asyncio.Future[Answer] = asyncio.get_running_loop().create_future() answer.set_result(done[1]) return answer - task: asyncio.Task[Answer] = asyncio.create_task( - self.run_data(job) if isinstance(job, DataRunRequest) else self.render(job) - ) + work: Coroutine[Any, Any, Answer] + if isinstance(job, DataRunRequest): + work = self.run_data(job) + elif isinstance(job, ConvertRequest): + work = self.convert(job) + else: + work = self.render(job) + task: asyncio.Task[Answer] = asyncio.create_task(work) self._running[key] = RunningJob(print_, task, waiters=1) task.add_done_callback(lambda finished: self._forget(key, print_, finished)) return task @@ -365,6 +383,51 @@ async def _run_data(self, job: DataRunRequest, started: float) -> DataRunRespons ) return response + async def convert(self, job: ConvertRequest) -> ConvertResponse: + started = time.monotonic() + try: + return await self._convert(job, started) + finally: + self._exit_if_stuck() + + async def _convert(self, job: ConvertRequest, started: float) -> ConvertResponse: + """One file conversion in its own sandbox, inside the same single slot as the renders.""" + if self.volume is False: + raise Unavailable("volume_missing") + async with self.slot.hold(self.settings.slot_wait_s): + waited = time.monotonic() - started + floor = self.settings.min_mem_available_mb + available = mem_available_mb() + if floor and available is not None and available < floor: + raise Unavailable("low_memory") + run = await self.executor.run_convert(job) + response = ConvertResponse( + **run.model_dump(), + job_id=job.job_id, + timings=RenderTimings(slot_wait_s=round(waited, 3), total_s=round(time.monotonic() - started, 3)), + version=self.version, + mem_available_mb=mem_available_mb(), + ) + logger.info( + json.dumps( + { + "event": "convert", + "job_id": job.job_id, + "exit_code": response.exit_code, + "reason": response.reason, + "refusal": response.refusal, + "kind": None if response.meta is None else response.meta.kind, + "measured": response.measured, + "limit": response.limit, + "attempts": response.attempts, + "text_bytes": None if response.text is None else len(response.text.encode("utf-8")), + "slot_wait_s": response.timings.slot_wait_s, + "total_s": response.timings.total_s, + } + ) + ) + return response + def _exit_if_stuck(self) -> None: """On Cloud Run, a stuck launcher ends this instance: no render could run on it again.""" if self.terminating or not self.settings.cloud_run_service or not self.executor.stuck: @@ -472,6 +535,13 @@ async def run_data(job: DataRunRequest, renderer: RendererDep, request: Request) return await _serve(job, renderer, request) +@app.post("/convert", dependencies=caller_checked) +async def convert(job: ConvertRequest, renderer: RendererDep, request: Request) -> JSONResponse: + """Convert one data file (Excel, OpenDocument, Parquet, Feather, Arrow, NumPy) to CSV text in its own sandbox, + with the harness in convert mode, inside the same render slot.""" + return await _serve(job, renderer, request) + + async def _serve(job: Job, renderer: Renderer, request: Request) -> JSONResponse: """Join or start the job, race it against the client's disconnect, and answer it (see the module docstring).""" task = renderer.join(job) diff --git a/agents/renderer/wire.py b/agents/renderer/wire.py index 56b4d6c5a4b..087d79c653a 100644 --- a/agents/renderer/wire.py +++ b/agents/renderer/wire.py @@ -14,9 +14,11 @@ pattern) are pinned equal by `tests/unit/agents/renderer/test_wire.py`. """ +import base64 +import binascii import json import math -from typing import Any, Literal +from typing import Annotated, Any, Literal, get_args from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -239,6 +241,133 @@ class DataRunResponse(DataRun): mem_available_mb: int | None = None +MAX_CONVERT_INPUT_BYTES = 5 * 1024 * 1024 +"""The largest file `POST /convert` takes (`convert.MAX_INPUT_BYTES`, the agents' and the BFF's raw cap).""" +MAX_CONVERT_BASE64_CHARS = 4 * ((MAX_CONVERT_INPUT_BYTES + 2) // 3) +MAX_CONVERT_TIMEOUT_S = 15.0 +"""A conversion's time limit: the agents ask for 10 s, so a conversion never holds the render slot for long.""" +MAX_CONVERT_TEXT_BYTES = 1024 * 1024 +"""The text a conversion returns: the paste cap (`data/parse.py` `MAX_INPUT_BYTES`), because it goes into the paste box.""" +MAX_CONVERT_META_BYTES = 16 * 1024 +MAX_SHEET_NAME_CHARS = 255 +"""A requested sheet name: Excel allows 31 characters, OpenDocument any number; the converter names at most 100.""" +CONVERT_INPUT_FILE = "input.bin" +CONVERT_REQUEST_FILE = "request.json" # harness.CONVERT_REQUEST_FILE +CONVERT_META_FILE = "meta.json" # harness.CONVERT_META_FILE +CONVERT_REFUSAL_BASE = 10 +ConvertRefusal = Literal[ + "unsupported", + "encrypted", + "pickle", + "hdf5", + "corrupt", + "archive_limits", + "too_many_rows", + "too_many_columns", + "too_large", + "empty", + "sheet_not_found", + "array_shape", + "object_array", + "formulas_only", + "too_complex", +] +CONVERT_REASONS: tuple[ConvertRefusal, ...] = get_args(ConvertRefusal) +"""Why the converter refused a file (`convert.REFUSALS`, `harness.CONVERT_REFUSALS`, pinned equal): the harness exits +with `CONVERT_REFUSAL_BASE` plus the index.""" +ConvertKind = Literal["xlsx", "xlsm", "xlsb", "xls", "ods", "parquet", "feather", "arrow", "npy", "npz"] +ConvertWarning = Literal[ + "formulas_without_value", "hidden_sheets", "dropped_columns", "list_columns", "utc_timestamps", "renamed_columns" +] + + +def convert_refusal(exit_code: int | None) -> ConvertRefusal | None: + """The refusal a convert-mode exit code stands for, or None for any other code.""" + if exit_code is None: + return None + index = exit_code - CONVERT_REFUSAL_BASE + return CONVERT_REASONS[index] if 0 <= index < len(CONVERT_REASONS) else None + + +class ConvertRequest(BaseModel): + """One file to convert to CSV text in a sandbox of its own (`POST /convert`), the bytes as base64.""" + + model_config = ConfigDict(extra="forbid") + + job_id: str = Field(pattern=JOB_ID_PATTERN) + data_base64: str = Field(min_length=4, max_length=MAX_CONVERT_BASE64_CHARS) + sheet: str | None = Field(default=None, min_length=1, max_length=MAX_SHEET_NAME_CHARS) + timeout_s: float = Field(gt=0, le=MAX_CONVERT_TIMEOUT_S) + + @field_validator("data_base64") + @classmethod + def _base64(cls, value: str) -> str: + try: + raw = base64.b64decode(value, validate=True) + except (ValueError, binascii.Error): + raise ValueError("data_base64 is not base64") from None + if not raw or len(raw) > MAX_CONVERT_INPUT_BYTES: + raise ValueError("the file is empty or over the cap") + return value + + def data(self) -> bytes: + return base64.b64decode(self.data_base64, validate=True) + + +class ConvertMeta(BaseModel): + """`meta.json`, what the converter found (`convert.Conversion.meta`); read only after exit code 0.""" + + model_config = ConfigDict(extra="forbid") + + kind: ConvertKind + format: Literal["csv", "json"] = "csv" + """`json` for a table with list columns: records the parser expands into long format.""" + rows: int = Field(ge=0, le=10 * MAX_PROBE_INT) + columns: int = Field(ge=0, le=1_000) + sheet: str | None = Field(default=None, max_length=MAX_SHEET_NAME_CHARS) + sheets: list[Annotated[str, Field(max_length=MAX_SHEET_NAME_CHARS)]] = Field(default_factory=list, max_length=10) + sheet_count: int = Field(default=0, ge=0, le=100_000) + warnings: dict[ConvertWarning, Annotated[int, Field(ge=0, le=MAX_PROBE_INT)]] = Field(default_factory=dict) + dropped: list[Annotated[str, Field(max_length=100)]] = Field(default_factory=list, max_length=20) + """The names of the columns left out (durations, intervals, binary, map, union, nested), at most 20: the page + names them, because a step that changes the data says so (owner rule of 2026-10-15).""" + + +CONVERT_HARNESS_EXITS = frozenset( + {0, 1, 2, 8, *range(CONVERT_REFUSAL_BASE, CONVERT_REFUSAL_BASE + len(CONVERT_REASONS))} +) +"""Every exit code the harness's convert mode writes (`harness.CONVERT_EXIT`, pinned by a test). Any other code is a +launcher or a process that died without its `status.json` (a SIGABRT of a native reader), which the agents +service answers `file_too_complex`.""" + + +class ConvertRun(BaseModel): + """What one conversion's sandbox left: the exit code, the refusal it stands for, the text and the meta.""" + + model_config = ConfigDict(extra="forbid") + + exit_code: int | None + """The harness's exit code (`harness.CONVERT_EXIT`); None when the renderer killed the run.""" + timed_out: bool = False + reason: RunReason | None = None + refusal: ConvertRefusal | None = None + measured: int | None = None + limit: int | None = None + text: str | None = Field(default=None, max_length=MAX_CONVERT_TEXT_BYTES) + meta: ConvertMeta | None = None + wall_s: float = Field(default=0.0, ge=0) + attempts: int = Field(default=1, ge=1, le=2) + + +class ConvertResponse(ConvertRun): + """`POST /convert`: the run, with the job id, the timings and the instance's figures added.""" + + job_id: str = Field(pattern=JOB_ID_PATTERN) + timings: RenderTimings + version: str + mem_available_mb: int | None = None + + class RendererStatus(BaseModel): """`GET /status`: what the instance is doing and how much memory it has left.""" diff --git a/api/routers/agent.py b/api/routers/agent.py index 1018cc43d1f..fd8c756b65f 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -17,6 +17,8 @@ from __future__ import annotations import asyncio +import base64 +import binascii import gzip import hashlib import hmac @@ -84,6 +86,16 @@ MAX_DATASET_BYTES = 1024 * 1024 """Pasted data, "1 MB" of UTF-8 (200 KB before 2026-10-14): the agents parser's `MAX_INPUT_BYTES`, copied because the API image does not ship the agents package.""" +MAX_CONVERT_BYTES = 5 * 1024 * 1024 +"""A dropped binary data file (Excel, OpenDocument, Parquet, Feather, Arrow, NumPy), raw: the agents service's and +the renderer's cap (`agents/renderer/wire.py` `MAX_CONVERT_INPUT_BYTES`, copied because the API image does not ship +the agents package). It travels as base64 in JSON, about 6.7 MB, because the CSRF guard takes JSON only.""" +MAX_CONVERT_BASE64_CHARS = 4 * ((MAX_CONVERT_BYTES + 2) // 3) +MAX_REQUEST_BODY_BYTES = MAX_CONVERT_BASE64_CHARS + 4096 +"""The largest body any agent route takes, by its declared length: the conversion's base64 and the JSON around it +(the paste's 1 MB of text stays far below even with every character escaped).""" +CONVERT_PATH_SUFFIX = "/dataset/convert" +MAX_SHEET_NAME_CHARS = 255 MAX_MESSAGE_CHARS = 2000 MAX_FEEDBACK_CHARS = 500 """The cap of a feedback message after the strip, the site feedback's own (`api/routers/feedback.py`).""" @@ -160,6 +172,34 @@ """The agents run queue's `HEARTBEAT_S`: a waiting run's status repeats at least this often, so its run may have started up to this long before the BFF sees its first event.""" +# Why the agents service did not convert a dropped file (`agents/renderer/wire.py` `CONVERT_REASONS` with +# `file_` in front, plus the run failures of `agents/main.py` `CONVERT_FAILURES`; copied, because the API +# image does not ship the agents package). These pass with `measured` and `limit` when the agents service +# sends them, so the page can say "21,000 rows, the cap is 20,000". +CONVERT_REFUSAL_CODES = frozenset( + f"file_{reason}" + for reason in ( + "unsupported", + "encrypted", + "pickle", + "hdf5", + "corrupt", + "archive_limits", + "too_many_rows", + "too_many_columns", + "too_large", + "empty", + "sheet_not_found", + "array_shape", + "object_array", + "formulas_only", + "timeout", + "too_complex", + "unreadable", + ) +) +_MAX_MEASURE = 2**53 + # Error codes the agents service documents for its /v1 routes; an upstream error # body is never forwarded, only one of these codes when it names one. _UPSTREAM_CODES = frozenset( @@ -181,6 +221,8 @@ "daily_limit", "locked_out", "session_closed", + "convert_unavailable", + *CONVERT_REFUSAL_CODES, } ) # A spec data role, as the agents contract defines it (`agents/anyplot/schemas.py`, @@ -227,6 +269,7 @@ def __init__( errors: list[str] | None = None, *, case_id: str | None = None, + measure: dict[str, int] | None = None, ) -> None: super().__init__(detail) self.status_code = status_code @@ -234,6 +277,7 @@ def __init__( self.ref = ref self.errors = errors self.case_id = case_id + self.measure = measure async def agent_http_error_handler(request: Request, exc: AgentHTTPError) -> JSONResponse: @@ -247,6 +291,8 @@ async def agent_http_error_handler(request: Request, exc: AgentHTTPError) -> JSO content["errors"] = exc.errors if exc.case_id: content["case_id"] = exc.case_id + if exc.measure: + content.update(exc.measure) return JSONResponse(status_code=exc.status_code, content=content, headers=headers) @@ -305,6 +351,29 @@ def require_same_site_client(request: Request) -> None: raise AgentHTTPError(403, "json_required") +def require_body_within_cap(request: Request) -> None: + """Refuse a request body over `MAX_REQUEST_BODY_BYTES` by its declared `Content-Length`, before it is read. + + Placed before `require_utf8_body`, which reads the body. A route that declares a body + model has its body read by FastAPI before any dependency runs, so for those this is a + second line; the file conversion takes its body by hand (`convert_dataset_file`), so its + 5 MB file is refused here unread. The conversion also needs a declared length, whatever + else the request carries: a chunked body, or an HTTP/2 or proxied one with neither header, + has none to check (Copilot review of #12194). + """ + declared = request.headers.get("content-length") + if declared is None: + if request.method == "POST" and request.url.path.endswith(CONVERT_PATH_SUFFIX): + raise AgentHTTPError(411, "length_required") + return + try: + size = int(declared) + except ValueError: + raise AgentHTTPError(400, "bad_request") from None + if size > MAX_REQUEST_BODY_BYTES: + raise AgentHTTPError(413, "too_long") + + async def require_utf8_body(request: Request) -> None: """Refuse a JSON body that carries a lone surrogate (`"\\ud800"`) with a clean 422. @@ -382,6 +451,7 @@ def get_case_bucket() -> CaseBucket: Depends(require_admin), Depends(require_agent_enabled), Depends(require_same_site_client), + Depends(require_body_within_cap), Depends(require_utf8_body), ], ) @@ -447,6 +517,13 @@ class DatasetBody(_StrictBody): source: Annotated[str, Field(min_length=1, max_length=1_024)] | None = None +class ConvertBody(_StrictBody): + # A dropped binary data file as base64 (the CSRF guard takes JSON only), at most 5 MB raw, and the + # sheet (or `.npz` array) the page chose from the previous answer's `sheets`. + data_base64: Annotated[str, Field(min_length=4, max_length=MAX_CONVERT_BASE64_CHARS)] + sheet: Annotated[str, Field(min_length=1, max_length=MAX_SHEET_NAME_CHARS)] | None = None + + class Binding(_StrictBody): role: Annotated[str, Field(pattern=ROLE_PATTERN)] column: Annotated[str, Field(min_length=1, max_length=64)] | None = None @@ -617,9 +694,21 @@ def _upstream_error(response: httpx.Response, ref: str, *, binding_errors: bool code = "upstream" if status >= 500 else _GENERIC_CODES.get(status, "rejected") if binding_errors and status == 422 and code == "invalid": return AgentHTTPError(status, code, ref, errors=_binding_errors(body)) + if code in CONVERT_REFUSAL_CODES and isinstance(body, dict): + return AgentHTTPError(status, code, ref, measure=_measure(body)) return AgentHTTPError(status, code, ref) +def _measure(body: dict[str, Any]) -> dict[str, int] | None: + """A conversion refusal's `measured` and `limit`, when each is a whole number in range; nothing else passes.""" + kept = { + key: value + for key in ("measured", "limit") + if isinstance(value := body.get(key), int) and not isinstance(value, bool) and 0 <= value <= _MAX_MEASURE + } + return kept or None + + async def _call_upstream( client: httpx.AsyncClient, ctx: AgentContext, @@ -974,6 +1063,33 @@ async def upload_dataset(sid: SessionId, body: DatasetBody, ctx: Ctx, client: Cl return await _call_upstream(client, ctx, "POST", f"/sessions/{sid}/dataset", json_body=payload) +@router.post("/sessions/{sid}/dataset/convert") +async def convert_dataset_file(sid: SessionId, request: Request, ctx: Ctx, client: Client) -> Any: + """Read a dropped binary data file into the text the paste box takes (`{data_base64, sheet?}`): `{text, format, + kind, rows, columns, sheet, sheets, sheet_count, warnings}`, nothing stored. The agents service converts it in the + renderer's sandbox (Excel, OpenDocument, Parquet, Feather, Arrow, NumPy; pickle and HDF5 never). The body is taken + by hand, so `require_body_within_cap` refuses an oversized one by its `Content-Length` before it is read; 413 + above 5 MB raw; `422 file_` with `measured` and `limit` for a broken cap, `429 rate_limited` / + `daily_limit` past the conversion limits, `503 capacity` and `503 convert_unavailable`, passed through.""" + raw = await request.body() + if len(raw) > MAX_REQUEST_BODY_BYTES: + raise AgentHTTPError(413, "too_long", ctx.request_id) + try: + body = ConvertBody.model_validate_json(raw) + except ValueError: # pydantic's ValidationError, for JSON and shape alike; its message echoes the input + raise AgentHTTPError(422, "invalid", ctx.request_id) from None + try: + size = len(base64.b64decode(body.data_base64, validate=True)) + except (ValueError, binascii.Error): + raise AgentHTTPError(422, "invalid", ctx.request_id) from None + if size > MAX_CONVERT_BYTES: + raise AgentHTTPError(413, "too_long", ctx.request_id) + payload: dict[str, Any] = {"data_base64": body.data_base64} + if body.sheet is not None: + payload["sheet"] = body.sheet + return await _call_upstream(client, ctx, "POST", f"/sessions/{sid}/dataset/convert", json_body=payload) + + @router.put("/sessions/{sid}/bindings") async def put_bindings( sid: SessionId, bindings: Annotated[list[Binding], Body(max_length=50)], ctx: Ctx, client: Client diff --git a/app/scripts/agent-bff-mock.mjs b/app/scripts/agent-bff-mock.mjs index 001758e3533..3597d49a728 100644 --- a/app/scripts/agent-bff-mock.mjs +++ b/app/scripts/agent-bff-mock.mjs @@ -111,7 +111,9 @@ * - anything else: a refinement with a repair round, shipped as * `needs_attention`; "dark" in the text renders the dark theme. * Pasted data containing "ignore previous" answers `403 data_refused`; pasted - * text that looks like a formula answers `422 formula`. + * text that looks like a formula answers `422 formula`. `POST .../dataset/convert` + * (a dropped binary file) answers by the first bytes: a zip is a workbook with two + * sheets, Parquet one table, bytes holding "TOO_MANY_ROWS" the row-cap refusal. * * Environment: PORT (8010), MOCK_STEP_MS (700, per pipeline step), * MOCK_QUEUE_MS (2500, per queue position), MOCK_QUEUE (2, the starting @@ -1746,6 +1748,62 @@ async function route(req, res) { session.languageSource = 'chosen'; return json(res, 200, { language: session.language, source: 'chosen' }, ref); } + if (method === 'POST' && rest === '/dataset/convert') { + // The renderer reads the file in production; here the first bytes pick a canned answer. A zip is a + // workbook with the sheets "2024" and "2025", Parquet a single table, a file whose bytes hold + // "TOO_MANY_ROWS" the row-cap refusal with its counts, anything else `file_unsupported`. + if (typeof body?.data_base64 !== 'string') return fail(res, 422, 'invalid', ref); + const bytes = Buffer.from(body.data_base64, 'base64'); + if (bytes.length > 5 * 1024 * 1024) return fail(res, 413, 'too_long', ref); + await sleep(PARSE_MS); + if (bytes.includes('TOO_MANY_ROWS')) { + res.writeHead(422, { 'Content-Type': 'application/json', 'X-Request-Id': ref }); + res.end(JSON.stringify({ detail: 'file_too_many_rows', ref, measured: 48213, limit: 20000 })); + return; + } + const head = bytes.subarray(0, 4).toString('latin1'); + if (head === 'PK\u0003\u0004') { + const sheet = body.sheet ?? '2024'; + if (!['2024', '2025'].includes(sheet)) return fail(res, 422, 'file_sheet_not_found', ref); + const year = Number(sheet); + const text = `Umsatz ${year}\n\nmonth,revenue\n${year}-01,${year - 1900}\n${year}-02,${year - 1880}\n${year}-03,${year - 1870}\n`; + return json( + res, + 200, + { + text, + format: 'csv', + kind: 'xlsx', + rows: 5, + columns: 2, + sheet, + sheets: ['2024', '2025'], + sheet_count: 2, + warnings: { formulas_without_value: 2, hidden_sheets: 1 }, + }, + ref + ); + } + if (head === 'PAR1') { + return json( + res, + 200, + { + text: 'station,temp_c\nNord,4.5\nSüd,7.25\n', + format: 'csv', + kind: 'parquet', + rows: 2, + columns: 2, + sheet: null, + sheets: [], + sheet_count: 0, + warnings: {}, + }, + ref + ); + } + return fail(res, 422, 'file_unsupported', ref); + } if (method === 'POST' && rest === '/dataset/confirm') { if (session.active) return fail(res, 409, 'run_active', ref); if (!session.dataset) return fail(res, 422, 'no_dataset', ref); diff --git a/app/src/hooks/useAgentSession.ts b/app/src/hooks/useAgentSession.ts index d60bae1ea67..e77189bd20e 100644 --- a/app/src/hooks/useAgentSession.ts +++ b/app/src/hooks/useAgentSession.ts @@ -139,6 +139,7 @@ import { type AgentEvent, type ArtifactName, type Binding, + type ConvertAnswer, countBucket, type DatasetChanged, type DatasetParsed, @@ -1256,6 +1257,16 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess [token] ); + // A dropped binary file read on the server; the data card keeps its own state for it (nothing is stored). + const convertFile = useCallback( + async (dataBase64: string, sheet?: string): Promise => { + const sid = stateRef.current.sessionId; + if (!sid) throw new AgentApiError(0, 'no_session'); + return agentApi.convertFile(token, sid, dataBase64, sheet); + }, + [token] + ); + const loadImage = useCallback( async (number: number, theme: Theme, status: 'ok' | 'needs_attention') => { safeDispatch({ type: 'version_image', number, theme, image: { state: 'loading' } }); @@ -1826,6 +1837,7 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess switchLibrary, requestTheme, fetchArtifact, + convertFile, saveSession, setLanguage, sendFeedback, diff --git a/app/src/lib/agent.test.ts b/app/src/lib/agent.test.ts index 4a36a292ddd..d04958b9955 100644 --- a/app/src/lib/agent.test.ts +++ b/app/src/lib/agent.test.ts @@ -5,12 +5,14 @@ import { agentFetch, browserLocale, caseListQuery, + convertAnswer, countBucket, longSuggestion, parseAgentEvent, renderedThemes, sizeBucket, toAgentApiError, + toBase64, } from 'src/lib/agent'; import { ACCESS_RELOAD_KEY } from 'src/utils/adminAuth'; @@ -487,6 +489,74 @@ describe('toAgentApiError', () => { ); expect(storage.caseId).toBeNull(); }); + + it('reads the counts of a refused conversion, whole numbers only', async () => { + const counted = await toAgentApiError( + new Response( + JSON.stringify({ detail: 'file_too_many_rows', measured: 21000, limit: 20000 }), + { status: 422 } + ) + ); + expect(counted).toMatchObject({ code: 'file_too_many_rows', measured: 21000, limit: 20000 }); + const odd = await toAgentApiError( + new Response(JSON.stringify({ detail: 'file_corrupt', measured: '3', limit: 1.5 }), { + status: 422, + }) + ); + expect(odd).toMatchObject({ measured: null, limit: null }); + }); +}); + +describe('convertAnswer', () => { + it('keeps the known fields and the positive warning counts', () => { + expect( + convertAnswer({ + text: 'a\n1\n', + format: 'json', + kind: 'parquet', + rows: 1, + columns: 1, + sheet: null, + sheets: ['x', 3], + sheet_count: 0, + warnings: { list_columns: 2, dropped_columns: 0, other: 5 }, + dropped: ['span', 7], + }) + ).toEqual({ + text: 'a\n1\n', + format: 'json', + kind: 'parquet', + rows: 1, + columns: 1, + sheet: null, + sheets: ['x'], + sheet_count: 0, + warnings: { list_columns: 2 }, + dropped: ['span'], + }); + }); + + it('refuses an answer without text', () => { + expect(convertAnswer({ kind: 'xlsx' })).toBeNull(); + expect(convertAnswer('text')).toBeNull(); + }); + + it('posts the file as base64 with the chosen sheet', async () => { + const fetchMock = vi.fn( + async () => + new Response(JSON.stringify({ text: 'a\n', kind: 'xlsx' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + vi.stubGlobal('fetch', fetchMock); + const answer = await agentApi.convertFile('t', 'S1', toBase64(new Uint8Array([1, 2, 3])), 'B'); + expect(answer.text).toBe('a\n'); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toContain('/debug/agent/sessions/S1/dataset/convert'); + expect(JSON.parse(String(init.body))).toEqual({ data_base64: 'AQID', sheet: 'B' }); + vi.unstubAllGlobals(); + }); }); describe('caseListQuery', () => { diff --git a/app/src/lib/agent.ts b/app/src/lib/agent.ts index 309addf55e4..480856bcad3 100644 --- a/app/src/lib/agent.ts +++ b/app/src/lib/agent.ts @@ -509,13 +509,17 @@ export class AgentApiError extends Error { * missing, so the page can name it in the next edit's `replaces`; `null` otherwise. */ readonly caseId: string | null; + /** A refused file conversion's measured value and cap (`422 file_too_many_rows` and kin); null otherwise. */ + readonly measured: number | null; + readonly limit: number | null; constructor( status: number, code: string, ref: string | null = null, errors: string[] = [], - caseId: string | null = null + caseId: string | null = null, + measure: { measured?: number | null; limit?: number | null } = {} ) { super(`agent request failed: ${status} ${code}`); this.name = 'AgentApiError'; @@ -524,6 +528,8 @@ export class AgentApiError extends Error { this.ref = ref; this.errors = errors; this.caseId = caseId; + this.measured = measure.measured ?? null; + this.limit = measure.limit ?? null; } /** 401 and 403 from the admin gate: the browser is not (or no longer) signed in as an admin. */ @@ -551,6 +557,7 @@ export async function toAgentApiError(response: Response): Promise= 0; +} + +/** The raw file cap of a conversion: 5 MB, as the BFF and the agents service count it. */ +export const MAX_CONVERT_BYTES = 5 * 1024 * 1024; + +/** What the server counted while it read a file (`agents/renderer/wire.py` `ConvertWarning`). */ +export interface ConvertWarnings { + formulas_without_value?: number; + hidden_sheets?: number; + dropped_columns?: number; + list_columns?: number; + utc_timestamps?: number; + renamed_columns?: number; +} + +/** A converted file: the text for the box, what kind of file it was, the sheet read and the others. */ +export interface ConvertAnswer { + text: string; + /** `json` for a table with list columns: records the parser expands into long format. */ + format: 'csv' | 'json'; + kind: string; + rows: number; + columns: number; + sheet: string | null; + sheets: string[]; + sheet_count: number; + warnings: ConvertWarnings; + /** The names of the columns left out (at most 20); `warnings.dropped_columns` counts all of them. */ + dropped: string[]; +} + +const WARNING_KEYS = [ + 'formulas_without_value', + 'hidden_sheets', + 'dropped_columns', + 'list_columns', + 'utc_timestamps', + 'renamed_columns', +] as const; + +/** The conversion route's answer, checked field by field; null when it is not that shape. */ +export function convertAnswer(value: unknown): ConvertAnswer | null { + if (!value || typeof value !== 'object') return null; + const record = value as Record; + if (typeof record.text !== 'string' || typeof record.kind !== 'string') return null; + const warnings: ConvertWarnings = {}; + const raw = record.warnings; + if (raw && typeof raw === 'object') { + for (const key of WARNING_KEYS) { + const count = (raw as Record)[key]; + if (isCount(count) && count > 0) warnings[key] = count; + } + } + return { + text: record.text, + format: record.format === 'json' ? 'json' : 'csv', + kind: record.kind, + rows: isCount(record.rows) ? record.rows : 0, + columns: isCount(record.columns) ? record.columns : 0, + sheet: typeof record.sheet === 'string' ? record.sheet : null, + sheets: strings(record.sheets).slice(0, 10), + sheet_count: isCount(record.sheet_count) ? record.sheet_count : 0, + warnings, + dropped: strings(record.dropped).slice(0, 20), + }; +} + +/** Base64 of a file's bytes, in chunks, so a 5 MB file never builds one huge argument list. */ +export function toBase64(bytes: Uint8Array): string { + let binary = ''; + const chunk = 0x8000; + for (let index = 0; index < bytes.length; index += chunk) { + binary += String.fromCharCode(...bytes.subarray(index, index + chunk)); + } + return btoa(binary); +} + /** The BFF passes at most this many binding check lines. */ const MAX_BINDING_ERRORS = 20; @@ -679,6 +768,29 @@ export const agentApi = { ...(source ? { source } : {}), }) ), + /** + * A dropped binary data file (Excel, OpenDocument, Parquet, Feather, Arrow, + * NumPy) read on the server, in the renderer's sandbox, into the text the + * paste box takes. Nothing is stored: the page puts `text` into the box and + * the user's `.parse()` sends it through `uploadDataset`. `sheet` asks for + * another sheet (or `.npz` array) of the same file, by a name of `sheets`. + * An answer the page cannot read is `422 invalid`. + */ + convertFile: async ( + token: string, + sid: string, + dataBase64: string, + sheet?: string + ): Promise => { + const answer = await agentJson( + `${sessionPath(sid)}/dataset/convert`, + token, + post({ data_base64: dataBase64, ...(sheet ? { sheet } : {}) }) + ); + const parsed = convertAnswer(answer); + if (!parsed) throw new AgentApiError(422, 'invalid'); + return parsed; + }, putBindings: (token: string, sid: string, bindings: Binding[]) => agentJson(`${sessionPath(sid)}/bindings`, token, { method: 'PUT', diff --git a/app/src/pages/AgentChatPage.tsx b/app/src/pages/AgentChatPage.tsx index 6d72db5d362..ec4967a09b2 100644 --- a/app/src/pages/AgentChatPage.tsx +++ b/app/src/pages/AgentChatPage.tsx @@ -554,6 +554,7 @@ function AgentChat({ specId, library, language, token, onLibraryChange }: AgentC onParse={(text, source) => void session.parseData(text, false, source)} onConfirmTranscription={() => void session.confirmTranscription()} onTrack={trackEvent} + onConvert={session.convertFile} onDiscardTranscription={session.discardTranscription} onSendFormula={() => void session.sendFormula()} onBind={session.setBinding} diff --git a/app/src/sections/agent-chat/DataPanel.test.tsx b/app/src/sections/agent-chat/DataPanel.test.tsx index 4dbb3c2f1e7..024498ddfae 100644 --- a/app/src/sections/agent-chat/DataPanel.test.tsx +++ b/app/src/sections/agent-chat/DataPanel.test.tsx @@ -5,7 +5,7 @@ import { initialAgentState, type TranscriptionState, } from 'src/hooks/useAgentSession'; -import type { RoleSpec } from 'src/lib/agent'; +import { AgentApiError, type ConvertAnswer, type RoleSpec } from 'src/lib/agent'; import { bindingRows } from 'src/sections/agent-chat/bindings'; import { DataPanel } from 'src/sections/agent-chat/DataPanel'; import { fireEvent, render, screen, userEvent, waitFor, within } from 'src/test-utils'; @@ -1093,7 +1093,10 @@ describe('DataPanel file drop', () => { ...overrides, }); - function renderPanel(state: AgentSessionState = emptyState()) { + function renderPanel( + state: AgentSessionState = emptyState(), + onConvert?: (dataBase64: string, sheet?: string) => Promise + ) { const onParse = vi.fn(); const onConfirmTranscription = vi.fn(); const onTrack = vi.fn(); @@ -1108,6 +1111,7 @@ describe('DataPanel file drop', () => { onConfirmData={vi.fn()} onUndo={vi.fn()} onTrack={onTrack} + onConvert={onConvert} /> ); return { ...view, onParse, onConfirmTranscription, onTrack }; @@ -1289,27 +1293,168 @@ describe('DataPanel file drop', () => { expect(screen.getByRole('button', { name: 'Open a data file' })).toBeDisabled(); }); - it('answers a workbook and a pickle by their format', async () => { - const { onTrack } = renderPanel(); - const zip = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0, 0, 0, 0]); + const ZIP = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0, 0, 0, 0]); + const workbook = (name = 'Bericht.xlsx') => new File([ZIP], name); + const answer = (overrides: Partial = {}): ConvertAnswer => ({ + text: 'Report 2024\n\nmonth,revenue\nJan,120\n', + format: 'csv', + kind: 'xlsx', + rows: 4, + columns: 2, + sheet: '2024', + sheets: ['2024', '2025'], + sheet_count: 2, + warnings: {}, + dropped: [], + ...overrides, + }); + + it('sends a workbook to the server as base64 and fills the box with its sheet, the sheet in the source', async () => { + const onConvert = vi.fn(async () => answer({ warnings: { formulas_without_value: 3 } })); + const { onParse, onTrack } = renderPanel(emptyState(), onConvert); + + drop(workbook()); + + await waitFor(() => expect(box()).toHaveValue('Report 2024\n\nmonth,revenue\nJan,120\n')); + expect(onConvert).toHaveBeenCalledWith(btoa(String.fromCharCode(...ZIP)), undefined); + expect(screen.getByTestId('file-status')).toHaveTextContent( + '# read "Bericht.xlsx" on the server: sheet "2024" of xlsx · 4 rows × 2 columns · 3 lines · 3 formula cells saved without a value' + ); + expect(onTrack).toHaveBeenCalledWith('agent_data_file', { + kind: 'xlsx', + encoding: 'none', + outcome: 'filled', + }); + await userEvent.click(screen.getByRole('button', { name: '.parse()' })); + expect(onParse).toHaveBeenCalledWith( + 'Report 2024\n\nmonth,revenue\nJan,120\n', + 'Bericht.xlsx · sheet "2024"' + ); + }); - drop(new File([zip], 'sales.xlsx')); + it('names the columns the server left out', async () => { + const onConvert = vi.fn(async () => + answer({ + sheet: null, + sheets: [], + warnings: { dropped_columns: 3 }, + dropped: ['span', 'raw'], + }) + ); + renderPanel(emptyState(), onConvert); + + drop(workbook()); + + await waitFor(() => + expect(screen.getByTestId('file-status')).toHaveTextContent( + '3 columns of durations, intervals, binary, map or nested values left out: "span", "raw" and 1 more' + ) + ); + }); + + it('offers the other sheets and reads the chosen one again on the server', async () => { + const onConvert = vi.fn(async (_data: string, sheet?: string) => + sheet === '2025' ? answer({ text: 'a,b\n1,2\n', sheet: '2025' }) : answer() + ); + const { onParse, onTrack } = renderPanel(emptyState(), onConvert); + + drop(workbook()); + const picker = (await screen.findByLabelText(/# sheet/)) as HTMLSelectElement; + expect(picker).toHaveValue('2024'); + expect( + within(picker) + .getAllByRole('option') + .map(option => option.textContent) + ).toEqual(['2024', '2025']); + + await userEvent.selectOptions(picker, '2025'); + // The box still held this file's own text, so the new sheet replaces it without a question. + await waitFor(() => expect(box()).toHaveValue('a,b\n1,2\n')); + expect(onConvert).toHaveBeenLastCalledWith(expect.any(String), '2025'); + expect(onTrack).toHaveBeenLastCalledWith('agent_data_file', { + kind: 'xlsx', + encoding: 'none', + outcome: 'sheet', + }); + await userEvent.click(screen.getByRole('button', { name: '.parse()' })); + expect(onParse).toHaveBeenCalledWith('a,b\n1,2\n', 'Bericht.xlsx · sheet "2025"'); + }); + + it('asks before a new sheet replaces text the user edited', async () => { + const onConvert = vi.fn(async (_data: string, sheet?: string) => + sheet ? answer({ text: 'a,b\n1,2\n', sheet }) : answer() + ); + renderPanel(emptyState(), onConvert); + drop(workbook()); + const picker = await screen.findByLabelText(/# sheet/); + await userEvent.type(box(), 'x'); + + await userEvent.selectOptions(picker, '2025'); + + expect(await screen.findByTestId('file-replace')).toHaveTextContent( + 'replace the text in the box with "Bericht.xlsx · sheet "2025"" (2 lines)?' + ); + }); + + it('names the counts of a refused file and keeps the box', async () => { + const onConvert = vi.fn(async () => { + throw new AgentApiError(422, 'file_too_many_rows', 'ref-1', [], null, { + measured: 21000, + limit: 20000, + }); + }); + const { onTrack } = renderPanel(emptyState(), onConvert); + + drop(workbook('big.xlsx')); expect(await screen.findByRole('alert')).toHaveTextContent( - '"sales.xlsx" is a spreadsheet: reading Excel and ODS files comes in a later step' + '"big.xlsx" has more rows than a table may have; filter it, or copy a range and paste it (21,000 rows, the cap is 20,000) · ref ref-1' ); + expect(box()).toHaveValue(''); expect(onTrack).toHaveBeenCalledWith('agent_data_file', { kind: 'spreadsheet', encoding: 'none', outcome: 'refused', }); + }); - drop(new File([new Uint8Array([0x80, 0x05, 0x95])], 'frame.pkl')); + it('waits for a conversion: .parse() and .open() hold, and a second file is refused', async () => { + let finish: (value: ConvertAnswer) => void = () => undefined; + const onConvert = vi.fn( + () => + new Promise(resolve => { + finish = resolve; + }) + ); + renderPanel(emptyState(), onConvert); + + drop(workbook()); await waitFor(() => - expect(screen.getByRole('alert')).toHaveTextContent( - 'never read because loading one can run code' + expect(screen.getByTestId('file-status')).toHaveTextContent( + '# reading "Bericht.xlsx" on the server …' ) ); + expect(screen.getByRole('button', { name: 'Open a data file' })).toBeDisabled(); + + drop(csvFile()); + expect(await screen.findByRole('alert')).toHaveTextContent( + 'a file is still being read on the server' + ); + expect(onConvert).toHaveBeenCalledTimes(1); + + finish(answer()); + await waitFor(() => expect(box()).toHaveValue('Report 2024\n\nmonth,revenue\nJan,120\n')); + }); + + it('answers a pickle by its format, without the server', async () => { + const onConvert = vi.fn(); + renderPanel(emptyState(), onConvert); + + drop(new File([new Uint8Array([0x80, 0x05, 0x95])], 'frame.pkl')); + expect(await screen.findByRole('alert')).toHaveTextContent( + 'never read because loading one can run code' + ); + expect(onConvert).not.toHaveBeenCalled(); expect(box()).toHaveValue(''); }); diff --git a/app/src/sections/agent-chat/DataPanel.tsx b/app/src/sections/agent-chat/DataPanel.tsx index 35e01645c39..1f4171b438b 100644 --- a/app/src/sections/agent-chat/DataPanel.tsx +++ b/app/src/sections/agent-chat/DataPanel.tsx @@ -80,6 +80,18 @@ * request); the data line shows it. Typing keeps the name, a text paste or an * empty box drops it; a table read from the file keeps it for `.use()`. * Several files at once: the first is read, and the line says so. + * + * A binary data file (Excel, OpenDocument, Parquet, Feather, Arrow, NumPy, at + * most 5 MB) goes to the server instead (`onConvert`, `POST .../dataset/convert`), + * which reads it in the renderer's sandbox and answers the text for the box; + * then it fills the box like a text file, and `.parse()` sends that text with + * the name and the sheet as `source` (`Bericht.xlsx · sheet "2024"`). While it + * is read, `.open()`, `.parse()` and the drop zone wait. The line names what + * was read and what the server counted (formula cells without a saved value, + * hidden sheets, columns left out); a refusal names its numbers. A workbook + * with several sheets keeps the file in the page and offers a sheet picker, + * which reads the chosen sheet again: over the file's own untouched text at + * once, over an edited box after `.replace()`. */ import { useEffect, useLayoutEffect, useRef, useState } from 'react'; @@ -96,16 +108,20 @@ import { type TranscriptionState, } from 'src/hooks/useAgentSession'; import { + AgentApiError, type ColumnProfile, + type ConvertAnswer, type LongSuggestion, longSuggestion, MAX_DATASET_BYTES, MAX_MESSAGE_CHARS, + toBase64, utf8Bytes, } from 'src/lib/agent'; import { bindingRows, kindsHint, signHint } from 'src/sections/agent-chat/bindings'; import { Ellipsis } from 'src/sections/agent-chat/Ellipsis'; import { + countLines, FILE_ACCEPT, type FileEncoding, type FileKind, @@ -114,9 +130,12 @@ import { } from 'src/sections/agent-chat/fileText'; import { CONFIRM_LOST_TEXT, + describeConvertFailure, + describeConvertWarnings, describeDataFailure, describeFileRefusal, FILE_BUSY_TEXT, + FILE_CONVERTING_TEXT, } from 'src/sections/agent-chat/messages'; import { inStep2 } from 'src/sections/agent-chat/steps'; import { @@ -601,16 +620,66 @@ function dragHasFiles(transfer: DataTransfer | null): boolean { /** What the server keeps for a file's name its content check refused (`dataset_context.FILE_NAME_WITHHELD`). */ const FILE_NAME_WITHHELD = '(name withheld)'; +/** A binary file the server converted: kept in the browser for the sheet picker, which converts it again. */ +interface ConvertedFile { + name: string; + base64: string; + /** The browser's reading of the format, for the analytics of a failed re-conversion. */ + browserKind: FileKind; + /** The server's reading of the format (`xlsx`, `ods`, `parquet`, ...). */ + kind: string; + sheet: string | null; + sheets: string[]; + sheetCount: number; + /** The text the conversion put into the box, to tell whether the user edited it since. */ + text: string; +} + /** A file read and waiting for the user's `.replace()` because the box holds text. */ interface PendingFile { name: string; - reading: Extract; + /** What travels as the dataset route's `source`: the name, and the sheet for a converted workbook. */ + source: string; + text: string; + lines: number; + /** How it was read, after the name: `as utf-8`, `from the sheet "2024" (xlsx)`. */ + read: string; + /** What the server counted while it converted the file. */ + notes: string[]; /** How many files came with it; only the first is read. */ of: number; + kind: string; + encoding: FileEncoding | null; + converted: ConvertedFile | null; } /** What happened to a dropped, chosen or pasted file, for `agent_data_file`. */ -type FileOutcome = 'filled' | 'replaced' | 'kept' | 'refused' | 'too_large' | 'busy' | 'unreadable'; +type FileOutcome = + | 'filled' + | 'replaced' + | 'kept' + | 'refused' + | 'too_large' + | 'busy' + | 'unreadable' + | 'failed' + | 'sheet'; + +/** `Bericht.xlsx · sheet "2024"`: the source of a converted workbook names the sheet that was read. */ +function convertedSource(name: string, answer: ConvertAnswer): string { + if (answer.sheet === null) return name; + return `${name} · ${answer.kind === 'npz' ? 'array' : 'sheet'} "${answer.sheet}"`; +} + +/** The "read ... " part of the line after a conversion. */ +function convertedRead(answer: ConvertAnswer): string { + const where = + answer.sheet === null + ? '' + : ` ${answer.kind === 'npz' ? 'array' : 'sheet'} "${answer.sheet}" of`; + const shape = `${countOf(answer.rows, 'row')} × ${countOf(answer.columns, 'column')}`; + return `on the server:${where} ${answer.kind} · ${shape}${answer.format === 'json' ? ' as JSON records' : ''}`; +} interface DataPanelProps { state: AgentSessionState; @@ -636,6 +705,11 @@ interface DataPanelProps { decisionRef?: React.Ref; /** Analytics: `agent_data_file{kind, encoding, outcome}`; never the file's name. */ onTrack?: (event: string, props?: Record) => void; + /** + * A dropped binary data file read on the server (`POST .../dataset/convert`); `sheet` + * asks for another sheet of the same file. Without it, such files are refused. + */ + onConvert?: (dataBase64: string, sheet?: string) => Promise; } export function DataPanel({ @@ -652,8 +726,13 @@ export function DataPanel({ confirmRef, decisionRef, onTrack, + onConvert, }: DataPanelProps) { const [text, setText] = useState(''); + // A binary file on its way to the server and back; the box, `.open()` and `.parse()` wait for it. + const [converting, setConverting] = useState(false); + // The converted file whose text is in the box, for the sheet picker; a text paste or an empty box drops it. + const [converted, setConverted] = useState(null); // The name of the file the box's text came from; typing keeps it, a text paste or an empty box drops it. const [source, setSource] = useState(null); const [dragOver, setDragOver] = useState(false); @@ -685,6 +764,8 @@ export function DataPanel({ // A turn, a theme render, a parse, a binding change or a confirmation in // flight: the server would refuse or race a second one. const busy = sessionBusy(state); + // Whether a dropped file would be taken now: not while a step runs or another file is being read. + const dropReady = ready && !busy && !converting; const transcription = state.transcription; const dataset = state.dataset; const columns = dataset?.parsed.profile.columns ?? []; @@ -759,14 +840,11 @@ export function DataPanel({ else onParse(text); }; - const trackFile = ( - kind: FileKind | 'unknown', - encoding: FileEncoding | null, - outcome: FileOutcome - ) => onTrack?.('agent_data_file', { kind, encoding: encoding ?? 'none', outcome }); + const trackFile = (kind: string, encoding: FileEncoding | null, outcome: FileOutcome) => + onTrack?.('agent_data_file', { kind, encoding: encoding ?? 'none', outcome }); // The file's text goes into the box as it is; the user reads it and presses `.parse()`. - const fillFromFile = ({ name, reading, of }: PendingFile, outcome: FileOutcome) => { + const fillFromFile = (pending: PendingFile, outcome: FileOutcome) => { // A table read from the box's earlier text belongs to that text: it goes, so `.use()` // never stores it under this file's name and `.parse()` is free for the new text. The // card stays open, also when dropping the table takes the page back to step 2. @@ -774,22 +852,109 @@ export function DataPanel({ setExpanded(true); onDiscardTranscription(); } - setText(reading.text); - setSource(name); + setText(pending.text); + setSource(pending.source); + setConverted(pending.converted); setPendingFile(null); // Several files dropped at once: the first is read, and the line says so (owner, 2026-10-15). - const several = of > 1 ? `first of ${of} files read — one file at a time · ` : ''; + const several = + pending.of > 1 ? `first of ${pending.of} files read — one file at a time · ` : ''; + const notes = pending.notes.length > 0 ? ` · ${pending.notes.join(' · ')}` : ''; setFileNotice({ tone: 'ok', - text: `${several}read "${name}" as ${reading.encoding} · ${countOf(reading.lines, 'line')} · check it, then .parse()`, + text: `${several}read "${pending.name}" ${pending.read} · ${countOf(pending.lines, 'line')}${notes} · check it, then .parse()`, }); setFills(count => count + 1); - trackFile('text', reading.encoding, outcome); + trackFile(pending.kind, pending.encoding, outcome); + }; + + // Into an empty box, or held for `.replace()` when the box has text the user did not get from this + // very conversion (a sheet switch over the untouched text of the same file replaces it at once). + const offerFile = (pending: PendingFile, outcome: FileOutcome = 'filled') => { + const current = textRef.current; + if (current.trim() && !(outcome === 'sheet' && converted && current === converted.text)) { + setFileNotice(null); + setPendingFile(pending); + return; + } + fillFromFile(pending, outcome); + }; + + // A binary format goes to the server (in the renderer's sandbox) and comes back as text for the box. + const convertOnServer = async ( + file: { name: string; base64: string; browserKind: FileKind }, + of: number, + seq: number, + sheet?: string + ) => { + if (!onConvert) { + setFileNotice({ tone: 'error', text: describeFileRefusal(file.name, 'binary') }); + trackFile(file.browserKind, null, 'refused'); + return; + } + setConverting(true); + setFileNotice({ + tone: 'ok', + text: sheet + ? `reading the sheet "${sheet}" of "${file.name}" on the server …` + : `reading "${file.name}" on the server …`, + }); + let answer: ConvertAnswer; + try { + answer = await onConvert(file.base64, sheet); + } catch (err) { + if (seq !== readSeq.current) return; + const failure = + err instanceof AgentApiError ? err : new AgentApiError(0, 'unreachable', null); + setFileNotice({ + tone: 'error', + text: describeConvertFailure( + file.name, + failure.code, + failure.measured, + failure.limit, + failure.ref + ), + }); + trackFile(file.browserKind, null, failure.code.startsWith('file_') ? 'refused' : 'failed'); + return; + } finally { + if (seq === readSeq.current) setConverting(false); + } + if (seq !== readSeq.current) return; + offerFile( + { + name: file.name, + source: convertedSource(file.name, answer), + text: answer.text, + lines: countLines(answer.text), + read: convertedRead(answer), + notes: describeConvertWarnings(answer.warnings, answer.dropped), + of, + kind: answer.kind, + encoding: null, + converted: { + ...file, + kind: answer.kind, + sheet: answer.sheet, + sheets: answer.sheets, + sheetCount: answer.sheet_count, + text: answer.text, + }, + }, + sheet ? 'sheet' : 'filled' + ); }; // A drop, `.open()` or a pasted file: refused while the session works, read in the // browser, then into an empty box, or held for `.replace()` when the box has text. const handleFile = async (file: File, of = 1) => { + // A file the server still reads keeps its place: the next one is refused, not swapped in. + if (converting) { + setFileNotice({ tone: 'error', text: FILE_CONVERTING_TEXT }); + trackFile('unknown', null, 'busy'); + return; + } // Only the latest file counts: a slow read of an earlier one must not land after it. readSeq.current += 1; const seq = readSeq.current; @@ -809,28 +974,51 @@ export function DataPanel({ return; } if (seq !== readSeq.current) return; + if (!reading.ok && reading.reason === 'convert') { + await convertOnServer( + { name: file.name, base64: toBase64(reading.bytes), browserKind: reading.kind }, + of, + seq + ); + return; + } if (!reading.ok) { const code = reading.reason === 'binary' ? reading.kind : reading.reason; setFileNotice({ tone: 'error', text: describeFileRefusal(file.name, code) }); trackFile( reading.kind, reading.encoding, - reading.reason === 'too_large' ? 'too_large' : 'refused' + reading.reason === 'too_large' || reading.reason === 'too_large_binary' + ? 'too_large' + : 'refused' ); return; } - const pending = { name: file.name, reading, of }; - if (textRef.current.trim()) { - setFileNotice(null); - setPendingFile(pending); - return; - } - fillFromFile(pending, 'filled'); + offerFile({ + name: file.name, + source: file.name, + text: reading.text, + lines: reading.lines, + read: `as ${reading.encoding}`, + notes: [], + of, + kind: 'text', + encoding: reading.encoding, + converted: null, + }); + }; + + // The sheet picker: the same file again, another sheet, read on the server. + const handleSheet = (sheet: string) => { + if (!converted || sheet === converted.sheet || !ready || busy || converting) return; + readSeq.current += 1; + setPendingFile(null); + void convertOnServer(converted, 1, readSeq.current, sheet); }; // `.keep()`: the box stays as it was, and so does the focus's place in it, not on the page. const handleKeep = () => { - if (pendingFile) trackFile('text', pendingFile.reading.encoding, 'kept'); + if (pendingFile) trackFile(pendingFile.kind, pendingFile.encoding, 'kept'); setPendingFile(null); boxRef.current?.focus(); }; @@ -875,7 +1063,7 @@ export function DataPanel({ event.preventDefault(); // Stop here, so the page's guard above does not mark the box as no drop target. event.stopPropagation(); - event.dataTransfer.dropEffect = ready && !busy ? 'copy' : 'none'; + event.dataTransfer.dropEffect = dropReady ? 'copy' : 'none'; }; const handleDragLeave = (event: React.DragEvent) => { @@ -906,9 +1094,10 @@ export function DataPanel({ void handleFile(file); return; } - // Pasted text is not the file's any more: its name, its line and a file waiting for - // `.replace()` (which would overwrite the paste) all go. + // Pasted text is not the file's any more: its name, its line, its sheet picker and a file + // waiting for `.replace()` (which would overwrite the paste) all go. setSource(null); + setConverted(null); setFileNotice(null); setPendingFile(null); }; @@ -928,6 +1117,7 @@ export function DataPanel({ const handleSendFormula = () => { setText(''); setSource(null); + setConverted(null); onSendFormula(); }; @@ -940,7 +1130,8 @@ export function DataPanel({ htmlFor="agent-data" sx={{ ...labelSx, display: 'block', mb: dataset ? 0.75 : 0.25 }} > - # your data — paste or drop CSV, TSV, semicolon CSV, JSON or plain text + # your data — paste or drop CSV, TSV, JSON, plain text, Excel, ODS, Parquet, Feather or + NumPy {/* Until there is data, the other source: a curve the assistant computes. */} {!dataset && ( @@ -956,7 +1147,7 @@ export function DataPanel({ colour, or with the warning colour while a step runs and the drop would be refused. */} ) => { setText(event.target.value); - if (!event.target.value.trim()) setSource(null); + if (!event.target.value.trim()) { + setSource(null); + setConverted(null); + } setFileNotice(null); // An edit after the question wins: `.replace()` would overwrite it. setPendingFile(null); @@ -991,10 +1185,10 @@ export function DataPanel({ whiteSpace: 'pre', overflowX: 'auto', ...(dragOver && { - borderColor: ready && !busy ? colors.primary : colors.warning, + borderColor: dropReady ? colors.primary : colors.warning, borderStyle: 'dashed', - outline: `1px dashed ${ready && !busy ? colors.primary : colors.warning}`, - bgcolor: `color-mix(in srgb, ${ready && !busy ? colors.primary : colors.warning} 10%, var(--bg-elevated))`, + outline: `1px dashed ${dropReady ? colors.primary : colors.warning}`, + bgcolor: `color-mix(in srgb, ${dropReady ? colors.primary : colors.warning} 10%, var(--bg-elevated))`, }), }} /> @@ -1020,15 +1214,17 @@ export function DataPanel({ fontSize: fontSize.md, color: 'var(--ink)', bgcolor: 'var(--bg-elevated)', - border: `1px dashed ${ready && !busy ? colors.primary : colors.warning}`, + border: `1px dashed ${dropReady ? colors.primary : colors.warning}`, borderRadius: '4px', px: 1.5, py: 0.75, }} > - {ready && !busy + {dropReady ? 'drop the file to read it into the box' - : 'a step is still running: drop the file when it is done'} + : converting + ? 'a file is still being read: drop the next one when it is done' + : 'a step is still running: drop the file when it is done'} )} @@ -1057,7 +1253,7 @@ export function DataPanel({ component="button" type="button" onClick={() => fileInputRef.current?.click()} - disabled={!ready || busy} + disabled={!ready || busy || converting} aria-label="Open a data file" sx={actionButtonSx} > @@ -1079,7 +1275,7 @@ export function DataPanel({ type="button" ref={parseButtonRef} onClick={handleParse} - disabled={!ready || !text.trim() || over || busy || !!transcription} + disabled={!ready || !text.trim() || over || busy || converting || !!transcription} aria-describedby={transcription || state.parsing ? 'agent-parse-hint' : undefined} sx={ghostButtonSx} > @@ -1109,6 +1305,37 @@ export function DataPanel({ {fileNotice.text} )} + {/* The other sheets of the workbook in the box (or arrays of an .npz): a choice reads the + same file again on the server. Only the visible sheets are named, at most ten. */} + {converted && converted.sheets.length > 1 && ( + + + # {converted.kind === 'npz' ? 'array' : 'sheet'} + {converted.sheetCount > converted.sheets.length + ? ` (first ${converted.sheets.length} of ${converted.sheetCount})` + : ` (${converted.sheets.length})`} + + ) => + handleSheet(event.target.value) + } + sx={{ ...nativeControlSx, maxWidth: '100%', minWidth: 0 }} + > + {converted.sheets.map(name => ( + + ))} + + + )} {pendingFile && ( - replace the text in the box with "{pendingFile.name}" ( - {countOf(pendingFile.reading.lines, 'line')})? + replace the text in the box with "{pendingFile.source}" ( + {countOf(pendingFile.lines, 'line')})? { }); }); + it.each([ + [bytesOf([0x50, 0x4b, 0x03, 0x04], 'rest'), 'sales.xlsx', 'spreadsheet'], + [bytesOf('PAR1', [0x15]), 'data.parquet', 'parquet'], + [bytesOf('FEA1', [0, 0]), 'old.feather', 'arrow'], + [bytesOf([0xff, 0xff, 0xff, 0xff, 0x10]), 'rows.arrows', 'arrow'], + [bytesOf([0x93], 'NUMPY', [0x01, 0x00]), 'a.npy', 'numpy'], + ])('hands %#: %s to the server with its bytes', (bytes, name, kind) => { + const reading = readBytes(bytes, name); + expect(reading).toMatchObject({ ok: false, reason: 'convert', kind }); + expect(reading.ok === false && reading.reason === 'convert' && reading.bytes).toEqual(bytes); + }); + it('refuses text with NUL bytes or with more than 1 % control characters', () => { expect(readBytes(bytesOf('abc', [0x00, 0x00, 0x00], 'def'))).toMatchObject({ ok: false, @@ -182,10 +194,26 @@ describe('readFile', () => { expect(whole).toHaveBeenCalledTimes(1); }); - it('answers a large workbook by its format, not its size', async () => { + it('reads a workbook between the text cap and the conversion cap whole, for the server', async () => { const workbook = new File([bytesOf([0x50, 0x4b, 0x03, 0x04]), new Uint8Array(16)], 'big.xlsx'); Object.defineProperty(workbook, 'size', { value: MAX_FILE_BYTES + 1 }); - await expect(readFile(workbook)).resolves.toMatchObject({ ok: false, kind: 'spreadsheet' }); + await expect(readFile(workbook)).resolves.toMatchObject({ + ok: false, + reason: 'convert', + kind: 'spreadsheet', + }); + }); + + it('refuses a workbook over the conversion cap before reading it whole', async () => { + const workbook = new File([bytesOf([0x50, 0x4b, 0x03, 0x04]), new Uint8Array(16)], 'huge.xlsx'); + Object.defineProperty(workbook, 'size', { value: MAX_CONVERT_BYTES + 1 }); + const read = vi.spyOn(workbook, 'arrayBuffer'); + await expect(readFile(workbook)).resolves.toMatchObject({ + ok: false, + reason: 'too_large_binary', + kind: 'spreadsheet', + }); + expect(read).not.toHaveBeenCalled(); }); it('refuses a text file over the raw cap before reading it', async () => { diff --git a/app/src/sections/agent-chat/fileText.ts b/app/src/sections/agent-chat/fileText.ts index e84bceeda3b..0d50ab3cc8a 100644 --- a/app/src/sections/agent-chat/fileText.ts +++ b/app/src/sections/agent-chat/fileText.ts @@ -6,9 +6,11 @@ * with its size cap, control-character refusal, pre-filter and judges). * * The order: the raw size against `MAX_FILE_BYTES` before anything is read; - * then the magic bytes of the binary formats people drop (spreadsheets, - * Parquet, Arrow, NumPy, HDF5, pickle, PDF, archives), each with its own - * answer; then the text encoding: a byte order mark (UTF-8, UTF-16LE as in + * then the magic bytes of the binary formats people drop. Spreadsheets, + * Parquet, Arrow and NumPy (`CONVERTIBLE_KINDS`) are read whole, up to + * `MAX_CONVERT_BYTES`, and go to the server, which converts them in the + * renderer's sandbox and finds the format again from the bytes; HDF5, pickle, + * PDF and archives each get their own answer. Then the text encoding: a byte order mark (UTF-8, UTF-16LE as in * Excel's "Unicode text" export, UTF-16BE), NUL bytes on every other byte as * UTF-16 without one, strict UTF-8, else windows-1252. Not ISO-8859-1: that * would turn the bytes 0x80–0x9F (`€`, `„`, `’` in windows-1252) into C1 @@ -18,7 +20,7 @@ * against the paste cap (`MAX_DATASET_BYTES`). */ -import { MAX_DATASET_BYTES, utf8Bytes } from 'src/lib/agent'; +import { MAX_CONVERT_BYTES, MAX_DATASET_BYTES, utf8Bytes } from 'src/lib/agent'; /** The largest file the browser reads at all: UTF-16 doubles the bytes of a 1 MB text, and a margin. */ export const MAX_FILE_BYTES = 4 * 1024 * 1024; @@ -39,11 +41,18 @@ export type FileKind = export type FileEncoding = 'utf-8' | 'utf-8-bom' | 'utf-16le' | 'utf-16be' | 'windows-1252'; /** Why a file did not go into the box. */ -export type FileRefusal = 'binary' | 'too_large' | 'empty'; +export type FileRefusal = 'binary' | 'too_large' | 'too_large_binary' | 'empty'; + +/** The binary formats the server reads (`POST .../dataset/convert`, in the renderer's sandbox). */ +export type ConvertibleKind = 'spreadsheet' | 'parquet' | 'arrow' | 'numpy'; + +export const CONVERTIBLE_KINDS: readonly FileKind[] = ['spreadsheet', 'parquet', 'arrow', 'numpy']; export type FileReading = | { ok: true; text: string; encoding: FileEncoding; lines: number } - | { ok: false; reason: FileRefusal; kind: FileKind; encoding: FileEncoding | null }; + | { ok: false; reason: FileRefusal; kind: FileKind; encoding: FileEncoding | null } + /** A binary format the server converts: the bytes go there, the text comes back for the box. */ + | { ok: false; reason: 'convert'; kind: ConvertibleKind; encoding: null; bytes: Uint8Array }; /** * The file picker's filter: MIME types first (iOS and Android filter by them), @@ -59,6 +68,13 @@ export const FILE_ACCEPT = [ 'text/plain', 'application/json', 'application/vnd.ms-excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.ms-excel.sheet.macroEnabled.12', + 'application/vnd.ms-excel.sheet.binary.macroEnabled.12', + 'application/vnd.oasis.opendocument.spreadsheet', + 'application/vnd.apache.parquet', + 'application/vnd.apache.arrow.file', + 'application/vnd.apache.arrow.stream', '.csv', '.tsv', '.tab', @@ -66,6 +82,17 @@ export const FILE_ACCEPT = [ '.json', '.dat', '.psv', + '.xlsx', + '.xlsm', + '.xlsb', + '.xls', + '.ods', + '.parquet', + '.feather', + '.arrow', + '.ipc', + '.npy', + '.npz', ].join(','); const ascii = (text: string) => Array.from(text, char => char.charCodeAt(0)); @@ -76,6 +103,8 @@ const MAGIC: ReadonlyArray = [ [[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1], 'binary'], // OLE: xls, doc [ascii('PAR1'), 'parquet'], [ascii('ARROW1'), 'arrow'], // Arrow IPC file, Feather v2 + [ascii('FEA1'), 'arrow'], // Feather v1 + [[0xff, 0xff, 0xff, 0xff], 'arrow'], // Arrow IPC stream (a continuation marker; never valid text) [[0x93, ...ascii('NUMPY')], 'numpy'], [[0x89, ...ascii('HDF\r\n'), 0x1a, 0x0a], 'hdf5'], [ascii('%PDF'), 'pdf'], @@ -189,8 +218,18 @@ export function countLines(text: string): number { * `\n` once here, as a textarea holds them: the box, its byte counter, the * line count and the text `.parse()` sends then agree. */ +export function isConvertible(kind: FileKind | null): kind is ConvertibleKind { + return kind !== null && CONVERTIBLE_KINDS.includes(kind); +} + export function readBytes(bytes: Uint8Array, name = '', type = ''): FileReading { const kind = sniffBinary(bytes, name, type); + if (isConvertible(kind)) { + if (bytes.length > MAX_CONVERT_BYTES) { + return { ok: false, reason: 'too_large_binary', kind, encoding: null }; + } + return { ok: false, reason: 'convert', kind, encoding: null, bytes }; + } if (kind) return { ok: false, reason: 'binary', kind, encoding: null }; const decoded = decodeText(bytes); if (!decoded || looksBinary(decoded.text)) { @@ -225,6 +264,11 @@ export async function readFile(file: File): Promise { if (file.size > MAX_FILE_BYTES) { const head = new Uint8Array(await fileBytes(file.slice(0, 16))); const kind = sniffBinary(head, file.name, file.type); + // A binary format the server reads takes up to its own cap; past it the file is not read at all. + if (isConvertible(kind) && file.size <= MAX_CONVERT_BYTES) { + return readBytes(new Uint8Array(await fileBytes(file)), file.name, file.type); + } + if (isConvertible(kind)) return { ok: false, reason: 'too_large_binary', kind, encoding: null }; if (kind) return { ok: false, reason: 'binary', kind, encoding: null }; return { ok: false, reason: 'too_large', kind: 'text', encoding: null }; } diff --git a/app/src/sections/agent-chat/messages.ts b/app/src/sections/agent-chat/messages.ts index a7ebc1060b4..c1e63b23cb6 100644 --- a/app/src/sections/agent-chat/messages.ts +++ b/app/src/sections/agent-chat/messages.ts @@ -110,32 +110,136 @@ export const IMAGE_FAILURE: Record = { /** * Why a dropped, chosen or pasted file did not go into the data box, by the * format its first bytes name (`fileText.ts`) or the refusal. Spreadsheets, - * Parquet, Arrow and NumPy are a later step; pickle and HDF5 are never read. + * Parquet, Arrow and NumPy go to the server (`CONVERT_FAILURE` below); pickle + * and HDF5 are never read. */ export const FILE_REFUSAL: Record = { - spreadsheet: - 'is a spreadsheet: reading Excel and ODS files comes in a later step; for now, copy the cells and paste them here', - parquet: - 'is a Parquet file: reading Parquet comes in a later step; for now, export it as CSV and drop that', - arrow: - 'is an Arrow or Feather file: reading those comes in a later step; for now, export it as CSV and drop that', pickle: 'is a pickle file, which is never read because loading one can run code; export it as CSV or Parquet', hdf5: 'is an HDF5 file, which is not read here; export it as CSV or Parquet', - numpy: - 'is a NumPy file: reading .npy and .npz files comes in a later step; for now, export it as CSV and drop that', pdf: 'is a PDF, which is not read here; copy the table in it and paste the text', - archive: 'is an archive (zip, gzip or 7z); unpack it and drop the CSV inside', - binary: 'is not a text file; drop a CSV, TSV, JSON or plain-text file', + archive: 'is an archive (zip, gzip or 7z); unpack it and drop the file inside', + binary: + 'is not a format read here; drop a CSV, TSV, JSON, Excel, ODS, Parquet, Feather, Arrow or NumPy file', too_large: 'is more than 1 MB as text; drop a smaller file or paste fewer rows', + too_large_binary: + 'is more than 5 MB; save a smaller file, or copy the cells you need and paste them', empty: 'is empty', unreadable: 'could not be read by the browser; try again', }; +/** Why the server did not convert a dropped binary file (`422 file_*` and the route's other errors). */ +export const CONVERT_FAILURE: Record = { + file_unsupported: + 'is not a format the server reads; save it as CSV, Excel (.xlsx), ODS or Parquet', + file_encrypted: + 'is protected by a password; remove the password (in Excel: File › Info › Protect Workbook), save it and drop it again', + file_pickle: + 'is a pickle file, which is never read because loading one can run code; export it as CSV or Parquet', + file_hdf5: 'is an HDF5 file, which is not read here; export it as CSV or Parquet', + file_corrupt: 'could not be read: the file looks damaged, or it is not the format it seems to be', + file_archive_limits: + 'unpacks to more than the server reads (100 MB, at most 250 times its own size); save a smaller file', + file_too_many_rows: + 'has more rows than a table may have; filter it, or copy a range and paste it', + file_too_many_columns: + 'has more columns than a table may have; copy a range of fewer columns and paste it', + file_too_large: 'gives more than 1 MB of text; filter it, or copy a range and paste it', + file_empty: 'has no cells with values', + file_sheet_not_found: 'no longer has that sheet; drop the file again', + file_array_shape: 'holds no 2-D or structured array; save the data as a 2-D array or a table', + file_formulas_only: + 'contains only formulas without saved values; open it in Excel and save it once', + file_object_array: + 'holds a pickled object array, which is never read because loading one can run code; save numbers or text', + file_timeout: 'took longer than the 10 seconds a file may take to read; save a smaller file', + file_too_complex: 'needed more memory than the reader may use; save a smaller file', + file_unreadable: 'could not be read on the server; save it again or export it as CSV', + convert_unavailable: + 'cannot be read right now, the file reader is not available; export it as CSV for now', + rate_limited: 'could not be read: too many files in a short time; wait a minute and try again', + daily_limit: "could not be read: today's limit of files is reached; try again tomorrow", + locked_out: 'could not be read: paused until tomorrow after several refused requests today', + session_closed: 'could not be read: this chat is closed; reload the page to start a new session', + capacity: 'could not be read: the server is busy; try again in a minute', + run_active: 'could not be read: another file is still being read; wait for it', + too_long: 'is more than 5 MB; save a smaller file', +}; + +/** The line for a file the server did not convert, the name in quotes first; a broken cap names its numbers. */ +export function describeConvertFailure( + name: string, + code: string, + measured: number | null, + limit: number | null, + ref: string | null +): string { + let text = CONVERT_FAILURE[code] ?? DATA_FAILURE[code] ?? `could not be read (${code})`; + const counted = (noun: string) => + measured !== null && limit !== null + ? ` (${measured.toLocaleString('en-US')} ${noun}, the cap is ${limit.toLocaleString('en-US')})` + : ''; + if (code === 'file_too_many_rows') text += counted('rows'); + if (code === 'file_too_many_columns') text += counted('columns'); + if (code === 'file_too_large') text += counted('bytes'); + return `"${name}" ${text}${ref ? ` · ref ${ref}` : ''}`; +} + +/** What the server counted while it read a file, as the parts of the line under the box. */ +export function describeConvertWarnings( + warnings: { + formulas_without_value?: number; + hidden_sheets?: number; + dropped_columns?: number; + list_columns?: number; + utc_timestamps?: number; + renamed_columns?: number; + }, + dropped: string[] = [] +): string[] { + const parts: string[] = []; + const count = (value: number, noun: string) => + `${value.toLocaleString('en-US')} ${noun}${value === 1 ? '' : 's'}`; + if (warnings.utc_timestamps) { + parts.push(`${count(warnings.utc_timestamps, 'time-zone column')} written in UTC`); + } + if (warnings.renamed_columns) { + parts.push(`${count(warnings.renamed_columns, 'repeated column name')} given a _2 suffix`); + } + if (warnings.formulas_without_value) { + parts.push( + `${count(warnings.formulas_without_value, 'formula cell')} saved without a value, left empty: open and save the file in Excel to store them` + ); + } + if (warnings.hidden_sheets) { + parts.push(`${count(warnings.hidden_sheets, 'hidden sheet')} skipped`); + } + if (warnings.dropped_columns) { + // The names, because a step that changes the data says what it changed (owner rule, 2026-10-15). + const more = warnings.dropped_columns - dropped.length; + const named = dropped.length + ? `: ${dropped.map(name => `"${name}"`).join(', ')}${more > 0 ? ` and ${more} more` : ''}` + : ''; + parts.push( + `${count(warnings.dropped_columns, 'column')} of durations, intervals, binary, map or nested values left out${named}` + ); + } + if (warnings.list_columns) { + parts.push( + `${count(warnings.list_columns, 'list column')}: the table comes as JSON records, and .parse() gives one row per value` + ); + } + return parts; +} + /** A file dropped or chosen while the session works on something: the server runs one thing at a time. */ export const FILE_BUSY_TEXT = 'a step is still running; drop or open the file again when it is done'; +/** A file dropped or chosen while the server still reads the one before. */ +export const FILE_CONVERTING_TEXT = + 'a file is still being read on the server; drop or open the next one when it is done'; + /** The line under the box for a refused file, the name in quotes first. */ export function describeFileRefusal(name: string, code: string): string { return `"${name}" ${FILE_REFUSAL[code] ?? FILE_REFUSAL.binary}`; diff --git a/changelog.d/agent-file-drop-binary.md b/changelog.d/agent-file-drop-binary.md new file mode 100644 index 00000000000..726d6125603 --- /dev/null +++ b/changelog.d/agent-file-drop-binary.md @@ -0,0 +1,33 @@ +### Added + +- **Excel, OpenDocument, Parquet, Feather, Arrow and NumPy files go into the agent chat's data box.** + A dropped, chosen or pasted binary file (at most 5 MB) goes to the new + `POST .../dataset/convert` route, which reads it in the renderer's sandbox + (`harness.py --convert`, the guard armed after the readers are loaded) and + answers the text the paste box takes; nothing is stored, and the user's + `.parse()` sends the text through the ordinary paste route with the file name + and sheet as `source`. The format comes from the first bytes: workbooks + through calamine, which reads stored values only (no formula, link or macro + runs; a formula without a saved value is empty and counted, a sheet of only + such formulas is refused with "open it in Excel and save it once"), Parquet + and Arrow through pyarrow (row groups counted first, at most 20,001 rows + read; slimmed in the image from 163 MB to 99 MB), NumPy with + `allow_pickle=False`; every zip is checked from its directory and inflated to + prove its declared sizes, an xlsx sheet's cell span is measured before it is + loaded, an encrypted workbook gets "remove the password", and pickle, joblib + and HDF5 are never read. Values keep their precision: timestamps every + fraction digit and their offset, decimals and 64-bit integers exactly, and + dropped columns (durations, intervals, binary) are named. A sheet comes as + its raw cell grid, so a title above the header stays a note; a workbook with + several sheets gets a sheet picker (empty sheets left out) that reads the + chosen one again; a table with list columns comes as JSON records, which the + parser expands into one row per value. Over a cap (20,000 rows, 50 columns, + 1 MB of text) the file is refused with its numbers. Conversions have their own + limits (5 a minute, 30 a day per user, one at a time, 10 s) and wait behind + every pipeline render. (#12194) +- **The data parser reads ISO timestamps with fractions and UTC offsets as datetimes.** + Up to nine fraction digits and an offset (`Z`, `+01`, `+0100`, `+01:00`) are + kept as written, so the plot's loader reads the column as zone-aware + timestamps with full precision; a column with several offsets is written in + UTC with a note, and `prepare_data` keeps fractions and offsets through its + changes. (#12194) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index c6997d546d8..7f62b446f06 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -84,7 +84,7 @@ Deviations from the first sketch, each evidence-backed: there is no data-intake 1. The `.adapt()` overlay button in `SpecDetailView` does a full navigation to `/debug/agent?spec=&library=&language=`. 2. `POST /debug/agent/sessions {spec_id, library, locale}`. The BFF validates the spec and library against `core/constants.py` and the enabled libraries from anyplot-agents `GET /v1/status` (cached), loads a catalogue snapshot through `core/catalogue/queries.py` (title, description, data roles, notes, noqa-stripped code, library version), derives the user id, and forwards it. anyplot-agents normalises the code, runs the readiness scan and the SECURITY validator, and answers `422 not_eligible` for blocked pairs (the 13 map specs, validator failures, files without a `THEME` block or with a non-literal savefig target). anyplot-agents has no database access in phase 1. `GET /debug/agent/eligibility?spec=&library=` gives the button the same answer. `locale` is the browser's locale, recorded with a feedback case; it sets no language, and every session starts in English (see [Session language](#session-language)). -3. `POST .../dataset {text}` runs the deterministic parse and returns a 20-row preview, the `DatasetProfile`, default bindings and warnings. A deterministic pre-filter over every header and cell for text addressed to an AI, then the dataset judge with a data rubric, refuse instruction-like datasets. Both look for injection only: names, postal and e-mail addresses, phone numbers, web addresses and amounts are the user's data and plot like any other value. The judge sees every header and the same five sample rows and five top values the root and the adapter get, with cells in full, in parts of about 4,000 characters, one judge call per part (one part for every eval fixture). When the paste carried notes around the table (`context_lines`, see [Parse](#parse)), they go through the same pre-filter and then, after the dataset judge passed the table, one scope-judge call with the scope rubric, the lines fenced as ``. A pre-filter hit, a verdict other than `in_scope` or no verdict drops every note with a warning and stores the table all the same; no verdict on notes is a strike, because a hidden injection in a copied web table is not the user's doing, and the judge's tokens are booked to the user like the dataset judge's (the `notes_judge` attribution line carries the verdict and the counts, never the text). The route answers the kept notes as `context_lines`, the stored dataset keeps them, and every version that `prepare_data`, `generate_data` or `.to_long()` makes from it carries them on; a new paste replaces them. Judging the notes at the paste rather than in the turn that delivers them keeps every place the root can read them (the summary turn, `get_dataset_profile`) behind one verdict, and lets the data panel show exactly what the assistant got (`# notes sent to the assistant`). No other LLM call happens for a table. Text that is no table the parser can read (prose such as "a bar chart with dog 7, cat 3 and bird 4", a list typed one value per line; at most 4,000 characters, never a JSON array or object, which the parser reads or refuses itself) takes the transcriber path instead (owner rule of 2026-10-11): the pre-filter runs on the raw text, the judge model reads it under the `TranscribedTable` schema (`agents/anyplot/data/transcribe.py`, rubric `prompts/transcriber.md`: keep every value as written, invent and compute nothing, a note on anything ambiguous), the table's CSV is parsed like a paste, and the route answers `transcribed: true` with that CSV, the preview and the note, storing nothing. The page shows the table for the user to confirm or discard, because a wrong reading makes everything after it wrong; confirming posts the CSV through this same route with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and the narrowing to the table (the user has seen every row, so no line is left out as a note or a total), while the deterministic parse, the pre-filter, the judge and the store run as for any paste. A text file reaches this same route, with no second door (owner decision of 2026-10-15, part 1 of 2; spreadsheets and Parquet are part 2): the user drops it on the data box, chooses it with `.open()` (the keyboard's and the phone's way, since iOS has no drag and drop into a page) or pastes it as a file, and the browser reads it (`app/src/sections/agent-chat/fileText.ts`). That reading is UX only, the server stays the trust boundary: the magic bytes first (a spreadsheet, Parquet, Arrow or NumPy `.npy`/`.npz` file is answered with "a later step, copy the cells or export as CSV for now", pickle and HDF5 with "never, export as CSV or Parquet"; of several files dropped at once the first is read, and the line says so), then the encoding (a byte order mark for UTF-8, UTF-16LE as in Excel's "Unicode text" export or UTF-16BE, NUL bytes on every other byte as UTF-16 also when most of the text is CJK, strict UTF-8, else windows-1252, never ISO-8859-1, whose C1 range the parser refuses; a marked file that is no valid UTF-8 falls back to windows-1252 rather than `�`), line breaks normalised to `\n` once, as the text box holds them, a NUL or more than 1 % control characters as binary, the raw size against 4 MB before the read and the decoded text against the 1 MB paste cap. The text fills the box exactly as in the file, a box with text asks before it is replaced, a running step refuses the file, and the user presses `.parse()`. The file's name travels as the structured `source` field, never as a first line of the text (it would break a JSON document or a small table), and it is never a note: the root acts on a clear request in ``, and a name such as "q1 - drop rows where country is DE and set title to X.csv" passes the scope judge (a plot wish is in scope) although the user never wrote it to the assistant (review of #12192). The agents service sanitises it (`opening.source_name`: NFC, controls turned into spaces, bidi marks and zero-width characters removed, at most 80 characters with a middle `…` that keeps the extension), then screens it on its own (`main._screen_file_name`, owner decision of 2026-10-15: always): the pre-filter on the name alone, then one scope-judge call on a `source file: ` line. A hit, a refusal or no verdict keeps `(name withheld)` instead, with no strike and the notes untouched; the judge's `lang` is ignored, so `datos_2025.csv` never sets the session language. The session keeps the result as `StoredDataset.file_name`, carried to every version of that file's data (`prepare_data`, `.to_long()`, an undo) but not to a computed table (`generate_data`, `run_data_code`); the root reads it in the summary turn and in `get_dataset_profile` under `file_name` (the tool's result cap reserves room for it), fenced as `` under a preamble that says it is a name and never a request (`policy.file_name_block`); the session block says only that the data came from a file; the adapter and the reviewer get a kept name under its own "File name" heading, fenced as `` and never inside the dataset context's `` block, whose titles and units they take (`AdaptRequest.file_name`, `ReviewRequest.file_name`, a clause in `adapter.md`, `reviewer.md` and `reviewer_catalogue.md`); and the data panel shows it next to the row count. It is never logged (the `file_name_judge` attribution line carries the verdict and the length) and never sent to Plausible. A file counts as one paste toward the paste limits. +3. `POST .../dataset {text}` runs the deterministic parse and returns a 20-row preview, the `DatasetProfile`, default bindings and warnings. A deterministic pre-filter over every header and cell for text addressed to an AI, then the dataset judge with a data rubric, refuse instruction-like datasets. Both look for injection only: names, postal and e-mail addresses, phone numbers, web addresses and amounts are the user's data and plot like any other value. The judge sees every header and the same five sample rows and five top values the root and the adapter get, with cells in full, in parts of about 4,000 characters, one judge call per part (one part for every eval fixture). When the paste carried notes around the table (`context_lines`, see [Parse](#parse)), they go through the same pre-filter and then, after the dataset judge passed the table, one scope-judge call with the scope rubric, the lines fenced as ``. A pre-filter hit, a verdict other than `in_scope` or no verdict drops every note with a warning and stores the table all the same; no verdict on notes is a strike, because a hidden injection in a copied web table is not the user's doing, and the judge's tokens are booked to the user like the dataset judge's (the `notes_judge` attribution line carries the verdict and the counts, never the text). The route answers the kept notes as `context_lines`, the stored dataset keeps them, and every version that `prepare_data`, `generate_data` or `.to_long()` makes from it carries them on; a new paste replaces them. Judging the notes at the paste rather than in the turn that delivers them keeps every place the root can read them (the summary turn, `get_dataset_profile`) behind one verdict, and lets the data panel show exactly what the assistant got (`# notes sent to the assistant`). No other LLM call happens for a table. Text that is no table the parser can read (prose such as "a bar chart with dog 7, cat 3 and bird 4", a list typed one value per line; at most 4,000 characters, never a JSON array or object, which the parser reads or refuses itself) takes the transcriber path instead (owner rule of 2026-10-11): the pre-filter runs on the raw text, the judge model reads it under the `TranscribedTable` schema (`agents/anyplot/data/transcribe.py`, rubric `prompts/transcriber.md`: keep every value as written, invent and compute nothing, a note on anything ambiguous), the table's CSV is parsed like a paste, and the route answers `transcribed: true` with that CSV, the preview and the note, storing nothing. The page shows the table for the user to confirm or discard, because a wrong reading makes everything after it wrong; confirming posts the CSV through this same route with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and the narrowing to the table (the user has seen every row, so no line is left out as a note or a total), while the deterministic parse, the pre-filter, the judge and the store run as for any paste. A binary data file reaches it the same way, through one pre-process (owner decision of 2026-10-15, part 2 of 2): Excel (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`), OpenDocument (`.ods`), Parquet, Feather (v1 and v2), Arrow IPC (file and stream) and NumPy (`.npy`, `.npz`) go as base64 in JSON (the BFF's CSRF guard takes JSON only; at most 5 MB raw, the BFF refuses a larger body by its `Content-Length` before reading it) to `POST /v1/sessions/{sid}/dataset/convert`, which converts the file in the renderer's sandbox and answers the text the paste box takes; nothing is stored and no model reads anything, so the text then goes through the ordinary paste route above (its limits, pre-filter and judges) when the user presses `.parse()`, with the file's name and sheet as `source` (`Bericht.xlsx · sheet "2024"`), screened as above. The renderer runs the harness in convert mode (`harness.py --convert input.bin`, `POST /convert`): like data mode it reads the bytes, loads the converter (`agents/anyplot/render/convert.py`, copied next to the harness in the image) and imports every reader it needs before it arms its guard (`ConvertGuard`: `DataGuard` with no code to run at all, but the one `compile` NumPy's `.npy` header reader makes to parse its dict literal), then parses from memory with the guard armed and writes `out.csv`, `meta.json` and `status.json`; a refusal is an exit code (`wire.CONVERT_REASONS`) with the measured value and the cap. The format comes from the first bytes, never from the name: every zip is checked from its central directory first (at most 2,000 entries, 100 MB declared uncompressed, 250 times the archive's size, no encrypted member) and every member is inflated, bounded by its declared size, to prove that size, because calamine's reader trusts none (security review of #12194); every pattern over file content is linear (the review's 30 MB run of `