feat(agents): read dropped Excel, ODS, Parquet, Feather, Arrow and NumPy files

.github/workflows/ci-image.yml

@@ -535,7 +535,7 @@ jobs:
           # What the image is built from: its Dockerfile, the service package,
           # the harness it copies, the dependency lock, README.md (copied next
           # to pyproject.toml) and the root .dockerignore (the build context).
-          RENDERER_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(agents/renderer/|agents/__init__\.py$|agents/anyplot/render/harness\.py$|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|\.github/workflows/ci-image\.yml$)' || true)
+          RENDERER_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(agents/renderer/|agents/__init__\.py$|agents/anyplot/render/harness\.py$|agents/anyplot/render/convert\.py$|tests/image/|tests/unit/agents/runtime/convert_fixtures\.py$|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|\.github/workflows/ci-image\.yml$)' || true)
 
           if [[ "$EVENT_NAME" == "workflow_dispatch" && "$FORCE_RUN" == "true" ]]; then
             echo "Manual trigger with force_run=true, will build the renderer image"
@@ -610,6 +610,19 @@ jobs:
           EOF
           tail -n 1 harness.out
 
+      # The file conversion inside the image (harness.py --convert): every format the
+      # page sends, with the image's slimmed pyarrow (owner decision of 2026-10-15),
+      # its calamine and NumPy, the guard's preload, and the sandbox's environment.
+      # A file the Dockerfile's slimming removed but a reader needs fails here.
+      - name: Convert every data format through the harness
+        if: steps.check.outputs.should_build == 'true'
+        run: |
+          set -euo pipefail
+          docker run --rm --network none --read-only --tmpfs /tmp \
+            --cap-drop ALL --security-opt no-new-privileges --user "$(id -u):$(id -g)" \
+            -v "$PWD:/repo:ro" \
+            anyplot-renderer:ci /app/.venv/bin/python -I /repo/tests/image/convert_smoke.py
+
       # The fallback fonts: the harness appends them after the font the code chose,
       # so this fixture does what catalogue code does to the font setup (seaborn's
       # rcParams reset, a bold title, a monospace text) and draws seven of the
@@ -689,6 +702,7 @@ jobs:
         run: |
           set -euo pipefail
           echo "image size: $(docker image inspect --format '{{.Size}}' anyplot-renderer:ci) bytes"
+          docker run --rm anyplot-renderer:ci du -sh /app/.venv/lib/python3.13/site-packages/pyarrow
           docker run --rm anyplot-renderer:ci sh -c 'grep PRETTY_NAME /etc/os-release; cat /usr/share/doc/fonts-noto/versions.txt; ls -l /usr/share/fonts/noto; du -sh /usr/share/fonts/noto; head -n 5 /usr/share/doc/fonts-noto/fonts-noto-core.copyright'
 
       # Also after a failed assert, so the PNGs show what went wrong.

agents/README.md

@@ -92,6 +92,10 @@ Three rules hold for everything here:
 | `AGENT_SESSION_CLOSE_WARNING` | `true` | The counted refusal one before the closing one ends with the fixed `close_warning` sentence |
 | `AGENT_USER_PASTES_PER_MINUTE` | `10` | Dataset pastes per user within any 60 seconds, across sessions; one more is `429 rate_limited`. Only a paste that reaches the pre-filter or a judge counts |
 | `AGENT_USER_PASTES_PER_DAY` | `100` | Dataset pastes per user and UTC day; one more is `429 daily_limit` |
+| `AGENT_USER_CONVERSIONS_PER_MINUTE` | `5` | File conversions (a dropped spreadsheet, Parquet, Arrow or NumPy file read in the renderer's sandbox) per user within any 60 seconds; one more is `429 rate_limited`. The paste that follows counts as one paste |
+| `AGENT_USER_CONVERSIONS_PER_DAY` | `30` | File conversions per user and UTC day; one more is `429 daily_limit` |
+| `AGENT_CONVERT_TIMEOUT_S` | `10` | Seconds one conversion may run in the sandbox, at most 15 |
+| `AGENT_CONVERT_SLOT_WAIT_S` | `20` | Seconds a conversion waits for the render slot behind every pipeline render; then `503 capacity` |
 | `AGENT_MAX_ATTEMPTS` | `2` | Rounds per pipeline run, 1 to 3: the first and at most one repair round by default (a self-review adds an adapter call to a round, never a round); the eval harness sets 1 or 3 (`--max-attempts`) to measure no repair or a second repair round |
 | `AGENT_REVIEWER_MODE` | `catalogue` | The reviewer's rubric: `catalogue` (the offline scorer's rubric, verdict decided by code from the points) or `checklist` (the narrow checklist used before 2026-10-13) |
 | `AGENT_REVIEW_BAR`, `AGENT_STATUS_BAR` | `0.95`, `0.95` | `catalogue` reviews: the share of technical points (VQ, SC, DQ) below which a render gets the repair (1.0 before 2026-10-14; owner decision pending), and the ship bar at or above which it shows without the review's notes |

agents/anyplot/data/parse.py

@@ -117,7 +117,7 @@
 from collections import Counter
 from collections.abc import Collection, Iterable, Mapping, Sequence
 from dataclasses import dataclass, field, replace
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from itertools import pairwise, product
 from typing import Any, Literal
 
@@ -353,6 +353,12 @@
 _DOT_DATE = re.compile(r"([0-9]{1,2})\.([0-9]{1,2})\.([0-9]{4})" + _LOCAL_TIME)
 _SLASH_DATE = re.compile(r"([0-9]{1,2})/([0-9]{1,2})/([0-9]{4})" + _LOCAL_TIME)
 
+_ISO_STAMP = re.compile(
+    r"([0-9]{4})-([0-9]{2})-([0-9]{2})"
+    r"(?:[T ]([0-9]{2}):([0-9]{2})(?::([0-9]{2})(?:[.,]([0-9]{1,9}))?)?(Z|[+-][0-9]{2}(?::?[0-9]{2})?)?)?"
+)
+"""An ISO date, or a timestamp with seconds, a fraction of up to nine digits and an offset (`_as_iso_stamps`)."""
+
 _DATE_FORMATS: tuple[tuple[re.Pattern[str], str], ...] = (
     (_ISO_DATE, "iso"),
     (_DOT_DATE, "dot"),
@@ -441,6 +447,8 @@ class _Column:
     bare: list[str] = field(default_factory=list)  # the cells the number typing read: `source`, or the number parts
     numbers: list[float] = field(default_factory=list)
     dates: list[datetime] = field(default_factory=list)
+    date_texts: list[str] = field(default_factory=list)  # the canonical text of each of `dates`, for exact min/max
+    date_rests: list[int] = field(default_factory=list)  # nanoseconds past each of `dates`' microsecond, for order
     with_time: bool = False
     decimal_comma: bool = False
     notes: list[str] = field(default_factory=list)
@@ -1948,7 +1956,8 @@ def _type_column(column: _Column) -> None:
     """Give the column its dtype and canonical cells, in this order of attempts.
 
     1. **datetime**: every non-missing cell matches one format: ISO
-       `YYYY-MM-DD[(T| )HH:MM[:SS]]`, `DD.MM.YYYY`, or `DD/MM/YYYY` / `MM/DD/YYYY`,
+       `YYYY-MM-DD[(T| )HH:MM[:SS[.fraction]][offset]]` (`_as_iso_stamps`: up to nine
+       fraction digits and the offset kept as written), `DD.MM.YYYY`, or `DD/MM/YYYY` / `MM/DD/YYYY`,
        the last three with an optional ` HH:MM[:SS]` and one- or two-digit day and
        month. A slash column is day-first when a first part exceeds 12, month-first
        when a second part does, not a date when both happen, and day-first with a
@@ -2199,7 +2208,85 @@ def _match_all(pattern: re.Pattern[str], cells: list[str]) -> dict[str, tuple[in
     return fields
 
 
+def _as_iso_stamps(column: _Column, present: list[str]) -> bool | None:
+    """ISO dates and timestamps with every digit and the offset as written (owner rule of 2026-10-15).
+
+    None when a cell is no ISO date, so the other formats are tried. A fraction of up to nine
+    digits stays (each cell padded to the column's longest fraction, which changes no value and
+    lets `pd.read_csv(parse_dates=...)` read the column as one format), and so does an offset
+    (`Z`, `+01`, `+0100`, `+01:00`, written `+01:00`; `Z` as `+00:00`). A column that mixes
+    cells with and without an offset is not a date column: no time is given a zone by guess.
+    A column with several offsets (summer and winter time) is written in UTC (`+00:00`) with a
+    note naming the offsets, because pandas reads one offset per column; the instants stay.
+    """
+    matches = []
+    for cell in present:
+        match = _ISO_STAMP.fullmatch(cell)
+        if match is None:
+            return None
+        matches.append((cell, match))
+    offsets: dict[str, int | None] = {}
+    stamps: dict[str, tuple[datetime, str]] = {}
+    for cell, match in matches:
+        year, month, day = int(match.group(1)), int(match.group(2)), int(match.group(3))
+        hour, minute, second = (int(match.group(index) or 0) for index in (4, 5, 6))
+        fraction, offset = match.group(7) or "", match.group(8)
+        try:
+            stamp = datetime(year, month, day, hour, minute, second, int((fraction + "000000")[:6]))
+        except ValueError:
+            return False
+        minutes: int | None = None
+        if offset is not None:
+            digits = offset[1:].replace(":", "") if offset != "Z" else "0000"
+            minutes = int(digits[:2]) * 60 + int(digits[2:4] or 0)
+            if minutes >= 24 * 60:
+                return False
+            minutes = -minutes if offset.startswith("-") else minutes
+            stamp = stamp.replace(tzinfo=timezone(timedelta(minutes=minutes)))
+        offsets[cell] = minutes
+        stamps[cell] = (stamp, fraction)
+    zones = set(offsets.values())
+    if None in zones and len(zones) > 1:
+        return False
+    mixed = len(zones) > 1
+    if mixed:
+        shown = ", ".join(_offset_text(value) for value in sorted(zone for zone in zones if zone is not None))
+        column.notes.append(f"times with the UTC offsets {shown} written in UTC (+00:00); the instants are the same")
+    digits = max((len(fraction) for _, fraction in stamps.values()), default=0)
+    column.with_time = any(":" in cell for cell in present)
+    texts: dict[str, str] = {}
+    for cell, (stamp, fraction) in stamps.items():
+        if mixed:
+            stamp = stamp.astimezone(timezone.utc)
+        if not column.with_time:
+            texts[cell] = stamp.date().isoformat()
+            continue
+        text = stamp.replace(microsecond=0, tzinfo=None).isoformat(timespec="seconds")
+        if digits:
+            text += "." + fraction.ljust(digits, "0")
+        if stamp.tzinfo is not None:
+            text += _offset_text(0 if mixed else offsets[cell])
+        texts[cell] = text
+    column.dtype = "datetime"
+    column.dates = [stamps[cell][0] for cell in column.source if cell]
+    column.date_texts = [texts[cell] for cell in column.source if cell]
+    column.date_rests = [int((stamps[cell][1] + "000000000")[6:9]) for cell in column.source if cell]
+    column.cells = [texts[cell] if cell else "" for cell in column.source]
+    return True
+
+
+def _offset_text(minutes: int | None) -> str:
+    """An offset in minutes as `+HH:MM`."""
+    value = minutes or 0
+    sign = "-" if value < 0 else "+"
+    hours, rest = divmod(abs(value), 60)
+    return f"{sign}{hours:02d}:{rest:02d}"
+
+
 def _as_dates(column: _Column, present: list[str]) -> bool:
+    stamped = _as_iso_stamps(column, present)
+    if stamped is not None:
+        return stamped
     kind, fields = "", None
     for pattern, candidate in _DATE_FORMATS:
         fields = _match_all(pattern, present)
@@ -2540,6 +2627,9 @@ def _profile_column(column: _Column) -> ColumnProfile:
     if column.dtype in ("integer", "number") and column.numbers:
         minimum, maximum = min(column.numbers), max(column.numbers)
         mean, median = _statistics(column.numbers)
+    elif column.dtype == "datetime" and column.dates and column.date_texts:
+        order = sorted(range(len(column.dates)), key=lambda index: (column.dates[index], column.date_rests[index]))
+        minimum, maximum = column.date_texts[order[0]], column.date_texts[order[-1]]
     elif column.dtype == "datetime" and column.dates:
         minimum, maximum = _iso(min(column.dates), column.with_time), _iso(max(column.dates), column.with_time)
     elif column.dtype in ("text", "boolean"):

agents/anyplot/data/prepare.py

@@ -1157,7 +1157,9 @@ def _cell_text(value: object, with_time: bool) -> str | None:
         number = float(value)
         return None if math.isnan(number) else repr(number)
     if isinstance(value, pd.Timestamp):
-        return value.isoformat(timespec="seconds") if with_time else value.date().isoformat()
+        # Every digit and the offset as the value holds them (owner rule of 2026-10-15): a fraction is written
+        # only when there is one, an offset only for a zone-aware value.
+        return value.isoformat() if with_time else value.date().isoformat()
     return str(value)
 
 

agents/anyplot/plugins/ledger.py

@@ -169,13 +169,14 @@ class _Day:
     user_data_runs: dict[str, int] = field(default_factory=dict)
     user_messages: dict[str, int] = field(default_factory=dict)
     user_pastes: dict[str, int] = field(default_factory=dict)
+    user_conversions: dict[str, int] = field(default_factory=dict)
 
 
 MESSAGE_WINDOW_S = 60.0
 """The sliding window of `AGENT_USER_MESSAGES_PER_MINUTE` and `AGENT_USER_PASTES_PER_MINUTE`."""
 MessageLimit = Literal["rate_limited", "daily_limit"]
 """Why `message_limit` or `paste_limit` refuses: the minute's count, or the day's."""
-WindowKind = Literal["message", "paste"]
+WindowKind = Literal["message", "paste", "convert"]
 
 
 class UsageBook:
@@ -291,6 +292,27 @@ def add_paste(self, user_id: str) -> None:
         day.user_pastes[user_id] = day.user_pastes.get(user_id, 0) + 1
         self._stamp("paste", user_id)
 
+    def user_conversions(self, user_id: str) -> int:
+        """The user's file conversions today that passed `convert_limit`."""
+        return self._current().user_conversions.get(user_id, 0)
+
+    def convert_limit(self, user_id: str, settings: AgentSettings) -> MessageLimit | None:
+        """Why the user may not convert another file now, or None: `AGENT_USER_CONVERSIONS_PER_MINUTE` and
+        `_PER_DAY`, across sessions, checked before the file reaches the sandbox."""
+        return self._limit(
+            "convert",
+            user_id,
+            self.user_conversions(user_id),
+            settings.user_conversions_per_minute,
+            settings.user_conversions_per_day,
+        )
+
+    def add_conversion(self, user_id: str) -> None:
+        """Count one file conversion against the user's minute and day."""
+        day = self._current()
+        day.user_conversions[user_id] = day.user_conversions.get(user_id, 0) + 1
+        self._stamp("convert", user_id)
+
     def _limit(
         self, kind: WindowKind, user_id: str, today_count: int, per_minute: int, per_day: int
     ) -> MessageLimit | None:

agents/anyplot/render/backends/fake.py

@@ -14,9 +14,10 @@
 
 from PIL import Image, ImageDraw
 
+from agents.renderer.wire import ConvertMeta
 from core.palette import IMPRINT
 
-from ..contract import DataJob, DataOutput, RenderJob, RenderResult, Theme, ThemeOutput
+from ..contract import ConvertJob, ConvertOutput, DataJob, DataOutput, RenderJob, RenderResult, Theme, ThemeOutput
 
 
 BACKGROUNDS: dict[Theme, str] = {"light": "#FAF8F1", "dark": "#1A1A17"}
@@ -57,13 +58,15 @@ class FakeOutcome:
 FIXTURE_CSV = "x,y\n0,0\n1,1\n2,4\n3,9\n"
 """What a fake data run returns by default: the table a snippet computing `y = x**2` for 0 to 3 writes."""
 DataScript = Callable[[DataJob], DataOutput]
+ConvertScript = Callable[[ConvertJob], ConvertOutput]
 
 
 @dataclass
 class FakeBackend:
     """Fixture renders; `script(job, theme)` decides the outcome (default: a clean render).
 
-    A data run returns `FIXTURE_CSV` with exit code 0 unless `data_script(job)` decides;
+    A data run returns `FIXTURE_CSV` with exit code 0 unless `data_script(job)` decides,
+    and so does a conversion (as a one-sheet workbook) unless `convert_script(job)` does;
     no code runs either way, so the fake is for tests and offline development only.
     """
 
@@ -72,6 +75,15 @@ class FakeBackend:
     jobs: list[RenderJob] = field(default_factory=list)
     data_script: DataScript | None = None
     data_jobs: list[DataJob] = field(default_factory=list)
+    convert_script: ConvertScript | None = None
+    convert_jobs: list[ConvertJob] = field(default_factory=list)
+
+    async def convert(self, job: ConvertJob) -> ConvertOutput:
+        self.convert_jobs.append(job)
+        if self.convert_script is not None:
+            return self.convert_script(job)
+        meta = ConvertMeta(kind="xlsx", rows=4, columns=2, sheet="Sheet1", sheets=["Sheet1"], sheet_count=1)
+        return ConvertOutput(exit_code=0, text=FIXTURE_CSV, meta=meta, wall_s=0.1)
 
     async def run_data(self, job: DataJob) -> DataOutput:
         self.data_jobs.append(job)