diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index d3bd9de3298..34784e165b0 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -1268,6 +1268,100 @@ jobs: print(f'OK: {NODE_BINARY} is in the image, and no node is on {SANDBOX_PATH}') " + # plotly and bokeh (PB-1 of step 4 in agentic/specs/261024-agent-all-libraries.md) render in + # two steps: the harness forks the plot before it arms anything, the plot writes its figure as + # JSON under the plot guard and the exec filter with the Python limits, and the harness, never + # armed, then runs the JavaScript row's harness on it, which draws it with the package's own + # bundled JavaScript. The sandbox's environment is a Python row's plus the JavaScript row's, as + # `executor.sandbox_env` builds it; each plot sets its size the way the catalogue's save does, + # and the canvas follows (plotly 800x450 at scale 4, bokeh 4800x2700 at 2/3). The image carries + # plotly and bokeh, and none of their export stacks. + - name: Render plotly and bokeh plots through the harness in figure mode + if: steps.check.outputs.should_build == 'true' + run: | + set -euo pipefail + mkdir -p work-figure/plotly work-figure/bokeh + python3 - > work-figure/row.flags <<'EOF' + from agents.renderer.commands import FIGURE_FLAG, NODE_BINARY, command_for + + for library in ("plotly", "bokeh"): + row = command_for("python", library) + assert row.figure == library and row.exec_deny and row.argv[0] == NODE_BINARY, row + with open(f"work-figure/{library}.argv", "w") as handle: + handle.write("\n".join([FIGURE_FLAG, library, row.code_file, "--", *row.argv]) + "\n") + assert command_for("python", "plotly").env == command_for("python", "bokeh").env + for key, value in row.env.items(): + print(f"-e\n{key}={value}") + EOF + mapfile -t ROW_FLAGS < work-figure/row.flags + cat > work-figure/plotly/plot.py <<'EOF' + import os + + import plotly.graph_objects as go + + THEME = os.getenv("ANYPLOT_THEME", "light") + fig = go.Figure(go.Bar(x=["a", "b", "c"], y=[1, 3, 2], marker={"color": "#009E73"})) + # An axis range builds plotly's info-array validator, an `eval` the guard admits (CODE_BUILDERS). + fig.update_layout(title={"text": "Größe – plotly"}, yaxis={"range": [0, 4]}, width=800, height=450) + fig.write_json(f"plot-{THEME}.json") + EOF + cat > work-figure/bokeh/plot.py <<'EOF' + import json + import os + + from bokeh.embed import json_item + from bokeh.plotting import figure + + THEME = os.getenv("ANYPLOT_THEME", "light") + p = figure(x_range=["a", "b", "c"], width=4800, height=2700, title="Größe – bokeh", toolbar_location=None) + p.vbar(x=["a", "b", "c"], top=[1, 3, 2], width=0.8, color="#009E73") + with open(f"plot-{THEME}.json", "w", encoding="utf-8") as handle: + json.dump(json_item(p), handle) + EOF + for library in plotly bokeh; do + mapfile -t ROW_ARGV < "work-figure/$library.argv" + echo "the $library row: ${ROW_ARGV[*]}" + printf 'x\n1\n' > "work-figure/$library/data.csv" + start=$(date +%s.%N) + docker run --rm --network none --read-only --tmpfs /tmp \ + --cap-drop ALL --security-opt no-new-privileges --user "$(id -u):$(id -g)" \ + -v "$PWD/work-figure/$library:/work" -w /work \ + -e ANYPLOT_THEME=light -e MPLBACKEND=Agg -e MPLCONFIGDIR=/tmp/mpl -e ANYPLOT_MPL_SEED=/opt/mplconfig \ + "${ROW_FLAGS[@]}" \ + -e HOME=/tmp -e PYTHONDONTWRITEBYTECODE=1 -e OPENBLAS_NUM_THREADS=1 -e OMP_NUM_THREADS=1 \ + -e ANYPLOT_RLIMIT_CPU_S=60 -e ANYPLOT_RLIMIT_FSIZE_MB=50 -e ANYPLOT_RLIMIT_AS_MB=1024 \ + -e ANYPLOT_DENY_EXEC=required \ + anyplot-renderer:ci /app/.venv/bin/python -I -X disable-remote-debug /opt/anyplot/harness.py \ + "${ROW_ARGV[@]}" \ + > "work-figure/$library/harness.out" 2> "work-figure/$library/harness.err" \ + || { echo "::error::the $library render failed"; cat "work-figure/$library/harness.out" "work-figure/$library/harness.err"; exit 1; } + echo "wall time: $(python3 -c "import sys; print(round(float(sys.argv[2]) - float(sys.argv[1]), 1))" "$start" "$(date +%s.%N)") s" + grep -qxF 'HARNESS {"event": "start"}' "work-figure/$library/harness.out" \ + || { echo "::error::no HARNESS start line"; cat "work-figure/$library/harness.out"; exit 1; } + python3 - "work-figure/$library" "$library" <<'EOF' + import json, os, struct, sys + + work, library = sys.argv[1:] + png = open(f"{work}/plot-light.png", "rb").read() + assert png[:8] == b"\x89PNG\r\n\x1a\n", "not a PNG" + width, height = struct.unpack(">II", png[16:24]) + assert (width, height) == (3200, 1800), (width, height) + end = json.loads(open(f"{work}/harness.out").read().splitlines()[-1].split(" ", 1)[1]) + assert end["exec_deny"] == "on", f"the plot step ran without the exec filter: {end}" + assert set(end["steps"]) == {"plot", "screenshot"}, end + assert not os.path.exists(f"{work}/plot-light.html"), "the screenshot step wrote a page" + print(f"OK: {library} plot-light.png {width}x{height}, {len(png)} bytes; steps {end['steps']}") + EOF + done + docker run --rm --network none anyplot-renderer:ci /app/.venv/bin/python -c " + import importlib.util + for name in ('plotly', 'bokeh'): + assert importlib.util.find_spec(name), name + for name in ('kaleido', 'choreographer', 'selenium', 'webdriver_manager'): + assert importlib.util.find_spec(name) is None, f'{name} is in the image' + print('OK: plotly and bokeh are in the image, their export stacks are not') + " + # The downloadable plot.html of a JavaScript plot (JS-4b, step 3g of # agentic/specs/261024-agent-all-libraries.md): the row builds the page with # ANYPLOT_PAGE_ONLY=1 and starts no browser; the agents' wrapper (agents/anyplot/render/page.py, diff --git a/agentic/specs/261024-agent-all-libraries.md b/agentic/specs/261024-agent-all-libraries.md index 77c8adfdbab..9a6c48c999e 100644 --- a/agentic/specs/261024-agent-all-libraries.md +++ b/agentic/specs/261024-agent-all-libraries.md @@ -2166,6 +2166,228 @@ Revised on 2026-10-30 after the second opinion on steps 3 and 4. Two PRs after J user's machine; the run form carries the sandbox's JSON save instead. - Fixtures, matrix, docs. +#### What PB-1 built (2026-10-31): the two-step exec mode, not served yet + +The renderer's plotly and bokeh rows render in two steps; `AGENT_LIBRARIES`, +`RUNTIME_LIBRARIES`, the runtime registry and the admin deploy's `_LIBRARIES` are +unchanged, so the agents service sends no plotly or bokeh job before PB-2. + +- **Harness, figure mode** (`harness.py --figure plot.py -- `). The start line, then `CPU` and `FSIZE` limits for both steps, then + `os.fork` before anything is armed. The child sets `RLIMIT_AS` and `RLIMIT_NPROC`, arms + `PlotGuard` with the kernel filter first (`ANYPLOT_DENY_EXEC=required` holds as in plot + mode), runs the plot, which may write nothing but `plot-.json`, and ends with + `os._exit`, so no `atexit` callback and no thread of the plot outlives it. The parent is + never armed: it checks the figure (a regular file opened without following a link, not + empty, at most `FIGURE_MAX_BYTES`, 16 MiB, the wire's cap for a typed table) and starts + the screenshot command, the JavaScript row's harness with `--figure=plot-.json` + appended, in the sandbox's environment without the plot step's variables + (`ANYPLOT_DENY_EXEC`, `ANYPLOT_RLIMIT_AS_MB`, `ANYPLOT_RLIMIT_NPROC`) and with + `ANYPLOT_FIGURE_PACKAGE`, the package's directory as the harness's own interpreter would + import it (`importlib.util.find_spec`, which runs no package code). The child reports + through a pipe opened before the fork. Its first line is the harness's own and comes + before any plot code, so the plot cannot forge it: `armed ` after the arming, or + `refused ` when a required filter is not in place and the plot never runs. Only + after `refused` is `EXEC_DENY_EXIT` (3) the harness's `ExecDenyUnavailable`; a plot that + ran and ended with the same code (`sys.exit(3)`) fails as a `plot` error like any other + code. The error record after the first line is advisory. The end line carries both + steps' totals, the plot step's `exec_deny`, and `steps` with each step's peak, CPU and + wall time. +- **The final error line** of JS-2 for the plot step as well: `plot plot.py:` + for an exception (its traceback before it, as the interpreter prints it), `figure` + (`NoFigure`, `FigureError`, `FigureTooLarge`), `crash PlotCrashed` for a signal and + `harness` (`UsageError`, `PackageMissing`, `ExecDenyUnavailable`, `SpawnError`); the + screenshot step's own line otherwise. The exit code is the failed step's. +- **`render.mjs --library= --figure=`**, sandbox mode only. The figure + is read without following a link, capped again, parsed and shape-checked in Node, and + carried as data: a `type="application/json"` element with `<`, U+2028 and U+2029 + escaped, parsed in the page and handed to `Plotly.newPlot` (`staticPlot`, + `typesetMath: false`) or `Bokeh.embed.embed_item`. The bundles are the packages' own: + plotly's `package_data/plotly.min.js` (plotly.js 4.1.1 in plotly 7.1.0) and bokeh's + `bokeh.min.js` with its WebGL, widget and table bundles (bokeh 3.10.1; its MathJax bundle + is not loaded). The page is sandbox mode's, routes, console guard and heap cap included, + under a stricter policy: scripts by a nonce of the page instead of `'unsafe-inline'`. +- **The canvas.** The figure's own size in CSS pixels (plotly's layout width and height, + bokeh's root width and height; a bokeh layout, which has none, is drawn once at scale 1 + and measured) picks the orientation, landscape at 4:3 and wider, and the device scale + makes the box exactly 3200 x 1800 or 2400 x 2400: plotly 800 x 450 draws at scale 4, + 1600 x 900 at 2, bokeh 4800 x 2700 at 2/3. A figure narrower than its box is centred on + the page's background; a plotly figure without a size gets 1600 x 900, and one that + names a single side keeps it and gets the other at 16:9 (width 1200 draws at 1200 x + 675); a side outside 320 to 6400, the derived one included, is `figure FigureError`. A screenshot of any other size fails as `harness + CanvasError` instead of leaving a PNG. +- **plotly's `CODE_BUILDERS` entry** is `_plotly_utils/basevalidators.py: + build_validator`: `InfoArrayValidator.__init__` calls it for each item of an info-array + property (`xaxis.range`, a `domain`), and it evaluates the item's validator class name, + `datatype.title().replace("_", "") + "Validator"`. plotly's schema names five types + (`angle`, `any`, `enumerated`, `number`, `subplotid`); a plot that calls the builder + itself gets title-cased text without underscores, which reaches only capitalised names + (the module's 23 validator classes, the builtins' classes and constants) and runs under + the guard. Every plotly plot that sets an axis range needs it. +- **Renderer.** `RuntimeCommand.figure` names the package of a two-step row; plotly's and + bokeh's rows run `NODE_BINARY render.mjs --library=` as the screenshot command + with `JS_ENV`, and keep the Python limits and the filter for the plot step. + `executor.build_argv` passes `--figure plot.py -- `, `sandbox_env` a + Python row's settings plus `JS_ENV`. The image installs the `lib-plotly` and `lib-bokeh` + extras without their export stacks (kaleido, choreographer, logistro, Selenium, + webdriver-manager, trio and the WebSocket clients) and trims what neither step reads + (plotly's JupyterLab and widget bundles, bokeh's TypeScript compiler, ES module sources + and unminified bundles): plotly and bokeh take 68 MB, and the renderer image grows from + 4,451 to 4,520 MB (+69 MB, against main on the same toolchain image). + +Measured in local Docker on the developer machine (32 CPUs, WSL 2, kernel 6.18), as the CI +step runs it: `--network none --read-only`, `--cpus 2 --memory 4g`, an unprivileged user, +the row's environment, `RLIMIT_AS` 1 GiB and `ANYPLOT_DENY_EXEC=required`; a hand-written +figure per library and the catalogue's `scatter-basic`, `bar-basic`, `line-basic` and +`heatmap-basic` with the save block swapped by hand for the JSON save (and Selenium's +imports dropped), each in both themes: + +| Measurement | Result | +|---|---| +| Rendered | 20 of 20, every PNG 3200 x 1800 or 2400 x 2400 (the heatmaps), `exec_deny` `on` in every plot step | +| Wall time, container start included | plotly 1.93 to 2.35 s, bokeh 1.60 to 1.85 s | +| Plot step | 0.14 to 0.54 s wall, 0.14 to 0.53 s CPU, 53 to 156 MB peak (bokeh's heatmap the largest) | +| Screenshot step | plotly 1.40 to 1.47 s wall, 0.74 to 0.77 s CPU, 186 to 193 MB for its largest process; bokeh 0.91 to 0.97 s, 0.68 to 0.73 s, 166 to 170 MB | +| The whole container (`memory.peak`) | plotly 399 to 416 MiB, bokeh 518 to 561 MiB | +| Under gVisor (`runsc-ptrace`, release-20261026.0), 12 of the renders | all 12 rendered, `exec_deny` `on`: wall 4.65 to 5.78 s, the plot step 0.40 to 1.15 s, the screenshot step 3.68 to 4.63 s; the same image's Chart.js render takes 3.68 s there and 1.06 s under runc | +| Against the published catalogue PNGs (light) | bokeh `bar-basic` and `heatmap-basic` a mean difference of 0.01 of 255 (no pixel off by more than 40); plotly `bar-basic` 0.54 %, `line-basic` 0.11 %, `heatmap-basic` 1.09 % and `scatter-basic` 0.68 % of the pixels off by more than 40; bokeh `scatter-basic` 0.67 % and `line-basic` 2.03 %. The two largest are drift between the files and their published PNGs (the published bokeh `line-basic` has a band and an annotation the file does not draw; the published plotly `scatter-basic` has axis lines its `showline: False` turns off) | +| Canvas against the catalogue's | plotly `line-basic` (1600 x 900 at scale 3) and bokeh `scatter-basic` and `line-basic` (4800 x 2700) publish 4800 x 2700 PNGs; this route draws the same layouts at 3200 x 1800 | +| Fonts | the page resolves the files' Helvetica and Arial to Liberation Sans through the JavaScript row's fontconfig file; plotly's `Inter` first in a stack is not in the image and falls through to it | + +Also found: + +- **No WebGL in the headless shell.** plotly draws a notice in place of a WebGL trace + ("WebGL is not supported by your browser"), so the page now fails such a render as + `draw NoWebGL` instead of passing with the notice for a plot. In the catalogue that is + `Scattergl` (4 files), `Scatter3d` (5), `Surface` (2), `Mesh3d`, `Parcoords` and + `Densitymap` (1 each) and `Scattermap` (8, which also needs map tiles); plotly express + switches `scatter` to WebGL above 1,000 points. bokeh's `output_backend="webgl"` falls + back to its canvas (no catalogue file uses it). WebGL would need a software renderer + in Chromium (`--enable-unsafe-swiftshader`), an owner decision for PB-2. +- **Markup in a figure.** bokeh's `Div` renders its HTML into the page: under sandbox + mode's `script-src 'unsafe-inline'` an `` in it ran (its fetch then stopped + at the policy). The figure page admits scripts by its nonce alone, so the handler does + not run; `CustomJS` and `CustomJSTickFormatter` cannot compile without + `'unsafe-eval'` (a render with the formatter fails as `page EvalError`). Markup in a + plotly title stays text. +- kaleido's `write_image` is refused at choreographer's socket pair (`socket.__new__`) and + bokeh's `export_png` at its temporary HTML file (`open`), both under the guard on a + developer machine where both stacks are installed; the image installs neither. +- **Fixed on the way: user code while a guard is paused.** Every place the harness pauses + a guard was checked for code the snippet or plot defined. Two ran there and wrote files + the armed guard refuses: data mode's `print_snippet_error` read `exc.__traceback__`, + which the snippet's exception class can make a property, and plot mode's + `writes_elsewhere` ran `realpath` on a path that subclasses `str`, whose `split` and + `startswith` it called. The traceback is now read through `BaseException`'s own + descriptor, the path is copied into an exact `str` while the guard is armed, and the + frame-name comparisons take exact `str` only (`plain_name`); regression tests in + `test_harness_guard.py` and `test_figure_mode.py` write a marker on the old harness. +- **Guard hardening found on the way** (Fable's review of PB-1 rated F6 high, Flash's + review added the descriptor flags; built 2026-10-31): + - F6, import trust. The plot guard trusted any code the import machinery ran, and the + frames it read. A plot drove that machinery itself and supplied the module body, + which walked the frames to the guard and disarmed it. Three rules now hold: + an import reads code only from the install (`install_prefixes`: the stdlib with + `lib-dynload` and its zip, the site packages, from `sysconfig` before arming), refused + at the read of a file loader's `get_data` or zipimport's archive + (`IMPORT_CODE_READERS`), whoever drives the loader; an import earns trust only when + its first import frame is the interpreter's `_call_with_frames_removed` under the + `exec_module` or `create_module` of exactly `SourceFileLoader`, + `SourcelessFileLoader`, `ExtensionFileLoader` or `zipimporter` (instance state + `name` and `path`, or `archive` and `prefix`, nothing else, the file in the install, + the class as it was before arming), + under `BuiltinImporter`, or is `FrozenImporter.exec_module` (`ImportMachinery`); + and an extension module loads only from the install, refused at the `import` event + that `_imp.create_dynamic` raises with its file before the library's native code runs. + - Native loads: an `ExtensionFileLoader` on a plot-written file raises that `import` + event (measured under runc and gVisor); `ctypes.CDLL` raises `ctypes.dlopen`, refused + already; cffi's `FFI().dlopen(path)` raises no audit event at all, so no guard rule + can see it (an open question for the owner: a `noexec` run directory, as Docker's + `tmpfs` already is; the validator refuses `cffi`). + - Descriptor flags: `fcntl.*` and `os.set_blocking` (no event, replaced) are refused in + plot mode, figure mode's plot step included, because stdout and stderr are one open + file description across the fork. + - The validator (F7 and the walk): a walk over every public attribute of the admitted + modules (1,331 modules in the image) found import machinery the validator admitted: + matplotlib's bundled `dateutil.zoneinfo.get_data` (`pkgutil.get_data`, a read of any + file relative to a package it imports by name) and `dateutil.tz.six.spec_from_loader`, + and the `types` module's `new_class` (a class from a callable body, where class + statements are refused), `ModuleType` and `LambdaType`. `dateutil` and `six` are + banned as module re-exports, `get_data`, the spec builders, `new_class`, + `prepare_class` and `ModuleType` as helpers, `LambdaType` with `FunctionType`; the + walk is now a test, and of the image's 222 machinery paths none is admitted. + - Measured: every catalogue file of the six served Python libraries (1,946) rendered + in the renderer image with the previous harness and with this one, 12 at a time: + 1,771 render under both, 175 fail under both (the guard's earlier refusals, out of + memory), no file changes outcome, and no run called `fcntl` or `os.set_blocking`. The + ten plotly and bokeh inputs above render as before under runc, two of them under + gVisor. The F6 routes (stock source, bytecode and zip loaders on a plot-written file, + the loader class of an admitted module, Fable's loader of its own, an instance that + overrides `get_code`, `exec` as a path hook, an extension module written by the plot) + each wrote a marker through the previous harness and are refused now, in plot and + figure mode; they are tests in `test_harness_guard.py`. + - Known limits of the in-process layer. The guard shares the interpreter with the + plot, so it is defense in depth behind the validator, the kernel's exec filter + (seccomp, on every thread and inherited by every child), the resource limits and the + per-job gVisor sandbox (spikes S and S2: no egress, as the metadata server, DNS and + TCP all fail; none of the service's environment or tokens; a read-only root + filesystem without `--write`; no view of the sibling sandboxes; one job's lifetime). + The harness's module docstring states the same. Two limits are known and stay: F8, + plot code that reaches the harness's own objects through state the interpreter + shares can change the guard's decisions, which no audit event sees; and cffi's + `dlopen`, which raises no audit event (above). The validator refuses what both need + (`cffi`, `sys` and every dunder name), and no structural change to the harness is + planned for F8. A class attribute changes without an audit event as well, so the + stock loaders' classes are read before arming, and a loader whose class no longer + resolves every attribute to the same object earns no trust (`ImportMachinery.intact`: + the same method resolution order, each class's own entries unchanged but the two + Python 3.14 adds when annotations are first read): Fable's review of the fix measured + a `get_code` set on `SourceFileLoader` itself, with a clean instance on a stdlib file, + running the plot's module body with an import's trust, and a `source_to_code` set + there did the same. Both wrote the marker through the first fix and are refused now; + the 1,946 catalogue files render with the class check exactly as without it (1,771 + render, 175 fail, the same refusals). + Deeper patches of the machinery (the module globals of `_frozen_importlib_external`, + `_bootstrap._call_with_frames_removed`, `builtins.compile`) are F8's kind and + knowingly out of the in-process layer's reach; the validator, which refuses every + name that reaches them, is the barrier there. +- **A two-step row needs Node.** `GET /status` still lists the toolchains per language, + but a render or page job is checked against its row (`executor.serves`): the language's + toolchain and the row's own binaries, so a plotly or bokeh job on a renderer without + Node is `503 runtime_unavailable` before any sandbox starts, not exit 127 with a + `harness SpawnError` line. Every renderer image has Node (the toolchain base). + +**Tests**: `tests/unit/agents/renderer/test_figure_mode.py` (92 cases without plotly or +bokeh: the package's lookup and the screenshot command stood in): fork before arm (both +steps children of one harness, the plot step one seccomp filter and the no-new-privileges +bit more than the harness, the screenshot step none more and able to start a program), +the limits and variables per step, a refused spawn, fork, exec, socket pair and PNG write, +the kernel's refusal when the audit hook is blind to `os.exec`, nothing of the plot after +it (`atexit`, a thread), the step-1 failures and their lines, a signal death, the cap at +one byte over, a filter that is not in place, a plot that exits with the filter's code +itself, the screenshot step's own failure and a command that cannot start, the rows, the +argv, the environment and Node's check, and the executor's one timeout over either step +(the launcher's process group killed with both children). The fork, the arming and the +kernel filter run only in those subprocess cases, which coverage does not see; in-process +cases, with `os.fork`, `os._exit`, the guard and the plot stood in, cover the parent's +verdict, the child's codes and report, the error record, the screenshot step's start and +the steps' order. `tests/unit/automation/test_js_render_figure.py` (41 cases, skipped +without Node, the lock's packages, the browser and plotly): the canvas rule, a plotly +figure with one side, a measured bokeh layout, markup +and handlers, `CustomJS`, WebGL, the figure checks, the usage errors, and the whole two +steps through the Python harness in both themes for both libraries. `ci-image.yml` +renders a plotly and a bokeh figure through the rows in the built image and checks that +the export stacks are absent. + +| Text above | Built | Why | +|---|---|---| +| `render.mjs --screenshot` | `render.mjs --library= --figure=`, refused without sandbox mode | the input is the figure; the page's rules must apply to it | +| `_plotly_utils/basevalidators.py: __init__` | `_plotly_utils/basevalidators.py: build_validator` | the `eval` sits in the static method that `InfoArrayValidator.__init__` calls (plotly 7.1.0, the lock's version); the guard judges the frame that calls `eval` | +| The page built inside `#container` at the canonical canvas | the figure's own CSS size, the device scale making the canvas | the catalogue's plotly files write 800 x 450 at scale 4 (146), 1600 x 900 at 3 (116), 600 x 600 at 4 (43) and 1200 x 1200 at 3 (9), and its bokeh files pass `width=4800` (113), `3200` (121) and `2400` (35): one CSS size would change every other file's type size against its plot | +| The vendored `plotly.min.js` or BokehJS | the installed packages' own bundles, BokehJS's WebGL, widget and table bundles included, found by the harness | the page's version is the plot's; a model of another bundle would not draw | +| Sandbox mode's page rules | the same with a nonce for scripts instead of `'unsafe-inline'` (tighter) | a bokeh `Div`'s handler ran under `'unsafe-inline'` (measured) | +| The `lib-plotly` and `lib-bokeh` extras | without the export stacks, and trimmed | they start browsers, which the guard refuses; 27 MB of Selenium drivers; the image grows by 69 MB | + ### 5. Julia: Makie Revised on 2026-10-29 after the second opinion on step 5 (Gemini 4 Argon and Fable; the diff --git a/agents/README.md b/agents/README.md index fe5d6695104..5e22f301e84 100644 --- a/agents/README.md +++ b/agents/README.md @@ -33,7 +33,7 @@ The model is **Claude Haiku 5.5 on Vertex AI** (`claude-haiku-5-5`) by default. | `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8/G9/G10/G11/G12), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, `serial.py`, the one render semaphore in front of every backend, `watermark.py`, the footer strip the artifact route and the feedback bundle add to the raw render, and `page.py`, the wrapper of a JavaScript plot's downloadable `plot.html` (no script of its own, the harness's policy plus `frame-src 'none'`, the page in a sandboxed `srcdoc` frame), which `anyplot/page_export.py` builds on demand for the artifact route since 2026-10-31 | | `anyplot/render/runtimes/` | The runtime registry (`runtime_for(library)`, `RUNTIME_LIBRARIES`): one `RuntimeAdapter` per library, which binds the language's code modules (normaliser, regions, readiness, validator, edits, loader, export), its error grammar and its render command for the pipeline, the theme toggle, the local backend and the eval harness; `python.py` serves matplotlib, seaborn, plotnine, altair, pygal and lets-plot (`PYTHON_LIBRARIES`; plotnine and altair since 2026-10-25, pygal and `letsplot` since 2026-10-26), each with its own save rule (`anyplot/code/regions.py`, `SAVE_RULES`) and validator profile; `r.py` serves ggplot2 since 2026-10-29 (`R_LIBRARIES`): the R code profile (`anyplot/code/profiles/r/`) and the renderer's R row in exec mode; `julia.py` serves Makie since 2026-10-30 (`JULIA_LIBRARIES`): the Julia code profile (`anyplot/code/profiles/julia/`) and the renderer's Julia row in exec mode; `javascript.py` serves Chart.js, D3, ECharts and Highcharts since 2026-10-30 (`JAVASCRIPT_LIBRARIES`; MUI X follows once its fixtures exist): the JavaScript code profile (`anyplot/code/profiles/javascript/`), the renderer's JavaScript row in exec mode, and the table as typed JSON (`RenderJob.data_json`), which the harness puts into the page. `runtime_for` picks the class by the library's language (`RUNTIME_CLASSES`). A later language adds a runtime here, never a backend (`agentic/specs/261024-agent-all-libraries.md`) | | `anyplot/render/fonts/` | JetBrains Mono 2.304 Regular and Bold for the footer strip, unmodified, with `OFL.txt` (SIL Open Font License 1.1) and a notice with their SHA-256 values ([fonts README](anyplot/render/fonts/README.md)) | -| `renderer/` | The `anyplot-renderer` service: `main.py` (`POST /render`, `POST /run-data`, `POST /convert`, `POST /page` for a JavaScript version's self-contained page since 2026-10-31, `GET /status`, one render slot), `executor.py` (one `sandbox do` per theme with the kill path, the byte watchdog and the launcher retry), `commands.py` (the command table: one fixed render command per language, selected by the job's `(language, library)` and never taken from the request; the harness's exec mode runs the non-Python commands; `GET /status` reports under `languages` which toolchains this image carries, and a job in a language without one answers `503 runtime_unavailable`), `auth.py` (the caller check), `settings.py` (`RENDERER_*`), `wire.py` (the JSON contract the `remote` backend shares), `page_probe.mjs` (the CI probe of the downloadable page's boundary in the image's Chromium, and since JS-4b-2 of the chat's preview relay), `Dockerfile` and `cloudbuild.yaml`. No ADK import and no `__init__.py`, so `adk web agents` does not list it as an agent | +| `renderer/` | The `anyplot-renderer` service: `main.py` (`POST /render`, `POST /run-data`, `POST /convert`, `POST /page` for a JavaScript version's self-contained page since 2026-10-31, `GET /status`, one render slot), `executor.py` (one `sandbox do` per theme with the kill path, the byte watchdog and the launcher retry), `commands.py` (the command table: one fixed render command per language, selected by the job's `(language, library)` and never taken from the request; the harness's exec mode runs the non-Python commands, and since 2026-10-31 its figure mode runs plotly's and bokeh's rows in two steps, the plot armed in a forked child writing its figure as JSON and the JavaScript row's harness drawing it, not served before PB-2 of `agentic/specs/261024-agent-all-libraries.md`; `GET /status` reports under `languages` which toolchains this image carries, and a job whose row cannot run here, its language's toolchain or the row's own binaries missing (Node for plotly's and bokeh's screenshot step), answers `503 runtime_unavailable`), `auth.py` (the caller check), `settings.py` (`RENDERER_*`), `wire.py` (the JSON contract the `remote` backend shares), `page_probe.mjs` (the CI probe of the downloadable page's boundary in the image's Chromium, and since JS-4b-2 of the chat's preview relay), `Dockerfile` and `cloudbuild.yaml`. No ADK import and no `__init__.py`, so `adk web agents` does not list it as an agent | | `anyplot/opening.py`, `session_state.py`, `services.py`, `briefs.py` | Opening a session and taking in a dataset, the server-set session state, the process-wide stores, and the spec and dataset briefs | | `anyplot/dev_fixture.py` | The development-only session seed from an eval fixture case | | `anyplot/settings.py` | `AgentSettings`, read from `AGENT_*` environment variables | diff --git a/agents/anyplot/code/validate.py b/agents/anyplot/code/validate.py index a4e6c95115f..c7696da87e3 100644 --- a/agents/anyplot/code/validate.py +++ b/agents/anyplot/code/validate.py @@ -761,6 +761,7 @@ "_interpchannels", "CodeType", "FunctionType", + "LambdaType", # `types.FunctionType` under its other name (the walk of 2026-10-31) "attrgetter", "methodcaller", "global_enum", @@ -833,6 +834,12 @@ "geopandas", "png", "palettable", + # matplotlib re-exports dateutil (`matplotlib.dates.dateutil`, `matplotlib.category.dateutil`), whose + # `zoneinfo.get_data` is `pkgutil.get_data`: import by name and a read of any file relative to the package + # (`'../..'`), and whose vendored `tz.six` hands out importlib's `spec_from_loader` (the reachability walk of + # 2026-10-31, F7 of PB-1's review). + "dateutil", + "six", } ) UNSAFE_SUBMODULES: frozenset[str] = frozenset( @@ -1001,6 +1008,20 @@ "get_boundaries", "get_limits", "extend_path", + # Import machinery and package data by name wherever a package re-exports it (the reachability walk of + # 2026-10-31, F7 of PB-1's review): `pkgutil.get_data` reads any file relative to a package it imports by name, + # and importlib's spec builders take a loader. matplotlib's `Line2D.get_data()` goes with them; no catalogue + # file calls it, and `get_xdata()` and `get_ydata()` stay. + "get_data", + "spec_from_loader", + "spec_from_file_location", + "module_from_spec", + # The `types` module is re-exported by typing, dataclasses, matplotlib.cbook and pandas (the same walk): + # `new_class` builds a class from a callable body, which a class statement and three-argument `type()` are + # refused for, and `ModuleType` a module object a loader could run code into. + "new_class", + "prepare_class", + "ModuleType", } ) """Public functions and classes of the allowed modules that exec, start a process, import by name, fetch or write diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 1a16c79b7b4..bda10ba5940 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -50,7 +50,9 @@ 5. compiles the plot file and runs it as `__main__` (`run_plot`, which `runpy.run_path` did before) under `PlotGuard`, an audit hook armed just before: from there on the interpreter refuses every `exec` but the plot's own run, the module bodies of an import - and `CODE_BUILDERS` (`namedtuple`, dataclasses, scipy's distributions, plotnine's + by the interpreter's own loaders (`ImportMachinery`; an import reads code only from the + install, and an extension module loads only from there: `IMPORT_CODE_READERS`, since + 2026-10-31) and `CODE_BUILDERS` (`namedtuple`, dataclasses, scipy's distributions, plotnine's expression strings, and cssselect2's selector compiler, which pygal's PNG export runs for every CSS rule of the chart), a code object or function built by hand or a `__code__` assignment, every `open` that writes anything but @@ -117,6 +119,34 @@ arms `PlotGuard` instead (step 5), which lets imports and reads through and refuses with `PermissionError`, so a plot that catches a refusal renders without what was refused. +**Figure mode** (`harness.py --figure plot.py -- `, the renderer's plotly and +bokeh rows, PB-1 of step 4 in agentic/specs/261024-agent-all-libraries.md): two steps, because a plot-mode harness +cannot start a browser once its guard is armed, and must not before. It prints the start line and sets the limits +of both steps (CPU seconds, the largest file), then forks the plot step before anything is armed: the child sets +the address-space and process limits, arms `PlotGuard` with the kernel filter first (`ANYPLOT_DENY_EXEC=required` +holds here as in plot mode), runs the plot, which may write nothing but `plot-.json`, the figure as JSON +(`fig.write_json` for plotly, `bokeh.embed.json_item` for bokeh), and ends with `os._exit`. The parent is never +armed: once the child ended with exit code 0, it checks the figure (a regular file of at most `FIGURE_MAX_BYTES`) +and starts the screenshot command, the JavaScript row's harness (`render.mjs --library=`) with +`--figure=plot-.json` appended, under the JavaScript row's limits and environment: the sandbox's without +`FIGURE_PLOT_ONLY_ENV`, with `ANYPLOT_FIGURE_PACKAGE` naming the package's directory, whose own bundled JavaScript +draws the figure, so the versions match. The plot step's failures end stderr with the final error line of the +JavaScript harness (`figure_error_line`), the screenshot step's with the harness's own; the exit code is the failed +step's, and the end line carries both steps' usage and the plot step's filter state (`main_figure`). + +**Security model.** `DataGuard` and `PlotGuard` are defense in depth, not the boundary: an audit hook shares the +interpreter with the code it judges, so code that changes the interpreter's shared state in place (the harness's own +objects, the import machinery's functions and module globals, the builtins) can change what the hook decides, and +no audit event sees such a change. The tests name every route the guards close; the known limits of this layer are +in the plan's PB-1 section (agentic/specs/261024-agent-all-libraries.md, step 4). The boundaries, in order: the +validator (`agents/anyplot/code/validate.py`), which refuses the names those routes need (`sys`, `cffi`, every dunder +name) before any code runs; the kernel's exec filter (`install_exec_deny`: seccomp, installed with +`SECCOMP_FILTER_FLAG_TSYNC` on every thread and inherited by every child), which no code in the process can lift; +the resource limits, soft and hard alike; and on Cloud Run the gVisor sandbox that `sandbox do` starts for each job +(spikes S and S2): no egress (the metadata server, DNS and TCP all fail), none of the service's environment or +tokens, a read-only root filesystem (the renderer passes no `--write`), no view of the sibling sandboxes, and a +lifetime of one job. + The probe feeds the advisory gates G3 (clipped text), G5 (annotation outside its axes), G7 (overlapping text: tick labels among themselves, any other text with any), G8 (more point marks than data rows), G9 (more plotted points than any dataset within the parser's caps can hold) and G10 @@ -240,6 +270,30 @@ before its `main`, so no user code is read.""" EXEC_DENY_PREFLIGHT_TIMEOUT_S = 30 """The preflight ends in its library's constructor; a program that outlives this ignored the preload.""" +FIGURE_FLAG = "--figure" +"""Figure mode (module docstring): `harness.py --figure plot.py -- `.""" +FIGURE_PACKAGES = ("plotly", "bokeh") +"""The packages whose figure the screenshot step draws with the package's own JavaScript (`render.mjs`, `BUNDLES`).""" +FIGURE_FILE = "plot-{theme}.json" +"""The one file the plot step may write: the figure as JSON, which the screenshot step reads (`--figure=`).""" +FIGURE_MAX_BYTES = 16 * 1024 * 1024 +"""The largest figure the screenshot step takes (`render.mjs` `FIGURE_MAX_BYTES`, pinned equal by a test): the wire's +cap for a JavaScript job's typed table (`wire.MAX_DATA_JSON_CHARS`), whose worst case at the parser's 20,000 rows and +50 columns measures 9.04 million characters; `RLIMIT_FSIZE` bounds the file itself.""" +FIGURE_PACKAGE_ENV = "ANYPLOT_FIGURE_PACKAGE" +"""The package's directory (`find_figure_package`), which the screenshot step reads the bundled JavaScript from.""" +FIGURE_PLOT_ONLY_ENV = ("ANYPLOT_DENY_EXEC", "ANYPLOT_RLIMIT_AS_MB", "ANYPLOT_RLIMIT_NPROC") +"""The variables of the plot step that the screenshot step does not get: the JavaScript row has no exec filter and +neither `RLIMIT_AS` nor `RLIMIT_NPROC` (`agents/renderer/commands.py`).""" +FIGURE_REPORT_BYTES = 64 * 1024 +"""What the parent reads of the plot step's report pipe (`plot_step`): its first line, the harness's own (`armed` or +`refused` with the filter's state), is written before any plot code runs; the last, the error record, is advisory like +every line the plot shares.""" +FIGURE_ERROR_CHARS = 200 +"""The final error line's message at most (`render.mjs` `ERROR_MESSAGE_CHARS`).""" +FIGURE_ERROR_KINDS = ("plot", "figure", "crash", "harness") +"""The final error line's kinds of the plot step: an exception in the plot, its figure file, a death by a signal, and +the harness itself (usage, the filter, the package); the screenshot step prints its own (`render.mjs`).""" DATA_FLAG = "--data" DATA_OUT_FILE = "out.csv" DATA_STATUS_FILE = "status.json" @@ -387,6 +441,7 @@ ("scipy/stats/_distn_infrastructure.py", "_attach_argparser_methods"), ("plotnine/mapping/_env.py", "eval"), ("cssselect2/compiler.py", "__init__"), + ("_plotly_utils/basevalidators.py", "build_validator"), } ) """The `(file suffix, function)` pairs whose `exec` and `compile` both guards let through when they call them: each @@ -410,11 +465,30 @@ classes, id, attributes and `lang`, nothing else. Only Python objects handed to `compile_selector_list` in place of tokens (a forged `dimension` token whose `int_value` is a string) reach an unquoted slot; the validator admits neither `cssselect2` nor `tinycss2` nor `cairosvg.css`/`cairosvg.parser`, and a lambda the compiler built runs under the same -guard as the plot (no write, process, socket or frame).""" +guard as the plot (no write, process, socket or frame). + +plotly's `InfoArrayValidator.build_validator` (plotly 7.1.0, figure mode) evaluates the name of the validator class an +info array's item needs: `eval(datatype.title().replace("_", "") + "Validator")`, the datatype read from the item's +`valType`. `InfoArrayValidator.__init__` calls it for each item when plotly builds the validator of an info-array +property, lazily, the first time a plot sets one (`xaxis.range`, a `domain`), so every plot that sets an axis range +needs it. The items come from plotly's own schema (`plotly/validators/_validators.json`, read by `ValidatorCache`), +whose info arrays name five types, `angle`, `any`, `enumerated`, `number` and `subplotid`, each a class of the module. +A plot that calls `build_validator` itself controls the text, but the text is title-cased with every underscore +removed and ends in `Validator` (a `#` can comment that out): no keyword but `True`, `False` and `None`, no lowercase +name (no builtin function, no module), no dunder, so it can only read and call capitalised names, the module's 23 +globals of that kind (all validator classes) and the builtins' classes and constants, and whatever that runs stays under +the same guard as the plot.""" IMPORT_FRAME_FILES = ("", "") """The files of the frames that run an import. The plot guard lets an `exec` through when, walking up from its caller, it meets one of these frames before a frame of the plot: a module body the plot imports runs (and builds code: scipy's -distributions, numpy's `f2py` rule tables, `typing.NamedTuple`) with no plot code between it and the import.""" +distributions, numpy's `f2py` rule tables, `typing.NamedTuple`) with no plot code between it and the import. Since +2026-10-31 only an import of the interpreter's own loaders counts (`ImportMachinery.trusts`): a plot drove +`importlib`'s machinery with a loader of its own, or with a stock loader on a file it wrote, and the module body it +supplied read the frames up to the guard and disarmed it (F6 of PB-1, Fable's review).""" +IMPORT_CODE_READERS = ("", "") +"""The files whose `open` events read code for an import: a file loader's `get_data` (source and bytecode) and +zipimport's archive reader. The plot guard refuses such a read outside the interpreter's install (`install_prefixes`), +whoever drives the loader, so no import reads code from the run directory or `/tmp`.""" CODE_OBJECT_EVENTS = ("code.__new__", "function.__new__") """Building a code object or a function from one: the plot guard treats them, and a `__code__` assignment, like `exec`. `types.coroutine` assigns `__code__` when `asyncio` is imported, which a library may do lazily.""" @@ -474,6 +548,7 @@ "sys.monitoring.", "sys.remote_exec", "cpython.remote_debugger_script", + "fcntl.", ) """Audit events `PlotGuard` refuses once it is armed (`ctypes.*` but `NATIVE_CALLERS`), next to `exec` and `CODE_OBJECT_EVENTS` and every `open` that writes anywhere but the plot's PNG and its probe. Unlike data mode it lets @@ -488,7 +563,11 @@ `setprofile` and their `_all_threads` forms raise, per thread) and `sys.monitoring.register_callback`, the one PEP 669 call that raises an event (`use_tool_id` and `set_events` do nothing without a callback). A trace function gets the real frame and walks `f_back`, which no event sees, to `run_plot`'s locals. The signal handler, which gets a frame -too, has no event and is replaced instead (`UNAUDITED_CALLS`). +too, has no event and is replaced instead (`UNAUDITED_CALLS`). `fcntl.*` (since 2026-10-31, the review of PB-1) and +`os.set_blocking`, which raises no event and is replaced in plot mode (`PLOT_UNAUDITED_CALLS`), would change the flags of +a descriptor the plot shares with its parent: stdout and stderr are one open file description across figure mode's +fork, and a plot that made them non-blocking would break the parent's writes. No admitted library calls either while +it draws (the catalogue sweep of 2026-10-31). The guard's state (`armed`, `paused_depth`) stays a plain attribute, settable by anyone holding the guard: CPython gives in-process code no state that a reference cannot reach (a closure cell is writable through `cell_contents`, a @@ -634,11 +713,17 @@ def refused_call(*args, **kwargs): raise PermissionError("blocked by the sandbox: a call the audit hook cannot see") -def disable_unaudited_calls(): - """Replace `UNAUDITED_CALLS` in the modules that are loaded or load without side effects.""" +PLOT_UNAUDITED_CALLS = (("os", "set_blocking"), ("posix", "set_blocking")) +"""What plot mode replaces next to `UNAUDITED_CALLS`: `os.set_blocking` changes a shared descriptor's flags with no +event, as `fcntl.fcntl` does with one (`PLOT_BLOCKED_PREFIXES`).""" + + +def disable_unaudited_calls(calls=None): + """Replace `calls` (`UNAUDITED_CALLS` when None, read at call time) in the modules that are loaded or load without + side effects.""" import importlib - for module_name, name in UNAUDITED_CALLS: + for module_name, name in UNAUDITED_CALLS if calls is None else calls: module = sys.modules.get(module_name) if module is None and module_name.startswith(("posix", "_posix", "signal", "_signal", "_interpreters")): try: @@ -826,10 +911,11 @@ def is_entry(self, event, args): def caller(self): """The frame that raised the event: the first frame above the guard's own (`sys._getframe` raises an audit - event of its own, which the data guard refuses, so the lookup runs paused).""" + event of its own, which the data guard refuses, so the lookup runs paused). The file names are compared as + exact `str` only (`plain_name`): a `str` subclass's `__eq__` would run code with the guard paused.""" with self.paused(): frame = sys._getframe(1) - while frame is not None and frame.f_code.co_filename == __file__: + while frame is not None and plain_name(frame.f_code.co_filename) == __file__: frame = frame.f_back return frame @@ -873,6 +959,160 @@ def __exit__(self, *exc): WRITE_FLAGS = os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_CREAT | os.O_TRUNC +def install_prefixes(): + """The real paths of the interpreter's own install, read from `sysconfig` before a guard is armed: the standard + library (`lib-dynload` in it) and its zipped form on `sys.path`, and the site packages. A read-only image holds them; + the run directory and `/tmp`, which the plot can write, are not among them.""" + import sysconfig + + paths = sysconfig.get_paths() + found = {paths[key] for key in ("stdlib", "platstdlib", "purelib", "platlib") if paths.get(key)} + found.add(os.path.join(sys.base_prefix, "lib", f"python{sys.version_info.major}{sys.version_info.minor}.zip")) + return tuple(sorted({os.path.realpath(path) for path in found})) + + +def exact_path(path): + """`path` (a `str`, `bytes` or path object) as an exact `str`, or None: a path object's `__fspath__` runs here, so + the caller must not have paused its guard, and `str.join` copies a `str` subclass without calling its methods.""" + try: + return "".join((os.fsdecode(path),)) + except Exception: + return None + + +def under_prefixes(path, prefixes): + """Whether `path`, an exact `str`, is one of `prefixes` or lies below one, after `realpath` (a link out of them + does not count).""" + real = os.path.realpath(path) + return any(real == prefix or real.startswith(prefix + os.sep) for prefix in prefixes) + + +TYPE_MRO = type.__dict__["__mro__"] +TYPE_DICT = type.__dict__["__dict__"] +TYPE_FLAGS = type.__dict__["__flags__"] +IMMUTABLE_TYPE = 1 << 8 +"""`Py_TPFLAGS_IMMUTABLETYPE`: a class whose attributes no code can set or delete (`object`).""" +LAZY_CLASS_ENTRIES = frozenset({"__annotate_func__", "__annotations_cache__"}) +"""What Python 3.14 adds to a class's own dict when its annotations are first read (`typing.get_type_hints`, +`inspect`); no attribute lookup of an import reads them.""" +ABSENT = object() +"""A class entry that is not there (`ImportMachinery.intact`).""" + + +def class_entries(cls): + """`cls`'s method resolution order and, for each class in it whose attributes can change, its own entries but + `LAZY_CLASS_ENTRIES`. Read through `type`'s own descriptors, so neither a metaclass nor a property runs; a class + dict's keys are exact `str`, which `type` copies a `str` subclass into.""" + mro = TYPE_MRO.__get__(cls) + mutable = (base for base in mro if not TYPE_FLAGS.__get__(base) & IMMUTABLE_TYPE) + return mro, tuple( + (base, {name: value for name, value in TYPE_DICT.__get__(base).items() if name not in LAZY_CLASS_ENTRIES}) + for base in mutable + ) + + +class ImportMachinery: + """What an import must look like for the plot guard to trust it (F6 of PB-1, Fable's review of 2026-10-31), read + before the guard is armed. + + The import frames a walk up from an event meets first must be the interpreter's own: `_call_with_frames_removed` + called by the `exec_module` or `create_module` of a loader of exactly one of `loader_state`'s types (a subclass + may change any method) whose instance holds nothing but its own state and whose file lies in the install + (`install_prefixes`), or by `BuiltinImporter`, or `FrozenImporter.exec_module` itself. Every other import frame + (a loader the plot built, a finder or path hook it installed that is a builtin such as `exec`) earns no trust. + The frames and their code objects are the interpreter's; the loader is read with the guard armed, so a property or + a `str` subclass of the plot runs judged. + + A class's attributes change without an audit event, so the stock loaders' classes are also read once here, before + arming (`class_entries`), and an import of a loader whose class no longer resolves every attribute to the same + object earns no trust: a `get_code` or `source_to_code` the plot set on the class (Fable's review of the F6 fix, + 2026-10-31), a base class it swapped. Deeper patches of the machinery (the module globals of + `_frozen_importlib_external`, `_bootstrap._call_with_frames_removed`, `builtins.compile`) are F8's kind and out of + this layer's reach; the validator, which refuses every name that reaches them, is the barrier there (the module + docstring's security model). The reads of the code are what `IMPORT_CODE_READERS` and the `import` event of an + extension module bind to the install. + """ + + def __init__(self): + import importlib.machinery as machinery + import zipimport + + bootstrap = sys.modules["_frozen_importlib"] + external = sys.modules["_frozen_importlib_external"] + self.prefixes = install_prefixes() + self.trampoline = bootstrap._call_with_frames_removed.__code__ + self.frozen_exec = bootstrap.FrozenImporter.exec_module.__code__ + # The entries whose loader is `self`, and the static ones of the builtin importer. + self.loader_entries = frozenset( + { + external._LoaderBasics.exec_module.__code__, + machinery.ExtensionFileLoader.create_module.__code__, + machinery.ExtensionFileLoader.exec_module.__code__, + } + ) + self.static_entries = frozenset( + {bootstrap.BuiltinImporter.create_module.__code__, bootstrap.BuiltinImporter.exec_module.__code__} + ) + file_state = (frozenset({"name", "path"}), "path") + self.loader_state = { + machinery.SourceFileLoader: file_state, + machinery.SourcelessFileLoader: file_state, + machinery.ExtensionFileLoader: file_state, + zipimport.zipimporter: (frozenset({"archive", "prefix"}), "archive"), + } + self.loader_classes = {loader: class_entries(loader) for loader in self.loader_state} + + def trusts(self, frame): + """Whether `frame`, the first import frame (`IMPORT_FRAME_FILES`) of a walk up from an event, runs an import + of the interpreter's own loaders (the class docstring).""" + code = frame.f_code + if code is self.frozen_exec: + return True + if code is not self.trampoline: + return False + entry = frame.f_back + if entry is None: + return False + if entry.f_code in self.static_entries: + return True + if entry.f_code not in self.loader_entries: + return False + try: + return self.trusted_loader(entry.f_locals["self"]) + except Exception: + return False + + def trusted_loader(self, loader): + """Whether `loader` is exactly a stock loader of an intact class (`intact`) whose instance holds only its own + state, of exact `str` names, and whose file (or archive) lies in the install.""" + cls = type(loader) + shape = self.loader_state.get(cls) + if shape is None or not self.intact(cls): + return False + names, path_key = shape + state = object.__getattribute__(loader, "__dict__") + if type(state) is not dict or not all(type(name) is str and name in names for name in state): + return False + path = state.get(path_key) + return type(path) is str and under_prefixes(path, self.prefixes) + + def intact(self, cls): + """Whether the stock loader class `cls` resolves every attribute as it did before arming: the same classes in + its method resolution order, and each mutable one's own entries, the same names bound to the same objects + (`class_entries`). C-level reads alone: exact `str` keys, identity comparisons, no plot code.""" + mro, classes = self.loader_classes[cls] + now = TYPE_MRO.__get__(cls) + if len(now) != len(mro) or not all(current is base for current, base in zip(now, mro, strict=True)): + return False + for base, entries in classes: + current = TYPE_DICT.__get__(base) + if sum(1 for name in current if name not in LAZY_CLASS_ENTRIES) != len(entries): + return False + if not all(current.get(name, ABSENT) is value for name, value in entries.items()): + return False + return True + + class PlotGuard(DataGuard): """The audit hook of plot mode: once armed, refuses `PLOT_BLOCKED_PREFIXES` (processes, sockets, native code, filesystem changes), every `open` that writes anywhere but `outputs`, a `__code__` assignment, every `exec` @@ -888,14 +1128,18 @@ def __init__(self, outputs): self.outputs = frozenset(os.path.realpath(path) for path in outputs) self.plot_file = None self.import_ctypes_helpers = ctypes_helper_addresses() + self.machinery = ImportMachinery() def arm(self, entry=None): self.plot_file = entry.co_filename if entry is not None else None + disable_unaudited_calls(PLOT_UNAUDITED_CALLS) super().arm(entry) def refuses(self, event, args): if event == "open": - return self.writes_elsewhere(args) + return self.writes_elsewhere(args) or self.reads_foreign_code(args) + if event == "import": + return self.loads_foreign_file(args) if event == "compile": return False if event == "exec" or event in CODE_OBJECT_EVENTS or sets_code(event, args): @@ -946,16 +1190,37 @@ def lets_frame_out(self): def nearest_origin(self, frame): """Walking up from `frame`: `"plot"` when a frame of the plot file comes first, `"import"` when an import - frame (`IMPORT_FRAME_FILES`) does, None when neither is on the stack.""" + frame (`IMPORT_FRAME_FILES`) of the interpreter's own loaders does (`ImportMachinery.trusts`), `"untrusted"` + when another import frame does, None when none is on the stack.""" while frame is not None: file_name = frame.f_code.co_filename if file_name == self.plot_file: return "plot" if file_name in IMPORT_FRAME_FILES: - return "import" + return "import" if self.machinery.trusts(frame) else "untrusted" frame = frame.f_back return None + def reads_foreign_code(self, args): + """Whether an `open` event is a read of the import machinery's code readers (`IMPORT_CODE_READERS`: a file + loader's `get_data`, zipimport's archive) outside the install (`install_prefixes`). Checked whoever drives the + loader and before its bytes exist, so a stock loader on a file the plot wrote reads nothing (F6 of PB-1).""" + caller = self.caller() + if caller is None or plain_name(caller.f_code.co_filename) not in IMPORT_CODE_READERS: + return False + path = exact_path(args[0]) if args else None + return path is None or not under_prefixes(path, self.machinery.prefixes) + + def loads_foreign_file(self, args): + """Whether an `import` event names a file outside the install: `_imp.create_dynamic` raises it with the path of + the extension module it is about to load, and no `open` event comes before the library's native code runs (an + `ExtensionFileLoader` on a file the plot wrote, measured on 2026-10-31). An `import` statement's event names no + file.""" + if len(args) < 2 or args[1] is None: + return False + path = exact_path(args[1]) + return path is None or not under_prefixes(path, self.machinery.prefixes) + def native_caller(self): """Whether the first frame outside the `ctypes` package that raised a `ctypes.*` event is one of `NATIVE_CALLERS`.""" @@ -972,6 +1237,11 @@ def writes_elsewhere(self, args): in another directory than the working one (`os.open("plot-light.png", flags, dir_fd=fd)`, or `open(..., opener=...)`, whose opener calls `os.open`): an `os.open` write passes only with an absolute path (#12177 review). `open` itself, which savefig and the probe use, takes no `dir_fd`. + + The path becomes an exact `str` before the guard pauses for `realpath`: a path object's `__fspath__` (or a + `bytes` subclass's `decode`) runs while the guard is armed, and `realpath` never calls a method of a `str` + subclass the plot defined, whose `split` and `startswith` ran paused before and wrote files the armed guard + refuses (PB-1's review of the paused paths, 2026-10-31). """ path, mode, flags = (*args, None, None, None)[:3] writes = (isinstance(mode, str) and any(letter in mode for letter in "wax+")) or ( @@ -981,11 +1251,16 @@ def writes_elsewhere(self, args): return False if not isinstance(path, str | bytes | os.PathLike): return True # a file descriptor, or anything else that names no path - if mode is None and not os.path.isabs(os.fsdecode(path)): + try: + # `str.join` copies a `str` subclass's characters into an exact `str` and calls none of its methods. + name = "".join((os.fsdecode(path),)) + except Exception: + return True + if mode is None and not os.path.isabs(name): return True with self.paused(): try: - return os.path.realpath(os.fsdecode(path)) not in self.outputs + return os.path.realpath(name) not in self.outputs except OSError, ValueError, TypeError: return True @@ -1042,18 +1317,29 @@ def preload_data_modules(): return zones +def plain_name(value): + """`value` when it is an exact `str`, else `"?"`: a name a paused guard may compare or format, since a `str` + subclass's methods (`__eq__`, `__format__`) are code the snippet or plot defined. `type()` runs none of them.""" + return value if type(value) is str else "?" + + def print_snippet_error(guard, exc): """The traceback of a snippet error on stderr: its frames' files and lines and the exception line. Written by hand, with the guard paused only to read the traceback's frames: the - `traceback` module reads source lines from files, which the armed guard refuses. + `traceback` module reads source lines from files, which the armed guard refuses. Nothing + the snippet defined may run while it is paused: the traceback is read through + `BaseException`'s own descriptor, which a `__traceback__` of the snippet's exception class + cannot replace (it could, and a property there wrote files the armed guard refuses), and + only exact `str` names are formatted (`plain_name`). The exception's class name and message + may run snippet code, which the armed guard judges. """ lines = ["Traceback (most recent call last):"] with guard.paused(): - tb = exc.__traceback__ + tb = BaseException.__traceback__.__get__(exc) while tb is not None: code = tb.tb_frame.f_code - lines.append(f' File "{code.co_filename}", line {tb.tb_lineno}, in {code.co_name}') + lines.append(f' File "{plain_name(code.co_filename)}", line {tb.tb_lineno}, in {plain_name(code.co_name)}') tb = tb.tb_next kind = type(exc) name = kind.__qualname__ if kind.__module__ == "builtins" else f"{kind.__module__}.{kind.__qualname__}" @@ -1079,10 +1365,21 @@ def _limit(kind: int, value: int) -> None: def set_limits() -> None: + set_run_limits() + set_plot_limits() + + +def set_run_limits() -> None: + """CPU seconds and the largest file, which every process of a run inherits: both steps of figure mode.""" cpu = int(os.environ.get("ANYPLOT_RLIMIT_CPU_S", "60")) fsize = int(os.environ.get("ANYPLOT_RLIMIT_FSIZE_MB", "32")) * 1024 * 1024 _limit(resource.RLIMIT_CPU, cpu) _limit(resource.RLIMIT_FSIZE, fsize) + + +def set_plot_limits() -> None: + """The address space and the processes, each only when set: in figure mode the plot step's alone, because the + screenshot step runs under the JavaScript row's limits, which set neither.""" address_space = os.environ.get("ANYPLOT_RLIMIT_AS_MB") if address_space: _limit(resource.RLIMIT_AS, int(address_space) * 1024 * 1024) @@ -2163,7 +2460,7 @@ def frame_for_readers(): return None -def run_plot(path, guard): +def run_plot(path, guard, armed=None): """Run the plot file as `__main__` under `guard`, the way `runpy.run_path` would. The file is read and compiled before the guard is armed, and the guard lets exactly @@ -2174,7 +2471,8 @@ def run_plot(path, guard): cairocffi, with the same `ldconfig` lookups) when the plot names pygal or cairosvg. No bytecode is written: the guard refuses the writes. The plot's frames carry the path as given, as under `runpy`, so a traceback still names - `plot.py`. + `plot.py`. `armed`, when given, is called with the kernel filter's state once the guard is + armed and before the plot runs (figure mode's report, `plot_step`). """ import builtins import ctypes # noqa: F401 (see the docstring) @@ -2197,6 +2495,8 @@ def run_plot(path, guard): "__cached__": None, } guard.arm(code) + if armed is not None: + armed(guard.exec_deny) exec(code, namespace) @@ -2443,9 +2743,359 @@ def _child_usage(): return {"max_rss_mb": round(usage.ru_maxrss / 1024, 1), "cpu_s": round(usage.ru_utime + usage.ru_stime, 3)} +def main_figure(argv): + """Figure mode (module docstring): ` -- `. + + The start line goes out first and the end line last, as in exec mode; the end line's `max_rss_mb` and `cpu_s` are + both steps' (the largest process and the CPU time of all), `exec_deny` is the plot step's filter (`off` when the + plot never ran armed; the screenshot step has none by design), and `steps` holds each step's own figures. The exit + code is the failed step's: the plot's, 128 plus a signal, `EXEC_DENY_EXIT`, 1 for a missing or refused figure, or + the screenshot command's. Every failure of the plot step and of the harness ends stderr with the final error line + (`figure_error_line`); a failure of the screenshot step ends it with that step's own. + """ + _report("start") + run = {"exec_deny": "off", "steps": {}} + code = 1 + try: + code = run_figure(argv, run) + finally: + try: + _report("end", **_child_usage(), exec_deny=run["exec_deny"], steps=run["steps"]) + except Exception: + pass # a closed stdout must not hide the steps' own exit code + return code + + +def run_figure(argv, run): + """`main_figure` after the start line: the arguments, the shared limits, the plot step, the figure's check and the + screenshot step, each failure with its final error line.""" + if len(argv) < 4 or argv[2] != "--" or argv[0] not in FIGURE_PACKAGES: + packages = "|".join(FIGURE_PACKAGES) + print(f"usage: harness.py {FIGURE_FLAG} <{packages}> -- [args...]", file=sys.stderr) + figure_error_line("harness", "UsageError", None, "usage: the package, the plot file and the command") + return 2 + package, path, command = argv[0], argv[1], argv[3:] + theme = os.environ.get(THEME_ENV, "light") + if theme not in ("light", "dark"): + figure_error_line("harness", "UsageError", None, f"{THEME_ENV} must be light or dark") + return 2 + set_run_limits() + package_dir = find_figure_package(package) + if package_dir is None: + figure_error_line("harness", "PackageMissing", None, f"{package} is not installed where the harness runs") + return 2 + figure = FIGURE_FILE.format(theme=theme) + try: + os.unlink(figure) # only the plot step's own figure reaches the screenshot + except OSError: + pass # missing, or no file at all, which `check_figure` refuses + code = plot_step(path, figure, run) + if code != 0: + return code + refusal = check_figure(figure) + if refusal is not None: + figure_error_line("figure", *refusal) + return 1 + return screenshot_step(command, figure, package_dir, run) + + +def find_figure_package(package): + """The directory of `package` as this interpreter would import it, found without importing it (a top-level name + runs no package code), or None when it is missing: the plot step imports the package from there, and the screenshot + step reads the package's bundled JavaScript from there, so the two versions are one.""" + import importlib.util + + try: + spec = importlib.util.find_spec(package) + except ImportError, ValueError: + return None + if spec is None or not spec.submodule_search_locations: + return None + return os.path.abspath(next(iter(spec.submodule_search_locations))) + + +def plot_step(path, figure, run): + """Fork the plot step, wait for it and judge it: its exit code, 0 when it ran and ended well. + + The child is forked before anything is armed, so this process never is: it starts the screenshot step afterwards. + The child reports through a pipe opened before the fork, its write end non-blocking, so a plot that fills the pipe + cannot hold the child's last write. Its first line is the harness's own and is written before any plot code runs, + so the plot cannot forge it: `armed ` once the guard is armed (`run_plot`'s `armed`), or `refused ` + when a required filter was not in place and the plot never ran. The last, an error record (`error {...}`), is + advisory, like everything the plot shares with the harness. The verdict is `plot_step_verdict`'s. + """ + import time + + read_end, write_end = os.pipe() + os.set_blocking(write_end, False) + os.set_blocking(read_end, False) + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() # the child inherits the buffers; a buffered line must not print twice + except Exception: + pass + started = time.monotonic() + pid = os.fork() + if pid == 0: + os.close(read_end) + plot_child(path, figure, write_end) # never returns + os.close(write_end) + _, status, usage = os.wait4(pid, 0) + run["steps"]["plot"] = _step_usage(usage, time.monotonic() - started) + report = _read_figure_report(read_end) + run["exec_deny"] = report.state + code, error = plot_step_verdict(status, report, path) + if error is not None: + figure_error_line(*error) + return code + + +def plot_step_verdict(status, report, path): + """The plot step's exit code and, when it failed, its final error line's `(kind, class, where, message)`, from its + wait status and its report (`FigureReport`). + + A death by a signal is `crash PlotCrashed`. `EXEC_DENY_EXIT` is the harness's own refusal only after a `refused` + line, which comes before any plot code: a plot that ran (`armed`) and ended with the same code chose it itself + (`sys.exit(3)`, `os._exit(3)`), which is a `plot` error like every other code but 0. + """ + if os.WIFSIGNALED(status): + number = os.WTERMSIG(status) + return 128 + number, ("crash", "PlotCrashed", None, f"the plot step died of {_signal_name(number)}") + code = os.waitstatus_to_exitcode(status) + if report.refused: + message = f"the sandbox could not refuse process starts ({report.state})" + return EXEC_DENY_EXIT, ("harness", "ExecDenyUnavailable", None, message) + if code == 0: + return 0, None + name, line, message = report.record or ("Error", None, f"the plot step ended with {code}") + return code, ("plot", name, None if line is None else f"{os.path.basename(path)}:{line}", message) + + +def plot_child(path, figure, report): + """The plot step, in the forked child: the plot's limits, `PlotGuard` that lets it write `figure` alone, the plot + run as in plot mode, then `os._exit`, so no handler of the parent's stack, no `atexit` callback and no thread of the + plot outlives it. Its exit code is 0, the plot's own `sys.exit` code, `EXEC_DENY_EXIT` (the plot never ran), or 1 + for an exception, whose traceback goes to stderr as the interpreter prints it and whose class, line and message go + to the report.""" + code = 1 + try: + set_plot_limits() + seed_mplconfig() + guard = PlotGuard((figure,)) + try: + run_plot(path, guard, armed=lambda state: _report_line(report, f"armed {state}")) + code = 0 + except ExecDenyUnavailable as exc: + _report_line(report, f"refused {guard.exec_deny}") + print(f"the sandbox could not refuse process starts ({exc}), which every run requires", file=sys.stderr) + code = EXEC_DENY_EXIT + except SystemExit as exc: + status = exc.code + if status is None or (isinstance(status, int) and status & 0xFF == 0): + code = 0 + else: + code = status & 0xFF if isinstance(status, int) else 1 + if not isinstance(status, int): + print(status, file=sys.stderr) + _report_line(report, "error " + json.dumps(_error_record(path, exc, guard, str(status)))) + except BaseException as exc: + try: + sys.__excepthook__(type(exc), exc, exc.__traceback__) + except BaseException: + pass + _report_line(report, "error " + json.dumps(_error_record(path, exc, guard))) + except BaseException: + pass # the harness's own failure: exit code 1, no record + finally: + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except BaseException: + pass + os._exit(code) + + +def _error_record(path, exc, guard, message=None): + """`[class, line, message]` of an exception the plot raised: the class's name when it is an identifier, the line of + the innermost frame of the plot file (a syntax error's own line), and the message. The traceback is read with the + guard paused and through C attributes only (`BaseException.__traceback__`'s own descriptor, which a subclass's + `__traceback__` cannot replace), so no plot code runs while the guard is paused; the name and the message may run + plot code (a metaclass, `__str__`), which the armed guard then judges.""" + try: + name = type(exc).__name__ + name = name if isinstance(name, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", name) else "Error" + except BaseException: + name = "Error" + line = None + try: + if type(exc) in (SyntaxError, IndentationError, TabError) and exc.filename == path: + line = exc.lineno # raised by `compile`, before the guard was armed + else: + with guard.paused(): + tb = BaseException.__traceback__.__get__(exc) + while tb is not None: + if plain_name(tb.tb_frame.f_code.co_filename) == path: + line = tb.tb_lineno + tb = tb.tb_next + except BaseException: + line = None + if message is None: + try: + message = str(exc) + except BaseException: + message = "" + if not isinstance(line, int) or isinstance(line, bool) or line < 1: + line = None + return [name, line, message[:FIGURE_ERROR_CHARS] if isinstance(message, str) else ""] + + +def _report_line(report, line): + """One line to the plot step's report pipe, in one write (a pipe takes up to 4 KiB at once); a full pipe or a + closed descriptor loses the line, never the child.""" + try: + os.write(report, (line[:4000] + "\n").encode("utf-8", errors="replace")) + except OSError: + pass + + +class FigureReport: + """What the plot step's report pipe said (`plot_step`): the filter's `state` (`off` when the plot never ran armed), + whether the arming was `refused`, and the advisory error `record`, `(class, line, message)` or None.""" + + def __init__(self, state="off", refused=False, record=None): + self.state = state + self.refused = refused + self.record = record + + +def _read_figure_report(read_end): + """The plot step's report from its pipe (`parse_figure_report`), which is closed after.""" + data = b"" + try: + while len(data) < FIGURE_REPORT_BYTES: + chunk = os.read(read_end, FIGURE_REPORT_BYTES - len(data)) + if not chunk: + break + data += chunk + except BlockingIOError: + pass + finally: + os.close(read_end) + return parse_figure_report(data) + + +def parse_figure_report(data): + """A `FigureReport` from the pipe's bytes: the first line, the harness's own, says `armed ` or + `refused ` (a state outside `install_exec_deny`'s reads as `unavailable`), and the record is the last line + that is one.""" + lines = data.decode("utf-8", errors="replace").splitlines() + state, refused, first = "off", False, None + if lines and lines[0].startswith(("armed ", "refused ")): + first, state = lines[0].split(" ", 1) + state = state if state in ("on", "unavailable", "unsupported") else "unavailable" + refused = first == "refused" + record = None + for line in reversed(lines[1:] if first is not None else lines): + if not line.startswith("error "): + continue + try: + name, number, message = json.loads(line.split(" ", 1)[1]) + except ValueError, TypeError, RecursionError: + continue + if isinstance(name, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", name) and isinstance(message, str): + number = number if isinstance(number, int) and not isinstance(number, bool) and number > 0 else None + record = (name, number, message) + break + return FigureReport(state, refused, record) + + +def check_figure(figure): + """Why the screenshot step must not take `figure`, as `(class, where, message)`, or None: it is missing, not a + regular file (opened without following a link), empty, or larger than `FIGURE_MAX_BYTES`.""" + import stat + + try: + descriptor = os.open(figure, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + return "NoFigure", None, f"the plot wrote no {figure}" + except OSError as exc: + return "FigureError", None, f"{figure} could not be opened as a regular file ({type(exc).__name__})" + try: + info = os.fstat(descriptor) + finally: + os.close(descriptor) + if not stat.S_ISREG(info.st_mode): + return "FigureError", None, f"{figure} is not a regular file" + if info.st_size == 0: + return "FigureError", None, f"{figure} is empty" + if info.st_size > FIGURE_MAX_BYTES: + return "FigureTooLarge", None, f"{figure} is {info.st_size} bytes, more than the {FIGURE_MAX_BYTES}-byte limit" + return None + + +def screenshot_step(command, figure, package_dir, run): + """Start the screenshot command with `--figure=
` appended, in the sandbox's environment without the plot + step's variables (`FIGURE_PLOT_ONLY_ENV`) and with the package's directory, and return its exit code (128 plus a + signal; 127 when it cannot start). It runs under the limits this process set and nothing else: no guard, no + filter, no address-space or process limit, as the JavaScript row runs.""" + import subprocess + import time + + env = {key: value for key, value in os.environ.items() if key not in FIGURE_PLOT_ONLY_ENV} + env[FIGURE_PACKAGE_ENV] = package_dir + argv = [*command, f"--figure={figure}"] + started = time.monotonic() + try: + # As exec mode starts its child: descriptors past stderr closed, SIGPIPE and SIGXFSZ back to their defaults. + child = subprocess.Popen(argv, env=env) + except OSError as exc: + print(f"{FIGURE_FLAG}: cannot run {argv[0]}: {type(exc).__name__}", file=sys.stderr) + figure_error_line("harness", "SpawnError", None, f"cannot run {argv[0]} ({type(exc).__name__})") + return 127 + # `wait4` for the step's own usage; the exit code goes back to the Popen, which then has nothing left to reap. + _, status, usage = os.wait4(child.pid, 0) + child.returncode = code = os.waitstatus_to_exitcode(status) + run["steps"]["screenshot"] = _step_usage(usage, time.monotonic() - started) + return code if code >= 0 else 128 - code + + +def figure_error_line(kind, name, where, message): + """The final error line of the JavaScript harness (`render.mjs`), for the plot step and the harness: + `render: error: :: `, `-` without a line, the message on one line and at most + `FIGURE_ERROR_CHARS` characters.""" + text = " ".join(str(message).split()) + if len(text) > FIGURE_ERROR_CHARS: + text = text[: FIGURE_ERROR_CHARS - 1] + "…" + try: + print(f"render: error: {kind} {name} {where or '-'}: {text}", file=sys.stderr, flush=True) + except Exception: + pass + + +def _step_usage(usage, wall): + """One step's figures for the end line: its largest process, its CPU time with its waited-for children's, its + wall time.""" + return { + "max_rss_mb": round(usage.ru_maxrss / 1024, 1), + "cpu_s": round(usage.ru_utime + usage.ru_stime, 3), + "wall_s": round(wall, 3), + } + + +def _signal_name(number): + import signal + + try: + return signal.Signals(number).name + except ValueError: + return f"signal {number}" + + def main(argv): if len(argv) >= 2 and argv[1] == EXEC_FLAG: return main_exec(argv[2:]) + if len(argv) >= 2 and argv[1] == FIGURE_FLAG: + return main_figure(argv[2:]) if len(argv) == 3 and argv[1] == DATA_FLAG: return main_data(argv[2]) if len(argv) == 3 and argv[1] == CONVERT_FLAG: diff --git a/agents/renderer/Dockerfile b/agents/renderer/Dockerfile index 614cc517dd4..eab00cbc6e9 100644 --- a/agents/renderer/Dockerfile +++ b/agents/renderer/Dockerfile @@ -65,8 +65,24 @@ COPY pyproject.toml uv.lock README.md ./ # directory, which uvicorn puts on `sys.path`. # The `convert` extra (python-calamine, pyarrow) is the file conversion's readers # (`harness.py --convert`, agents/anyplot/render/convert.py). +# plotly and bokeh (PB-1 of step 4) run in the harness's figure mode: the plot +# writes its figure as JSON, and the JavaScript row's harness draws it with the +# package's own bundled JavaScript (plotly's package_data/plotly.min.js, bokeh's +# server/static/js/*.min.js), so the page's version is the plot's. Their image +# export stacks stay out: kaleido with choreographer and logistro, Selenium with +# webdriver-manager, trio and the WebSocket clients. Each starts a browser, which +# the plot guard refuses anyway, and Selenium alone carries 27 MB of drivers; a +# plot that saves through them fails with the package's own "not installed" +# message instead. What the two extras add is plotly, bokeh, tornado and +# xyzservices. RUN uv venv && uv sync --frozen --no-install-project --extra lib-matplotlib --extra lib-seaborn \ - --extra lib-plotnine --extra lib-altair --extra lib-pygal --extra lib-letsplot --extra convert + --extra lib-plotnine --extra lib-altair --extra lib-pygal --extra lib-letsplot --extra convert \ + --extra lib-plotly --extra lib-bokeh \ + --no-install-package kaleido --no-install-package choreographer --no-install-package logistro \ + --no-install-package simplejson --no-install-package selenium --no-install-package webdriver-manager \ + --no-install-package trio --no-install-package trio-websocket --no-install-package outcome \ + --no-install-package sortedcontainers --no-install-package wsproto --no-install-package websocket-client \ + --no-install-package pysocks # pyarrow ships far more than the converter reads (owner decision of 2026-10-15: slim it). # The conversion needs the core, compute, IPC, Feather and Parquet; it never touches @@ -91,6 +107,24 @@ RUN p=.venv/lib/python3.14/site-packages/pyarrow \ && du -sh "$p" \ && .venv/bin/python -c "import pyarrow.parquet, pyarrow.feather, pyarrow.ipc, pyarrow.compute" +# plotly and bokeh ship JavaScript that neither step of figure mode reads (PB-1): plotly's +# JupyterLab extension and Jupyter widget bundle, bokeh's TypeScript compiler for custom +# models (it runs Node, which plot code cannot start), its ES module sources and the +# unminified twins of its bundles. The page loads the minified bundles alone +# (`automation/js-render/render.mjs`, FIGURES), which stay with MathJax's, and so does every +# Python module. The two packages' 94 MB installed (bytecode included) become 68 MB, and the +# image grows by 69 MB in all (2026-10-31); the imports below and the CI job's plotly and +# bokeh renders fail on a file that is needed. +RUN p=.venv/lib/python3.14/site-packages \ + && du -shc "$p/plotly" "$p/bokeh" \ + && rm -rf "$p/plotly/labextension" "$p/plotly/package_data/widgetbundle.js" \ + "$p/bokeh/server/static/js/compiler.js" "$p/bokeh/server/static/js/lib" \ + && for f in bokeh bokeh-api bokeh-gl bokeh-mathjax bokeh-tables bokeh-widgets; do \ + test -s "$p/bokeh/server/static/js/$f.min.js" && rm "$p/bokeh/server/static/js/$f.js"; \ + done \ + && du -shc "$p/plotly" "$p/bokeh" \ + && .venv/bin/python -c "import plotly.graph_objects, plotly.express, plotly.io, bokeh.plotting, bokeh.embed, bokeh.models, bokeh.layouts" + # The image does not install the project, so its version is written to a file # that `GET /status` reports. RUN .venv/bin/python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" > VERSION diff --git a/agents/renderer/commands.py b/agents/renderer/commands.py index a0c88c1e0f0..c816a4d1e0a 100644 --- a/agents/renderer/commands.py +++ b/agents/renderer/commands.py @@ -14,6 +14,7 @@ | Language | Code file | Command | |---|---|---| | `python` | `plot.py` | `python -I -X disable-remote-debug harness.py plot.py`: the harness runs the plot in process, with the plot guard and the savefig probe | +| `python` (plotly, bokeh) | `plot.py` | `harness.py --figure plot.py -- /opt/node/bin/node /opt/anyplot/js-render/render.mjs --library=`: the plot in a forked, armed child writes `plot-.json`, then the JavaScript harness draws it | | `javascript` | `plot.js` (`plot.tsx` for MUI X) | `harness.py --exec -- /opt/node/bin/node /opt/anyplot/js-render/render.mjs --library= plot.js` | | `r` | `plot.R` | `harness.py --exec -- /opt/R/4.6.1/lib/R/bin/exec/R --no-echo --no-restore --no-save --file=plot.R` | | `julia` | `plot.jl` | `harness.py --exec -- /opt/julia-1.13.2/bin/julia --project=/opt/julia-project --startup-file=no --history-file=no --compiled-modules=strict --heap-size-hint=1536M -L /opt/anyplot/toolchain/warmup.jl plot.jl` | @@ -35,7 +36,10 @@ while the plot code runs in the page, which has no process API. Node runs by its path as well (`NODE_BINARY`), with the harness's packages, the headless shell and the row's fonts named in its environment (`JS_ENV`, JS-1 of step 3), and the harness's sandbox mode switched on there -(JS-2: the page rules, the closed imports, the bounded console, the final error line). A row also says which limits +(JS-2: the page rules, the closed imports, the bounded console, the final error line). plotly's and bokeh's rows +run in two steps (`FIGURE_FLAG`, `RuntimeCommand.figure`, PB-1): the plot under the plot guard, the exec filter and the +Python limits in a child the harness forks before it arms anything, then the JavaScript row's harness, with its +environment and limits, on the figure the plot wrote. A row also says which limits differ from the Python defaults: V8, Chromium and Julia's LLVM reserve address space far beyond their resident memory, so `RLIMIT_AS` is off for the JavaScript and Julia rows and for altair's Python row (vl-convert embeds V8); one Chromium render runs seven @@ -72,6 +76,11 @@ EXEC_FLAG = "--exec" """The harness's exec-mode flag (`harness.EXEC_FLAG`): what follows `--` is the language's command.""" +FIGURE_FLAG = "--figure" +"""The harness's two-step flag (`harness.FIGURE_FLAG`): `--figure plot.py -- `.""" +FIGURE_LIBRARIES: tuple[str, ...] = ("plotly", "bokeh") +"""The Python libraries whose rows run in two steps (`harness.FIGURE_PACKAGES`, pinned equal by a test): the plot +writes its figure as JSON under the plot guard, and the JavaScript row's harness draws it in Chromium.""" LIBRARY_LANGUAGES: Mapping[str, Language] = MappingProxyType( { @@ -260,14 +269,23 @@ class RuntimeCommand: (`executor.stderr_tail`); empty for a row whose tail is the plain last `MAX_STDERR_CHARS` characters. Julia's `ERROR:`: a deep Makie failure prints about 7,500 characters of stack, so the last 2,000 lose the line that names the exception (J2a, 2026-10-29).""" + figure: str = "" + """The package of a two-step row (`FIGURE_FLAG`, plotly and bokeh): the harness runs the plot itself in a forked, + armed child, and `argv` is then the screenshot step's command, which draws the figure the plot wrote with that + package's JavaScript; empty for every other row. `exec_deny`, `rlimit_as` and `rlimit_nproc` are the plot step's; + the screenshot step runs as the JavaScript row does (`harness.FIGURE_PLOT_ONLY_ENV`), and `env` is its + environment, which the sandbox carries after the Python settings.""" def __post_init__(self) -> None: if self.rlimit_as_mb is not None and (not self.rlimit_as or self.rlimit_as_mb <= 0): raise ValueError("a row's own RLIMIT_AS needs rlimit_as on and a positive size") + if self.figure and (self.language != "python" or not self.argv): + raise ValueError("a two-step row runs a Python plot and a screenshot command") @property def in_process(self) -> bool: - """Whether the harness runs the code itself (Python) instead of a child command.""" + """Whether the harness runs the code itself (Python) and nothing else; a two-step row runs it itself and then + the screenshot command (`figure`).""" return not self.argv @@ -296,6 +314,25 @@ def _row(language: Language, library: str) -> RuntimeCommand: rlimit_as_mb=2048, render_p95_s=15.0, ) + if library in FIGURE_LIBRARIES: + # plotly and bokeh draw in a browser, which plot code must not start (the plot guard refuses kaleido's + # and Selenium's starts) and which an armed process cannot start either: the harness forks the plot step + # before it arms anything, the plot writes `plot-.json` under the guard and the exec filter with + # the Python limits, and the harness, never armed, then runs the JavaScript row's harness on that file + # with the JavaScript row's environment (`env`, JS_ENV) and limits: no exec filter, no RLIMIT_AS, no + # RLIMIT_NPROC (PB-1 of step 4 in agentic/specs/261024-agent-all-libraries.md). The page draws the figure + # with the package's own bundled JavaScript, read from the installed Python package. The repair reserve + # stays 15 s: both steps of 20 renders took 1.6 to 2.4 s of wall time in the renderer image at 2 CPUs, + # container start included, and 4.7 to 5.8 s under gVisor's ptrace platform (2026-10-31). + return RuntimeCommand( + language, + code_file, + (NODE_BINARY, JS_HARNESS, f"--library={library}"), + env=JS_ENV, + toolchain=(NODE_BINARY,), + render_p95_s=15.0, + figure=library, + ) # altair renders in vl-convert's embedded V8, which reserves address space the way the JavaScript # row's Chromium does: under RLIMIT_AS up to 64 GiB it aborts at start ("Check failed: Start()", # SIGTRAP), measured on 2026-10-25 through the harness; its resident set is about 270 MiB and the diff --git a/agents/renderer/executor.py b/agents/renderer/executor.py index 8e239ad4931..d635fb9f22f 100644 --- a/agents/renderer/executor.py +++ b/agents/renderer/executor.py @@ -102,7 +102,7 @@ from pathlib import Path from typing import Any, Protocol -from .commands import EXEC_DENY_ENV, EXEC_FLAG, LANGUAGES, RuntimeCommand, command_for, toolchains +from .commands import EXEC_DENY_ENV, EXEC_FLAG, FIGURE_FLAG, LANGUAGES, RuntimeCommand, command_for, toolchains from .settings import RendererSettings from .wire import ( CONVERT_INPUT_FILE, @@ -491,9 +491,13 @@ def sandbox_env( `data_json` says the run directory holds the job's `data.json` (`RenderRequest.data_json`): a JavaScript row then names it in `ANYPLOT_DATA_JSON`, which the harness injects as `window.ANYPLOT_DATA`; no other row reads it. `page` (a page build, `POST /page`) adds `ANYPLOT_PAGE_ONLY=1` to the JavaScript row's environment. + + A two-step row (`RuntimeCommand.figure`, plotly and bokeh) gets a Python row's settings and then its own + environment, the JavaScript row's, which the screenshot step reads; its limits and the filter variable are the + plot step's, and the harness keeps them from the screenshot step (`harness.FIGURE_PLOT_ONLY_ENV`). """ plot: dict[str, str] = {} - if theme is not None and command is not None and not command.in_process: + if theme is not None and command is not None and not command.in_process and not command.figure: plot = {"ANYPLOT_THEME": theme, **command.env} if data_json and command.language == "javascript": plot[DATA_JSON_ENV] = f"{WORKDIR}/{DATA_JSON_FILE}" @@ -549,8 +553,9 @@ def build_argv( """The `sandbox do` command of one theme, of a data run (`theme` None, the harness in data mode) or of a conversion (`convert`, the harness in convert mode); asserted to carry neither `--write` nor `--allow-egress`. - A theme's `command` is its row of the command table: a Python row runs the harness in plot mode on `plot.py`; - any other row runs the harness in exec mode (`EXEC_FLAG`) with the row's own argv and code file. The argv + A theme's `command` is its row of the command table: a Python row runs the harness in plot mode on `plot.py`, a + two-step row (plotly, bokeh) in figure mode (`FIGURE_FLAG`) on `plot.py` with the row's argv as the screenshot + command; any other row runs the harness in exec mode (`EXEC_FLAG`) with the row's own argv and code file. The argv comes from the table alone; the request only selected the row. `data_json` and `page`: see `sandbox_env`.""" argv = [ config.sandbox_binary, @@ -572,6 +577,8 @@ def build_argv( argv += [CONVERT_FLAG, CONVERT_INPUT_FILE] elif theme is None: argv += [DATA_FLAG, SNIPPET_FILE] + elif command is not None and command.figure: + argv += [FIGURE_FLAG, command.figure, command.code_file, "--", *command.argv] elif command is not None and not command.in_process: argv += [EXEC_FLAG, "--", *command.argv, f"{command.code_flag}{command.code_file}"] else: @@ -754,6 +761,8 @@ def available(self) -> bool: ... def languages(self) -> dict[str, bool]: ... + def serves(self, language: str, library: str) -> bool: ... + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: ... async def run_data(self, job: DataRunRequest) -> DataRun: ... @@ -801,6 +810,15 @@ def languages(self) -> dict[str, bool]: for language in LANGUAGES } + def serves(self, language: str, library: str) -> bool: + """Whether a job's row can run here: its language's toolchain (`languages`) and the row's own binaries, Node + for plotly's and bokeh's screenshot step (`RuntimeCommand.figure`) among them, which the Python language's + toolchain does not name.""" + if not self.languages().get(language, False): + return False + row = command_for(language, library) + return all(shutil.which(binary, path=SANDBOX_PATH) is not None for binary in row.toolchain) + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: """Render one theme; a launcher failure before the harness started is retried once.""" command = command_for(job.language, job.library) diff --git a/agents/renderer/main.py b/agents/renderer/main.py index 85765a24da4..7a1cce01e4b 100644 --- a/agents/renderer/main.py +++ b/agents/renderer/main.py @@ -11,7 +11,7 @@ | Route | Body | Result | Errors | |---|---|---|---| -| `POST /render` | `RenderRequest` (`wire.py`): `{job_id, language, library, source, data_csv, themes, timeout_s}`, and for a JavaScript job an optional `data_json` (the table as typed JSON, written as `data.json`) | `RenderResponse`: per theme the exit code, a timeout flag, the stderr tail, the probe, the PNG as base64, the attempts, a reason with the measured value and the limit; the slot wait and total time; `MemAvailable` | `401 unauthenticated`, `403 forbidden`, `409 job_conflict` (another payload for a `job_id` and themes in flight or just finished), `422 invalid_job`, `499 cancelled` or `499 client_gone` (nobody reads it), `500 internal`, `503 busy` (no slot within `RENDERER_SLOT_WAIT_S`), `503 low_memory`, `503 stuck`, `503 sandbox_unavailable`, `503 volume_missing`, `503 runtime_unavailable` (the language's toolchain is not in this image; `GET /status` lists them under `languages`), `503 io` | +| `POST /render` | `RenderRequest` (`wire.py`): `{job_id, language, library, source, data_csv, themes, timeout_s}`, and for a JavaScript job an optional `data_json` (the table as typed JSON, written as `data.json`) | `RenderResponse`: per theme the exit code, a timeout flag, the stderr tail, the probe, the PNG as base64, the attempts, a reason with the measured value and the limit; the slot wait and total time; `MemAvailable` | `401 unauthenticated`, `403 forbidden`, `409 job_conflict` (another payload for a `job_id` and themes in flight or just finished), `422 invalid_job`, `499 cancelled` or `499 client_gone` (nobody reads it), `500 internal`, `503 busy` (no slot within `RENDERER_SLOT_WAIT_S`), `503 low_memory`, `503 stuck`, `503 sandbox_unavailable`, `503 volume_missing`, `503 runtime_unavailable` (the toolchain of the job's row is not in this image: its language's, which `GET /status` lists under `languages`, or the row's own, Node for plotly's and bokeh's screenshot step; `executor.serves`), `503 io` | | `POST /run-data` | `DataRunRequest`: `{job_id, source, timeout_s}` (`timeout_s` at most 30) | `DataRunResponse`: the exit code (`harness.DATA_EXIT`), a timeout flag, a reason with the measured value and the limit, `csv_text` (the `out.csv` the harness wrote, only after exit code 0), the stderr tail, the attempts, the timings, the version, `MemAvailable` | the errors of `POST /render` | | `POST /convert` | `ConvertRequest`: `{job_id, data_base64, sheet?, timeout_s}` (the file at most 5 MiB, `timeout_s` at most 15) | `ConvertResponse`: the exit code (`harness.CONVERT_EXIT`), a timeout flag, a run reason, `refusal` (`wire.CONVERT_REASONS`) with the measured value and the cap, `text` (CSV, or JSON records for a table with list columns, at most 1 MiB) and `meta` (`{kind, format, rows, columns, sheet, sheets, sheet_count, warnings}`) after exit code 0, the attempts, the timings, the version, `MemAvailable` | the errors of `POST /render` | | `POST /page` | `PageRequest`: `{job_id, language: "javascript", library, source, data_json, theme, timeout_s}` (a JavaScript version's run form and typed table) | the self-contained page the harness builds for that theme (`ANYPLOT_PAGE_ONLY=1`: no browser starts), as `text/html; charset=utf-8`, at most `wire.MAX_PAGE_BYTES` | the errors of `POST /render`, `422 invalid_job` for another language, and `422 page_failed` with a `PageFailure` body (the exit code, a reason with the measured value and the limit, the stderr tail with the harness's final error line) for a build that left no page | @@ -324,7 +324,7 @@ async def render(self, job: RenderRequest) -> RenderResponse: async def _render(self, job: RenderRequest, started: float) -> RenderResponse: if self.volume is False: raise Unavailable("volume_missing") - if not self.executor.languages().get(job.language, False): + if not self.executor.serves(job.language, job.library): raise Unavailable("runtime_unavailable") async with self.slot.hold(self.settings.slot_wait_s): waited = time.monotonic() - started @@ -463,7 +463,7 @@ async def _page(self, job: PageRequest, started: float) -> PageResponse: """One page build in its own sandbox, inside the same single slot as the renders.""" if self.volume is False: raise Unavailable("volume_missing") - if not self.executor.languages().get(job.language, False): + if not self.executor.serves(job.language, job.library): raise Unavailable("runtime_unavailable") async with self.slot.hold(self.settings.slot_wait_s): waited = time.monotonic() - started diff --git a/automation/js-render/render.mjs b/automation/js-render/render.mjs index bb215902001..65bee690add 100644 --- a/automation/js-render/render.mjs +++ b/automation/js-render/render.mjs @@ -69,8 +69,9 @@ // agentic/specs/261024-agent-all-libraries.md). // ============================================================================= -import { readFile, writeFile } from "node:fs/promises"; -import { existsSync, realpathSync } from "node:fs"; +import { open, readFile, writeFile } from "node:fs/promises"; +import { constants, existsSync, realpathSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -273,6 +274,26 @@ const BUNDLES = { muix: { framework: "react" }, }; +// ----------------------------------------------------------------------------- +// Figure libraries (figure mode, sandbox only; see renderFigure): the Python +// plot writes its figure as JSON and the page draws it with the Python +// package's own bundled JavaScript, read from the package's directory +// (ANYPLOT_FIGURE_PACKAGE), so the two versions are one; `files` are relative +// to it. bokeh's WebGL, widget and table models live in bundles of their own; +// its MathJax bundle is left out (MathJax off). No snippet runs for these. +// ----------------------------------------------------------------------------- +const FIGURES = { + plotly: { files: ["package_data/plotly.min.js"] }, + bokeh: { + files: [ + "server/static/js/bokeh.min.js", + "server/static/js/bokeh-gl.min.js", + "server/static/js/bokeh-widgets.min.js", + "server/static/js/bokeh-tables.min.js", + ], + }, +}; + // ----------------------------------------------------------------------------- // React (framework=react) bundling — the MUI X path. // @@ -532,11 +553,12 @@ function buildFailure(err, snippetPath) { } function parseArgs(argv) { - const args = { square: false, library: null, file: null }; + const args = { square: false, library: null, file: null, figure: null }; for (const arg of argv) { if (arg === "--square") args.square = true; else if (arg === "--landscape") args.square = false; else if (arg.startsWith("--library=")) args.library = arg.slice("--library=".length); + else if (arg.startsWith("--figure=")) args.figure = arg.slice("--figure=".length); else if (arg.startsWith("--orientation=")) args.square = arg.slice("--orientation=".length) === "square"; else if (!arg.startsWith("--")) args.file = arg; } @@ -821,14 +843,469 @@ const CONTEXT = { refused: { requests: 0, sockets: 0 }, }; +// Sandbox mode: no popup (window.open returns null, so no page outside the +// context's routes), no WebRTC connection and the page's V8 heap capped +// (PAGE_HEAP_MB). Neither the routes nor the content security policy see +// WebRTC: a page's RTCPeerConnection reached a local STUN server over UDP and a +// TURN server over TCP (measured in JS-2). With the forced policy WebRTC uses +// no UDP and TCP only through the proxy, which is a closed port, also for the +// loopback, which a proxy otherwise bypasses. +function launchArgs() { + const args = ["--no-sandbox", "--force-color-profile=srgb"]; + if (SANDBOX) { + args.push( + "--block-new-web-contents", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", + `--proxy-server=${DEAD_PROXY}`, + "--proxy-bypass-list=<-loopback>", + `--js-flags=--max-old-space-size=${PAGE_HEAP_MB}`, + ); + } + return args; +} + +// Sandbox mode: `new Date("2026-03-01T00:00")` is midnight UTC whatever the +// host's zone; no service worker, whose requests no route sees. +const SANDBOX_PAGE_OPTIONS = { timezoneId: "UTC", serviceWorkers: "block", acceptDownloads: false }; + +// Sandbox mode's rules of a new page's context. +async function applyPageRules(page) { + // Every request of the context is aborted (fetch, XHR, sendBeacon, images, + // styles, frames), and every WebSocket closed, which the route does not see. + // setContent loads no URL, so the inlined page itself needs no request. + await page.context().route("**/*", (route) => { + CONTEXT.refused.requests += 1; + return route.abort("blockedbyclient"); + }); + await page.context().routeWebSocket("**", (ws) => { + CONTEXT.refused.sockets += 1; + ws.close(); + }); + // The context script guards the main frame's console; this, every frame's + // after it (setContent writes into the page's first document, which no init + // script reaches). + await page.context().addInitScript({ content: CONSOLE_GUARD }); + page.on("crash", () => { + CONTEXT.crashed = true; + }); +} + +// Sandbox mode: `promise`, or a `timeout` failure once `deadline` has passed. +function boundedBy(deadline) { + return (promise) => { + let timer; + const late = new Promise((_, reject) => { + timer = setTimeout( + () => reject(failure(`the page did not settle within ${SETTLE_TIMEOUT_MS / 1000} s`, "timeout", "TimeoutError")), + Math.max(0, deadline - Date.now()), + ); + }); + return Promise.race([promise, late]).finally(() => clearTimeout(timer)); + }; +} + +// Sandbox mode reads the page's own records cut in the page, as scalars, since +// the snippet can write them: the error record (returned), where the first +// error happened, and how many console errors the guard kept in the page. +async function readPageRecords(page, bounded) { + const inPageError = await bounded( + page.evaluate((limit) => { + try { + const value = window.__anyplotError; + return value == null ? null : String(value).slice(0, limit); + } catch (e) { + return "an error record that cannot be read"; + } + }, PAGE_ERROR_CHARS), + ); + CONTEXT.errorAt = await bounded( + page.evaluate(() => { + try { + const at = window.__anyplotErrorAt; + return at ? { file: String(at.file).slice(0, 200), line: Number(at.line) | 0 } : null; + } catch (e) { + return null; + } + }), + ); + // The console.error calls the guard kept in the page count too. + const seen = await bounded( + page.evaluate(() => { + try { + return Number(window.__anyplotConsoleSeen()) || 0; + } catch (e) { + return 0; + } + }), + ); + CONTEXT.console.total += Math.max(0, seen - CONSOLE_KEEP); + return inPageError; +} + function usage(...lines) { for (const line of lines) console.error(line); if (SANDBOX) console.error(`render: error: harness UsageError -: ${oneLine(lines[0].replace(/^render: /, ""), ERROR_MESSAGE_CHARS)}`); process.exit(2); } +// ----------------------------------------------------------------------------- +// Figure mode: plotly and bokeh (PB-1 of step 4 in +// agentic/specs/261024-agent-all-libraries.md). +// +// The renderer's Python harness runs the plot first, armed, in a child of its +// own, and the plot writes its figure as JSON (`fig.write_json`, +// `bokeh.embed.json_item`); the harness, never armed, then runs this file with +// `--library= --figure=plot-.json` and the package's +// directory in ANYPLOT_FIGURE_PACKAGE (agents/anyplot/render/harness.py, +// figure mode). The figure is the model-written plot's output, so it is data +// here: parsed in Node, carried by a `type="application/json"` element with `<`, +// U+2028 and U+2029 escaped, parsed again in the page and handed to +// `Plotly.newPlot` or `Bokeh.embed.embed_item`, never written into the page as +// markup or script. The page is sandbox mode's, its rules and console guard +// included, under a stricter policy (`figurePolicy`): scripts by the page's +// nonce alone and no `'unsafe-eval'`, so nothing the figure carries runs as +// code; figure mode refuses to run without ANYPLOT_SANDBOX=1. +// +// The canvas: the figure's own size in CSS pixels (plotly's layout width and +// height, bokeh's root width and height; a bokeh layout without one is drawn +// once at scale 1 and measured) picks the orientation, landscape at an aspect +// of 4:3 and wider, square below, and the box of that orientation around it; +// the device scale makes the box exactly 3200 x 1800 or 2400 x 2400 pixels. A +// plotly figure 800 x 450 is drawn at scale 4, 1600 x 900 at 2, a bokeh figure +// 4800 x 2700 at 2/3: the figure's layout and type sizes stay as the code wrote +// them and only the resolution follows the canvas, the way `write_image(width, +// height, scale)` and bokeh's `export_png` scale. A plotly figure that names no +// size gets 1600 x 900, one that names a single side keeps it and gets the +// other at 16:9 (`figureSize`), and one narrower than its box is centred on the +// page's background. +// ----------------------------------------------------------------------------- +const FIGURE_PACKAGE = process.env.ANYPLOT_FIGURE_PACKAGE || null; +// harness.FIGURE_MAX_BYTES, pinned equal by a test: the harness refuses a larger +// figure before this file runs; this is the check behind it. +const FIGURE_MAX_BYTES = 16 * 1024 * 1024; +// The sides a figure may name, in CSS pixels: the device scale stays between 10 +// (320 wide) and 1/2 (6400 wide). +const FIGURE_MIN_SIDE = 320; +const FIGURE_MAX_SIDE = 6400; +const FIGURE_JSON_ID = "anyplot-figure-json"; +const FIGURE_ID = "anyplot-figure"; +// plotly.js's notice in place of a WebGL trace when the page has no WebGL, as +// the headless shell has none: a render that shows it fails as `draw NoWebGL` +// instead of passing with the notice for a plot. +const PLOTLY_NO_WEBGL = "WebGL is not supported by your browser"; + +// A figure page's policy: sandbox mode's, with the scripts admitted by a nonce +// of the page instead of 'unsafe-inline'. Every script of a figure page is the +// harness's own, so markup the figure carries (a bokeh Div's ``, +// measured running under 'unsafe-inline') runs no handler and no +// `javascript:` URL; no 'unsafe-eval' either, so CustomJS models do not compile. +function figurePolicy(nonce) { + const policy = CONTENT_SECURITY_POLICY.replace("script-src 'unsafe-inline'", `script-src 'nonce-${nonce}'`); + if (policy === CONTENT_SECURITY_POLICY) throw failure("render: the figure policy lost its script source", "harness", "PolicyError"); + return policy; +} +// The pixels of each canvas (ORIENTATIONS at DEVICE_SCALE_FACTOR). +const CANVAS_PIXELS = Object.fromEntries( + Object.entries(ORIENTATIONS).map(([name, css]) => [name, [css.cssWidth * DEVICE_SCALE_FACTOR, css.cssHeight * DEVICE_SCALE_FACTOR]]), +); + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +// The figure file as the page carries it, and parsed: opened without following +// a link, at most FIGURE_MAX_BYTES, JSON of the library's shape. +async function readFigure(file, library) { + let handle; + try { + handle = await open(path.resolve(process.cwd(), file), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + } catch (err) { + throw failure(`the figure ${file} could not be opened as a regular file (${err.code || err.name})`, "figure", "FigureError"); + } + let text; + try { + const info = await handle.stat(); + if (!info.isFile()) throw failure(`the figure ${file} is not a regular file`, "figure", "FigureError"); + if (info.size > FIGURE_MAX_BYTES) { + throw failure(`the figure ${file} is ${info.size} bytes, more than the ${FIGURE_MAX_BYTES}-byte limit`, "figure", "FigureTooLarge"); + } + text = await handle.readFile("utf-8"); + } finally { + await handle.close(); + } + if (Buffer.byteLength(text, "utf-8") > FIGURE_MAX_BYTES) { + throw failure(`the figure ${file} grew past the ${FIGURE_MAX_BYTES}-byte limit while it was read`, "figure", "FigureTooLarge"); + } + let figure; + try { + figure = JSON.parse(text); + } catch (err) { + throw failure(`the figure ${file} is not JSON: ${err.message}`, "figure", "FigureError"); + } + const shaped = + library === "plotly" + ? isRecord(figure) && Array.isArray(figure.data) && (figure.layout === undefined || isRecord(figure.layout)) + : isRecord(figure) && + typeof figure.root_id === "string" && + isRecord(figure.doc) && + Array.isArray(figure.doc.roots) && + figure.doc.roots.some((root) => isRecord(root) && root.id === figure.root_id); + if (!shaped) { + const expected = library === "plotly" ? '{"data": [...], "layout": {...}}' : "bokeh.embed.json_item's {root_id, doc}"; + throw failure(`the figure ${file} is not a ${library} figure (${expected})`, "figure", "FigureError"); + } + // Inside valid JSON `<`, U+2028 and U+2029 occur only in strings, where the escape is the same character. + return { figure, text: text.replace(SCRIPT_BREAKING, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`) }; +} + +// The figure's own size in CSS pixels, or null when only drawing it tells +// (a bokeh root without both sides of its own). A plotly layout that names one +// side keeps it and gets the other at the landscape aspect, 16:9 (width 1200 +// gets height 675, height 600 gets width 1066.67), so the side the code chose +// is the one drawn. +function figureSize(library, figure) { + if (library === "plotly") { + const layout = figure.layout ?? {}; + const { cssWidth, cssHeight } = ORIENTATIONS.landscape; + const width = typeof layout.width === "number" ? layout.width : null; + const height = typeof layout.height === "number" ? layout.height : null; + if (width !== null && height !== null) return checkedSize({ width, height }); + if (width !== null) return checkedSize({ width, height: (width * cssHeight) / cssWidth }); + if (height !== null) return checkedSize({ width: (height * cssWidth) / cssHeight, height }); + return { width: cssWidth, height: cssHeight }; + } + const root = figure.doc.roots.find((model) => isRecord(model) && model.id === figure.root_id); + const attributes = isRecord(root.attributes) ? root.attributes : {}; + if (typeof attributes.sizing_mode === "string" && attributes.sizing_mode !== "fixed") { + // A responsive root fills its container: the landscape box. + return { width: ORIENTATIONS.landscape.cssWidth, height: ORIENTATIONS.landscape.cssHeight }; + } + if (typeof attributes.width === "number" && typeof attributes.height === "number") { + return checkedSize({ width: attributes.width, height: attributes.height }); + } + return null; +} + +function checkedSize(size) { + const fits = (side) => Number.isFinite(side) && side >= FIGURE_MIN_SIDE && side <= FIGURE_MAX_SIDE; + if (!fits(size.width) || !fits(size.height)) { + const shown = `${Number(size.width).toFixed(0)} x ${Number(size.height).toFixed(0)}`; + throw failure(`the figure is ${shown}, outside ${FIGURE_MIN_SIDE} to ${FIGURE_MAX_SIDE} CSS pixels a side`, "figure", "FigureError"); + } + return size; +} + +// The canvas around a figure: the orientation, its box in CSS pixels (the +// figure's own size at the box's aspect, a landscape width a multiple of 16, so +// both sides are whole pixels) and the device scale that makes the box the +// canvas's pixels. +function figureCanvas(size) { + if (3 * size.width < 4 * size.height) { + const side = Math.ceil(Math.max(size.width, size.height)); + return { orientation: "square", cssWidth: side, cssHeight: side, scale: CANVAS_PIXELS.square[0] / side }; + } + const width = 16 * Math.ceil(Math.max(size.width, (size.height * 16) / 9) / 16); + return { orientation: "landscape", cssWidth: width, cssHeight: (width * 9) / 16, scale: CANVAS_PIXELS.landscape[0] / width }; +} + +// The page's own drawing script: parse the figure from its JSON element, draw +// it with the library, and set the ready signal once it is drawn and the fonts +// are in, or the error record when the library refuses it. +function figureScript(library, size) { + const draw = + library === "plotly" + ? `var layout = figure.layout || {}; + layout.width = ${size.width}; + layout.height = ${size.height}; + layout.autosize = false; + var config = { staticPlot: true, displayModeBar: false, typesetMath: false, responsive: false, showTips: false }; + drawn = Plotly.newPlot(target, figure.data, layout, config);` + : `drawn = Bokeh.embed.embed_item(figure, ${JSON.stringify(FIGURE_ID)});`; + return ` +(function () { + var target = document.getElementById(${JSON.stringify(FIGURE_ID)}); + var failed = function (err) { window.__anyplotError = String(err && err.stack ? err.stack : err); }; + var drawn; + try { + var figure = JSON.parse(document.getElementById(${JSON.stringify(FIGURE_JSON_ID)}).textContent); + ${draw} + } catch (err) { + failed(err); + return; + } + Promise.resolve(drawn) + .then(function () { return document.fonts ? document.fonts.ready : null; }) + .then(function () { + requestAnimationFrame(function () { requestAnimationFrame(function () { window.__anyplotReady = true; }); }); + }, failed); +})(); +//# sourceURL=anyplot-figure.js`; +} + +// The figure's page: sandbox mode's policy, console guard and error records, +// the library's bundles, the figure as data and the drawing script. `canvas` +// null builds the measuring page, whose container takes the figure's own size. +function buildFigureHtml({ tokens, library, bundles, figureText, size, canvas }) { + const container = canvas + ? `width: ${canvas.cssWidth}px; height: ${canvas.cssHeight}px; display: flex; align-items: center; justify-content: center;` + : "display: inline-block;"; + const nonce = randomBytes(18).toString("base64"); + const scripts = bundles + .map( + (source, index) => + ``, + ) + .join("\n"); + return ` + + + + + +anyplot + + + +
+ + +${scripts} + + +`; +} + +// One page of a figure: sandbox mode's options and rules, the canvas's +// viewport and scale, drawn until the ready signal or the error record, both +// bounded by one deadline; a page error fails it as in a snippet's render. +async function drawFigurePage(browser, html, { width, height, scale, theme }) { + const page = await browser.newPage({ viewport: { width, height }, deviceScaleFactor: scale, ...SANDBOX_PAGE_OPTIONS }); + await applyPageRules(page); + await page.emulateMedia({ reducedMotion: "reduce", colorScheme: theme }); + page.on("console", (msg) => { + if (msg.type() !== "error") return; + CONTEXT.console.total += 1; + if (CONTEXT.console.kept.length < CONSOLE_KEEP) CONTEXT.console.kept.push(oneLine(msg.text(), CONSOLE_MESSAGE_CHARS)); + }); + page.on("pageerror", (err) => { + if (CONTEXT.pageError === null) CONTEXT.pageError = cappedPageError(err); + }); + const bounded = boundedBy(Date.now() + SETTLE_TIMEOUT_MS); + await bounded(page.setContent(html, { waitUntil: "load" })); + await bounded(page.waitForFunction(() => window.__anyplotReady === true || window.__anyplotError != null, null, { timeout: 0 })); + const inPageError = await readPageRecords(page, bounded); + if (CONTEXT.pageError || inPageError) { + CONTEXT.inPageError = inPageError; + throw failure(`render: the figure failed to draw:\n${String(CONTEXT.pageError?.stack || inPageError)}`, "page", "Error"); + } + const drew = await bounded( + page.evaluate((id) => { + const marks = (root) => { + if (root.querySelector("canvas, svg, img")) return true; + for (const element of root.querySelectorAll("*")) if (element.shadowRoot && marks(element.shadowRoot)) return true; + return false; + }; + const target = document.getElementById(id); + return !!target && marks(target); + }, FIGURE_ID), + ); + if (!drew) throw failure("render: the figure drew nothing into #container (no / mounted).", "draw", "NoOutput"); + const noWebGl = await bounded( + page.evaluate( + ([id, notice]) => Array.from(document.getElementById(id).querySelectorAll("p")).some((p) => p.textContent.startsWith(notice)), + [FIGURE_ID, PLOTLY_NO_WEBGL], + ), + ); + if (noWebGl) throw failure("render: a WebGL trace cannot draw: the page has no WebGL", "draw", "NoWebGL"); + return { page, bounded }; +} + +async function renderFigure(args) { + const library = args.library; + if (!Object.hasOwn(FIGURES, library ?? "")) { + usage(`render: --figure is for ${Object.keys(FIGURES).join(" and ")}, not "${library}"`); + } + const info = FIGURES[library]; + if (args.figure === null || args.file !== null) { + usage(`render: ${library} draws a figure: pass --figure= and no code file`); + } + if (!SANDBOX) usage("render: --figure needs ANYPLOT_SANDBOX=1"); + if (PAGE_ONLY) usage("render: ANYPLOT_PAGE_ONLY=1 builds no page for a figure"); + if (!FIGURE_PACKAGE) usage("render: --figure needs ANYPLOT_FIGURE_PACKAGE, the package's directory"); + + const theme = process.env.ANYPLOT_THEME === "dark" ? "dark" : "light"; + const tokens = themeTokens(theme); + CONTEXT.snippetName = path.basename(args.figure); + CONTEXT.snippetPath = path.resolve(process.cwd(), args.figure); + CONTEXT.bundleName = `anyplot-${library}.js`; + + const bundles = []; + for (const file of info.files) { + const bundlePath = path.join(FIGURE_PACKAGE, file); + if (!existsSync(bundlePath)) usage(`render: bundle missing: ${bundlePath}`); + bundles.push(await readFile(bundlePath, "utf-8")); + } + const { figure, text } = await readFigure(args.figure, library); + let size = figureSize(library, figure); + + const browser = await chromium.launch({ args: launchArgs() }); + try { + if (size === null) { + // A bokeh layout: drawn once at scale 1 on a page as large as a figure + // may be, where the container takes its size. + const html = buildFigureHtml({ tokens, library, bundles, figureText: text, size: null, canvas: null }); + const measuring = { width: FIGURE_MAX_SIDE, height: FIGURE_MAX_SIDE, scale: 1, theme }; + const { page, bounded } = await drawFigurePage(browser, html, measuring); + const box = await bounded( + page.evaluate((id) => { + const rect = document.getElementById(id).getBoundingClientRect(); + return { width: Math.ceil(rect.width), height: Math.ceil(rect.height) }; + }, FIGURE_ID), + ); + await page.close(); + size = checkedSize(box); + } + const canvas = figureCanvas(size); + const html = buildFigureHtml({ tokens, library, bundles, figureText: text, size, canvas }); + const shape = { width: canvas.cssWidth, height: canvas.cssHeight, scale: canvas.scale, theme }; + const { page, bounded } = await drawFigurePage(browser, html, shape); + const png = await bounded(page.locator("#container").screenshot()); + const [width, height] = [png.readUInt32BE(16), png.readUInt32BE(20)]; + const [expectedWidth, expectedHeight] = CANVAS_PIXELS[canvas.orientation]; + if (width !== expectedWidth || height !== expectedHeight) { + throw failure(`the screenshot is ${width}x${height}, not the ${canvas.orientation} canvas's ${expectedWidth}x${expectedHeight}`, "harness", "CanvasError"); + } + const pngOut = path.resolve(process.cwd(), `plot-${theme}.png`); + await writeFile(pngOut, png); + printSummary(CONTEXT); + console.log( + `render: ${library} ${theme} ${canvas.orientation} figure ${size.width}x${size.height} at scale ${canvas.scale.toFixed(3)} → ${pngOut} (${width}×${height})`, + ); + } finally { + await browser.close(); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); + if (args.figure !== null || Object.hasOwn(FIGURES, args.library ?? "")) { + await renderFigure(args); + return; + } if (!args.file) { usage("usage: node automation/js-render/render.mjs [--library=] [--square]"); } @@ -848,6 +1325,9 @@ async function main() { // Library id: explicit flag wins, else the file stem (chartjs.js → chartjs). const library = args.library || path.basename(filePath).replace(/\.(js|mjs|cjs|ts|tsx)$/i, ""); + if (Object.hasOwn(FIGURES, library)) { + usage(`render: ${library} draws a figure, not a code file: pass --library=${library} --figure=`); + } const bundleInfo = BUNDLES[library]; if (!bundleInfo) { usage( @@ -897,52 +1377,14 @@ async function main() { } if (!NO_HTML) await writeFile(htmlOut, html, "utf-8"); - // Sandbox mode: no popup (window.open returns null, so no page outside the - // context's routes), no WebRTC connection and the page's V8 heap capped - // (PAGE_HEAP_MB). Neither the routes nor the content security policy see - // WebRTC: a page's RTCPeerConnection reached a local STUN server over UDP and a - // TURN server over TCP (measured in JS-2). With the forced policy WebRTC uses - // no UDP and TCP only through the proxy, which is a closed port, also for the - // loopback, which a proxy otherwise bypasses. - const launchArgs = ["--no-sandbox", "--force-color-profile=srgb"]; - if (SANDBOX) { - launchArgs.push( - "--block-new-web-contents", - "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", - `--proxy-server=${DEAD_PROXY}`, - "--proxy-bypass-list=<-loopback>", - `--js-flags=--max-old-space-size=${PAGE_HEAP_MB}`, - ); - } - const browser = await chromium.launch({ args: launchArgs }); + const browser = await chromium.launch({ args: launchArgs() }); try { const page = await browser.newPage({ viewport: { width: css.cssWidth, height: css.cssHeight }, deviceScaleFactor: DEVICE_SCALE_FACTOR, - // Sandbox mode: `new Date("2026-03-01T00:00")` is midnight UTC whatever the - // host's zone; no service worker, whose requests no route sees. - ...(SANDBOX ? { timezoneId: "UTC", serviceWorkers: "block", acceptDownloads: false } : {}), + ...(SANDBOX ? SANDBOX_PAGE_OPTIONS : {}), }); - if (SANDBOX) { - // Every request of the context is aborted (fetch, XHR, sendBeacon, images, - // styles, frames), and every WebSocket closed, which the route does not see. - // setContent loads no URL, so the inlined page itself needs no request. - await page.context().route("**/*", (route) => { - CONTEXT.refused.requests += 1; - return route.abort("blockedbyclient"); - }); - await page.context().routeWebSocket("**", (ws) => { - CONTEXT.refused.sockets += 1; - ws.close(); - }); - // The context script guards the main frame's console; this, every frame's - // after it (setContent writes into the page's first document, which no init - // script reaches). - await page.context().addInitScript({ content: CONSOLE_GUARD }); - page.on("crash", () => { - CONTEXT.crashed = true; - }); - } + if (SANDBOX) await applyPageRules(page); // Static screenshot of an animated chart must catch the *final* frame, never // a mid-tween one. The library prompts already mandate disabling animation @@ -976,18 +1418,7 @@ async function main() { // Sandbox mode bounds every answer the page gives by one deadline // (SETTLE_TIMEOUT_MS), loading the page included: a snippet whose inline // script never returns, or a page that never settles, fails as `timeout`. - const deadline = Date.now() + SETTLE_TIMEOUT_MS; - const bounded = (promise) => { - if (!SANDBOX) return promise; - let timer; - const late = new Promise((_, reject) => { - timer = setTimeout( - () => reject(failure(`the page did not settle within ${SETTLE_TIMEOUT_MS / 1000} s`, "timeout", "TimeoutError")), - Math.max(0, deadline - Date.now()), - ); - }); - return Promise.race([promise, late]).finally(() => clearTimeout(timer)); - }; + const bounded = SANDBOX ? boundedBy(Date.now() + SETTLE_TIMEOUT_MS) : (promise) => promise; await bounded(page.setContent(html, { waitUntil: "load" })); @@ -1027,41 +1458,7 @@ async function main() { // main().catch turns the throw into a non-zero exit. // Sandbox mode reads the page's own records cut in the page, as scalars, since // the snippet can write them. - const inPageError = SANDBOX - ? await bounded( - page.evaluate((limit) => { - try { - const value = window.__anyplotError; - return value == null ? null : String(value).slice(0, limit); - } catch (e) { - return "an error record that cannot be read"; - } - }, PAGE_ERROR_CHARS), - ) - : await page.evaluate(() => window.__anyplotError); - if (SANDBOX) { - CONTEXT.errorAt = await bounded( - page.evaluate(() => { - try { - const at = window.__anyplotErrorAt; - return at ? { file: String(at.file).slice(0, 200), line: Number(at.line) | 0 } : null; - } catch (e) { - return null; - } - }), - ); - // The console.error calls the guard kept in the page count too. - const seen = await bounded( - page.evaluate(() => { - try { - return Number(window.__anyplotConsoleSeen()) || 0; - } catch (e) { - return 0; - } - }), - ); - CONTEXT.console.total += Math.max(0, seen - CONSOLE_KEEP); - } + const inPageError = SANDBOX ? await readPageRecords(page, bounded) : await page.evaluate(() => window.__anyplotError); if (pageError || inPageError) { CONTEXT.inPageError = inPageError; throw failure(`render: the snippet threw while drawing:\n${String(pageError?.stack || pageError || inPageError)}`, "page", "Error"); diff --git a/changelog.d/pb1-two-step-exec.md b/changelog.d/pb1-two-step-exec.md new file mode 100644 index 00000000000..efe5495da38 --- /dev/null +++ b/changelog.d/pb1-two-step-exec.md @@ -0,0 +1,37 @@ +### Added + +- **The renderer draws plotly and bokeh figures in two steps, not served yet.** + The render harness forks the plot before it arms anything: the plot runs + under the plot guard and the exec filter with the Python limits and may write + only its figure as JSON, and the harness, never armed, then hands the figure + to the JavaScript row's browser harness, which draws it as data with the + installed package's own plotly.js or BokehJS in the sandboxed page, with a + stricter policy that runs no script the figure carries. The figure keeps the + size its code gave it, and the scale makes the image exactly 3200x1800 or + 2400x2400, so the catalogue's layouts look as published. Failures in either + step end with the same final error line as a JavaScript render. The renderer + image now carries plotly and bokeh without their kaleido and Selenium export + stacks. The agent chat does not offer plotly or bokeh yet. (#12484) + +### Fixed + +- **No user code runs while a render or data guard is paused.** The harness + pauses its guard briefly to read a traceback or resolve a path, and code the + snippet defined could run in that moment and write files the guard refuses: + an exception class's own `__traceback__` property while a data run printed + its traceback, and a path that subclasses `str` while a plot's file write was + checked. The harness now reads the traceback through Python's own + descriptor, copies a path to a plain string before the check, and compares + only plain strings while paused. (#12484) +- **A plot can no longer run code of its own as an import.** The render guard + trusted any code the import machinery ran, so a plot that drove it, with a + loader of its own or a stock loader on a file it had written, ran a module + body that read the guard's frames and switched it off. The guard now trusts + an import only when the interpreter's own loader runs it, with its class as + it was before the plot started, refuses every code + read and every native module load outside the installed packages, and + refuses changing the flags of the output it shares with the screenshot step. + The plot validator also refuses the package-data reader, the import spec + builders and the dynamic class builder that matplotlib's bundled dateutil and + the standard library's `types` module hand out, found by a new walk over + everything the validator admits. (#12484) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index fa64b888d12..dfc05225446 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -383,7 +383,7 @@ Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` wi **Label density and number formats.** Two findings of the live tests of 2026-10-14/15 became measured gates. The catalogue draws some elements once per mark of its few example marks: bar-grouped puts a value label, a dot and a drop shadow on each of its 12 bars, and the adapter kept them on the 52 bars of 26 regions times two series, because "keep the same style" and the reviewer's house-style exemption covered them. The harness's `density_report` (probe field `density`, per axes) counts the bars and points, the texts at a data coordinate whose anchor sits at a bar's end, inside a bar's segment (`bar_label(label_type="center")`) or on a point (`ax.annotate`, `ax.bar_label`, `ax.text`; heatmap cell text and callouts in axes coordinates are not value labels, and neither is a text matplotlib clips away in a zoomed view), which of them are numbers, the dots on bar ends (scatter marks or `plot(..., "o")` markers) and the shadowed bars, and measures the median bar label width against the space between neighbouring labelled bars (unlabelled bars between them, such as an offset shadow series, do not shrink it). Bar ends come from the bars' data coordinates, so an inverted axis works. G11 writes a VQ-03 line (owner, 2026-10-15: the thresholds stay) when more than `MAX_VALUE_LABELS` (25) bar positions carry a label (a stacked bar's segment labels count once), when more than 25 numeric labels sit on points (named points, the point of scatter-annotated, never count), from six labels on when a bar label is wider than 90 % of the space per bar, or when more than `MAX_DECORATIONS` (25) dots or shadows are drawn. The adapter prompt and the reviewer's and scorer's house-style exemption now say that an element whose count grows with the data is not house style. The rule itself has one source shared with the catalogue pipeline: the "Per-mark Labels and Decorations" section of `prompts/default-style-guide.md` and the VQ-03 paragraph of `prompts/quality-criteria.md`, which the adapter reads verbatim; `adapter.md` only points at it. Second, number formats followed English whatever the plot language ("6.0 M" and "5.30" in German plots), and `locale` is a banned import. `language.NUMBER_FORMATS` is the server's table of the decimal mark, the thousands mark and the words for large numbers per language of `ENGLISH_NAMES` (`de`: `,`, `.`, Tsd./Mio./Mrd.); the adapter's static instruction carries it with the rule to format through a `FuncFormatter` or an f-string whose marks are swapped through a placeholder (`.replace(",", "_").replace(".", DECIMAL).replace("_", THOUSANDS)`), exact for the space-grouped languages too. A region such as `de-CH` with its apostrophe (7'184'003) is a follow-up of its own setting (owner, 2026-10-15). G12 reads the probe's `labels` field (the drawn texts by kind, from block F) and its `tick_axes` field (`harness.tick_axes`: whether each axis's tick labels are numbers, dates, categories or labels the code set itself), and writes a VQ-06 line when an annotation, or a tick label of a numeric axis, uses the other decimal mark, mathtext read as drawn: a single point followed by other than three digits in a decimal-comma language (`5.30`, `6.0 M`, `5.30 €`), a point after a whole part of 0 (`0.125`), a point after a comma, or commas grouping thousands; dates (a zero-padded `14.10` in an annotation reads as a day and month), years, version strings, titles, legend entries and tick labels the code set itself never count, and no label text reaches the line. -Later languages reuse the CI commands: R's binary on `plot.R` (since 2026-10-29 the row runs `/opt/R/4.6.1/lib/R/bin/exec/R --no-echo --no-restore --no-save --file=plot.R` with the environment R's `bin/R` script would set and `LANG=C.UTF-8`, not `Rscript`, whose wrapper starts would die under the exec filter; the harness preloads the filter into the child, `agents/renderer/execdeny/execdeny.c`, and refuses the run, exit 3, unless the library reported `on`; the agents service sends R jobs since 2026-10-29, through `RRuntime`); Julia's launcher on `plot.jl` (since 2026-10-29 the row runs `/opt/julia-1.13.2/bin/julia --project=/opt/julia-project --startup-file=no --history-file=no --compiled-modules=strict --heap-size-hint=1536M plot.jl` under the preloaded exec filter, with a stacked `JULIA_DEPOT_PATH=/tmp/depot:/opt/julia-depot`, the `JULIA_CPU_TARGET` the toolchain image precompiled the catalogue's `Project.toml` and `Manifest.toml` for, one thread and the offline switches; under `strict` a package image that does not load fails the render in about a second instead of compiling under its timeout; since J1.1 the row adds `-L /opt/anyplot/toolchain/warmup.jl` before `plot.jl`, which loads the toolchain's precompile workload, a package whose image holds the native code of the calls a plot makes most and of the data loader's read, compiled with CSV, DataFrames and TimeZones loaded, and cut an adapted render from 15.9 to 5.0 s locally; the agents service sends Makie jobs since 2026-10-30, through `JuliaRuntime`, and the row keeps Julia's first `ERROR:` line in front of the stderr tail); Node by its path on the catalogue's own harness (since 2026-10-30 the row runs `/opt/node/bin/node /opt/anyplot/js-render/render.mjs --library= plot.js`, `plot.tsx` for MUI X, without the exec filter, because Node starts esbuild and Chromium and Chromium starts its own processes, while the plot code runs in the page, which has no process API; the toolchain image carries Node 22, the pipeline's major, the repository's `package-lock.json` installed with `npm ci` under `/opt/anyplot/node_modules`, Playwright's Chromium headless shell alone under `/opt/pw-browsers`, and Liberation Sans, Serif and Mono for this row alone through its own `FONTCONFIG_FILE`, so the catalogue's page font stack resolves as on the pipeline's runner while the other rows keep their fonts; the agents service sends Chart.js, D3, ECharts and Highcharts jobs since 2026-10-30, through `JavaScriptRuntime`, each with its table as `data_json`; since 2026-10-30 the row's environment switches on the harness's sandbox mode, `ANYPLOT_SANDBOX=1` with `ANYPLOT_NO_HTML=1`, which the catalogue pipeline never sets and without which the harness renders as before: every request of the page aborted by a route and refused by a content security policy (`default-src 'none'`, inline scripts and styles, `data:` and `blob:` images, `data:` fonts, no form submission, no ``), every WebSocket closed, `window.open` returning null, WebRTC connecting nowhere, the zone UTC, the page's V8 heap capped at 1 GiB, the console kept to the first five errors, MUI X's imports closed to the image's allowlist of bare specifiers, and one final stderr line `render: error: :: ` that the renderer's 2,000-character tail always keeps; a job may carry the table as typed JSON, `data_json`, which the renderer writes as `data.json` and the harness injects as `window.ANYPLOT_DATA`, one object per row in column order); bokeh and plotly render through the same Chromium in a two-step exec mode, because a plot-mode harness cannot start a program once its guard is armed (the plot guard refuses a browser started from plot code; `agentic/specs/261024-agent-all-libraries.md`). Since 2026-10-24 the renderer holds these commands in one table (`agents/renderer/commands.py`, keyed by the job's `(language, library)`, the library checked against the catalogue's fifteen) and never takes a command from a request; a non-Python row runs through the harness's exec mode (`harness.py --exec -- plot.`: the start line, the limits set and inherited, the end line from the child's rusage), so the launcher retry, the watchdog and the kill path hold for every language; a row says which limits differ (`RLIMIT_AS` off for the JavaScript and Julia rows, whose V8, Chromium and LLVM reserve address space far beyond their resident memory, and `RLIMIT_NPROC` off for the JavaScript row alone, whose one Chromium render runs 77 tasks; since 2026-10-25 `RLIMIT_AS` is off for altair's Python row too, because vl-convert's embedded V8 aborts at start under any limit up to 64 GiB while its resident set is about 270 MiB), which limit it sets in place of the configured one (since 2026-10-26 lets-plot's Python row has its own 2048 MiB, `RuntimeCommand.rlimit_as_mb`, because its Kotlin/Native rasteriser aborts every 3200x1800 export under the configured 1 GiB while 2048 MiB still stops a runaway scale, which a 6400x3600 export shows at 2580 MiB; the renderer's switch still rules, so a configured limit of 0 leaves every row unlimited), which environment it adds (lets-plot's: Arrow's system allocator, `ARROW_DEFAULT_MEMORY_POOL=system`, because pyarrow's default pool reserves a 1 GiB arena on the first pandas string column, and `FONTCONFIG_FILE=/opt/anyplot/fonts.conf` for the fontconfig its bridge links; an in-process row's environment and limit reach the sandbox through `executor.sandbox_env` and the local Docker backend through `PythonRuntime.env`, which since 2026-10-26 is `sandbox_env` key for key, the sandbox's `PATH` and limits included, but `RLIMIT_NPROC`: the local container runs as the developer's user, whose processes on the whole machine count against it, so `--pids-limit` bounds the processes there) and what render time a repair reserves; `GET /status` reports under `languages` which toolchains the image carries, and a job in a language without one is `503 runtime_unavailable`. +Later languages reuse the CI commands: R's binary on `plot.R` (since 2026-10-29 the row runs `/opt/R/4.6.1/lib/R/bin/exec/R --no-echo --no-restore --no-save --file=plot.R` with the environment R's `bin/R` script would set and `LANG=C.UTF-8`, not `Rscript`, whose wrapper starts would die under the exec filter; the harness preloads the filter into the child, `agents/renderer/execdeny/execdeny.c`, and refuses the run, exit 3, unless the library reported `on`; the agents service sends R jobs since 2026-10-29, through `RRuntime`); Julia's launcher on `plot.jl` (since 2026-10-29 the row runs `/opt/julia-1.13.2/bin/julia --project=/opt/julia-project --startup-file=no --history-file=no --compiled-modules=strict --heap-size-hint=1536M plot.jl` under the preloaded exec filter, with a stacked `JULIA_DEPOT_PATH=/tmp/depot:/opt/julia-depot`, the `JULIA_CPU_TARGET` the toolchain image precompiled the catalogue's `Project.toml` and `Manifest.toml` for, one thread and the offline switches; under `strict` a package image that does not load fails the render in about a second instead of compiling under its timeout; since J1.1 the row adds `-L /opt/anyplot/toolchain/warmup.jl` before `plot.jl`, which loads the toolchain's precompile workload, a package whose image holds the native code of the calls a plot makes most and of the data loader's read, compiled with CSV, DataFrames and TimeZones loaded, and cut an adapted render from 15.9 to 5.0 s locally; the agents service sends Makie jobs since 2026-10-30, through `JuliaRuntime`, and the row keeps Julia's first `ERROR:` line in front of the stderr tail); Node by its path on the catalogue's own harness (since 2026-10-30 the row runs `/opt/node/bin/node /opt/anyplot/js-render/render.mjs --library= plot.js`, `plot.tsx` for MUI X, without the exec filter, because Node starts esbuild and Chromium and Chromium starts its own processes, while the plot code runs in the page, which has no process API; the toolchain image carries Node 22, the pipeline's major, the repository's `package-lock.json` installed with `npm ci` under `/opt/anyplot/node_modules`, Playwright's Chromium headless shell alone under `/opt/pw-browsers`, and Liberation Sans, Serif and Mono for this row alone through its own `FONTCONFIG_FILE`, so the catalogue's page font stack resolves as on the pipeline's runner while the other rows keep their fonts; the agents service sends Chart.js, D3, ECharts and Highcharts jobs since 2026-10-30, through `JavaScriptRuntime`, each with its table as `data_json`; since 2026-10-30 the row's environment switches on the harness's sandbox mode, `ANYPLOT_SANDBOX=1` with `ANYPLOT_NO_HTML=1`, which the catalogue pipeline never sets and without which the harness renders as before: every request of the page aborted by a route and refused by a content security policy (`default-src 'none'`, inline scripts and styles, `data:` and `blob:` images, `data:` fonts, no form submission, no ``), every WebSocket closed, `window.open` returning null, WebRTC connecting nowhere, the zone UTC, the page's V8 heap capped at 1 GiB, the console kept to the first five errors, MUI X's imports closed to the image's allowlist of bare specifiers, and one final stderr line `render: error: :: ` that the renderer's 2,000-character tail always keeps; a job may carry the table as typed JSON, `data_json`, which the renderer writes as `data.json` and the harness injects as `window.ANYPLOT_DATA`, one object per row in column order); bokeh and plotly render through the same Chromium in two steps, because a plot-mode harness cannot start a program once its guard is armed (the plot guard refuses a browser started from plot code; `agentic/specs/261024-agent-all-libraries.md`): since 2026-10-31 (PB-1, not served before PB-2) their rows run `harness.py --figure plot.py -- /opt/node/bin/node /opt/anyplot/js-render/render.mjs --library=`, in which the harness forks the plot step before it arms anything, the plot writes its figure as JSON (`fig.write_json`, `bokeh.embed.json_item`) under the plot guard, the exec filter and the Python limits, and the harness, never armed, then runs the JavaScript row's harness on it with that row's environment and limits, which draws it as data with the installed package's own bundled JavaScript in a sandbox-mode page whose policy admits scripts by a nonce alone, at the figure's own CSS size and the device scale that makes the canvas 3200x1800 or 2400x2400; the renderer image installs plotly and bokeh without kaleido or Selenium. Since 2026-10-24 the renderer holds these commands in one table (`agents/renderer/commands.py`, keyed by the job's `(language, library)`, the library checked against the catalogue's fifteen) and never takes a command from a request; a non-Python row runs through the harness's exec mode (`harness.py --exec -- plot.`: the start line, the limits set and inherited, the end line from the child's rusage), so the launcher retry, the watchdog and the kill path hold for every language; a row says which limits differ (`RLIMIT_AS` off for the JavaScript and Julia rows, whose V8, Chromium and LLVM reserve address space far beyond their resident memory, and `RLIMIT_NPROC` off for the JavaScript row alone, whose one Chromium render runs 77 tasks; since 2026-10-25 `RLIMIT_AS` is off for altair's Python row too, because vl-convert's embedded V8 aborts at start under any limit up to 64 GiB while its resident set is about 270 MiB), which limit it sets in place of the configured one (since 2026-10-26 lets-plot's Python row has its own 2048 MiB, `RuntimeCommand.rlimit_as_mb`, because its Kotlin/Native rasteriser aborts every 3200x1800 export under the configured 1 GiB while 2048 MiB still stops a runaway scale, which a 6400x3600 export shows at 2580 MiB; the renderer's switch still rules, so a configured limit of 0 leaves every row unlimited), which environment it adds (lets-plot's: Arrow's system allocator, `ARROW_DEFAULT_MEMORY_POOL=system`, because pyarrow's default pool reserves a 1 GiB arena on the first pandas string column, and `FONTCONFIG_FILE=/opt/anyplot/fonts.conf` for the fontconfig its bridge links; an in-process row's environment and limit reach the sandbox through `executor.sandbox_env` and the local Docker backend through `PythonRuntime.env`, which since 2026-10-26 is `sandbox_env` key for key, the sandbox's `PATH` and limits included, but `RLIMIT_NPROC`: the local container runs as the developer's user, whose processes on the whole machine count against it, so `--pids-limit` bounds the processes there) and what render time a repair reserves; `GET /status` reports under `languages` which toolchains the image carries, and a job whose row cannot run here, its language's toolchain or the row's own binaries missing (Node for plotly's and bokeh's screenshot step), is `503 runtime_unavailable`. ### Footer strip @@ -433,6 +433,15 @@ Plugin order on `App`, where the first non-None result wins and a plugin never r | Session poisoning or BFF bypass | Our own FastAPI, never `get_fast_api_app`; clients never write state or events; the BFF allowlists bodies; the agents app checks the IAM-forwarded ID-token claims; `ToolCallIntegrityPlugin` once sessions persist | | Phase 2 | Model Armor through direct sanitize calls on `modelarmor.eu.rep.googleapis.com` (once per user message and once per final text, failing closed, with `logSanitizeOperations=false`); `ModelArmorPlugin` only if a spike shows acceptable call counts, because it re-screens on every model call | +**Layers against code escape.** The harness's audit hooks (`DataGuard`, `PlotGuard` in `agents/anyplot/render/harness.py`) are defense in depth, not the boundary: a hook shares the interpreter with the code it judges, so code that changes the interpreter's shared state in place (the harness's own objects, the import machinery, the builtins) can change what the hook decides, and no audit event sees that change. Native libraries that cffi opens raise no audit event either. The boundaries are, in order: + +1. The validator, which refuses the names those routes need (`sys`, `cffi`, every dunder name) before any code runs. +2. The kernel's exec filter: a seccomp filter installed on every thread (`SECCOMP_FILTER_FLAG_TSYNC`) and inherited by every child, which refuses every `execve` and `execveat` and which no code in the process can lift. +3. The resource limits, soft and hard alike. +4. On Cloud Run, the gVisor sandbox that `sandbox do` starts for each job (spikes S and S2): no egress (the metadata server, DNS and TCP all fail), none of the service's environment or tokens, a read-only root filesystem (the renderer passes no `--write`), no view of the sibling sandboxes, and a lifetime of one job. + +The known limits of the in-process layer are listed in the PB-1 section of `agentic/specs/261024-agent-all-libraries.md`. + ## Feedback capture Every agent result can be reported with one click and a short text, and the report carries the full case, so prompts, validator rules, gates and the model pin can be tuned from real sessions. It mirrors the site feedback (`api/routers/feedback.py`, `FeedbackWidget`, triage under `/debug/feedback/*`) but stores a case bundle instead of a message. diff --git a/tests/unit/agents/code/test_validate.py b/tests/unit/agents/code/test_validate.py index 2571606e2c7..f9559053733 100644 --- a/tests/unit/agents/code/test_validate.py +++ b/tests/unit/agents/code/test_validate.py @@ -13,6 +13,7 @@ import pytest +from agents.anyplot.code import validate from agents.anyplot.code.validate import ( BANNED_ATTRIBUTES, BANNED_IO, @@ -1575,6 +1576,146 @@ def test_every_submodule_is_reviewed(root: str) -> None: assert not REVIEWED_SUBMODULES[root] & banned +# --- import machinery reachable from the admitted modules (F6 and F7 of PB-1's review, 2026-10-31) ----------------- + +IMPORT_MACHINERY_MODULES = frozenset( + { + "importlib", + "importlib.util", + "importlib.machinery", + "importlib.abc", + "importlib.resources", + "importlib.metadata", + "_frozen_importlib", + "_frozen_importlib_external", + "zipimport", + "pkgutil", + "runpy", + "sys", + "marshal", + } +) +"""Modules whose mere reach is a route. The `types` module is not one: typing, dataclasses, matplotlib.cbook and pandas +re-export it, and its dangerous members (`new_class`, `ModuleType`, `FunctionType`, `CodeType`) are banned by name.""" +IMPORT_MACHINERY_FUNCTIONS = frozenset( + {"importlib", "importlib.util", "_frozen_importlib", "_frozen_importlib_external", "zipimport", "pkgutil", "runpy"} +) + + +def _import_machinery(value: object) -> bool: + """A module of the import machinery, a loader class or `ModuleType`, a function of the machinery's modules, or one + of the `types` members that build classes and functions.""" + import importlib.abc + import types + + if isinstance(value, types.ModuleType): + return value.__name__ in IMPORT_MACHINERY_MODULES + if isinstance(value, type): + try: + return issubclass(value, importlib.abc.Loader) or value in ( + types.ModuleType, + types.FunctionType, + types.CodeType, + ) + except TypeError: + return False + if value is types.new_class or value is types.prepare_class: + return True + return callable(value) and getattr(value, "__module__", None) in IMPORT_MACHINERY_FUNCTIONS + + +def _reachable_import_machinery(depth: int = 4) -> list[tuple[str, str]]: + """`(root, path)` of every import-machinery value a public attribute chain reaches from the admitted modules: the + modules below a root up to `depth` attributes, and one level of attributes of the classes found there.""" + import importlib + import types + import warnings + + roots = sorted( + frozenset(validate.COMMON_IMPORTS) + | validate.SHARED_SCIENCE_IMPORTS + | frozenset().union(*validate.LIBRARY_IMPORTS.values()) + ) + found: list[tuple[str, str]] = [] + seen: set[int] = set() + queue: list[tuple[types.ModuleType, str, str, int]] = [] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for root in roots: + try: + queue.append((importlib.import_module(root), root, root, 0)) + except Exception: + continue # a library this test environment does not install + while queue: + module, root, path, level = queue.pop(0) + if id(module) in seen: + continue + seen.add(id(module)) + for name in [name for name in dir(module) if not name.startswith("_")]: + try: + value = getattr(module, name) + except Exception: + continue + here = f"{path}.{name}" + if _import_machinery(value): + found.append((root, here)) + if isinstance(value, types.ModuleType) and level < depth: + queue.append((value, root, here, level + 1)) + elif isinstance(value, type) and level < depth: + for attribute in [a for a in dir(value) if not a.startswith("_")][:400]: + try: + if _import_machinery(getattr(value, attribute)): + found.append((root, f"{here}.{attribute}")) + except Exception: + continue + return found + + +def test_no_admitted_module_hands_out_import_machinery(monkeypatch: pytest.MonkeyPatch) -> None: + """Every public attribute path from an admitted module to the import machinery (loaders, spec builders, `pkgutil`, + `types`, `sys`) is refused, so plot code cannot drive an import of its own (F6) or read package files by name + (F7: matplotlib's `dateutil.zoneinfo.get_data` and `dateutil.tz.six.spec_from_loader`, found by this walk). A + library upgrade that adds such a path fails here until it is banned.""" + monkeypatch.setenv("MPLBACKEND", "Agg") + profiles = validate.LIBRARY_IMPORTS + admitted = [] + paths = _reachable_import_machinery() + assert paths, "the walk reached nothing: the admitted modules did not import" + for root, path in paths: + library = next((name for name, modules in profiles.items() if root in modules), "matplotlib") + findings = [ + f for f in validate_security(f"import {root}\nx = {path}\n", library=library) if f.rule != "savefig-target" + ] + if not findings: + admitted.append(path) + assert not admitted, f"the validator admits import machinery: {admitted[:10]}" + + +@pytest.mark.parametrize( + "code", + [ + "import matplotlib\nraw = matplotlib.category.dateutil.zoneinfo.get_data('matplotlib', '../../etc/os-release')\n", + "import matplotlib\nreader = matplotlib.dates.dateutil\n", + "import matplotlib\nspec = matplotlib.category.dateutil.tz.six.spec_from_loader('x', None)\n", + "import numpy as np\nraw = np.get_data('numpy', 'x')\n", + "import numpy as np\nm = np.module_from_spec(None)\n", + "import typing\nText = typing.types.new_class('Text', (str,), {}, lambda ns: ns.update(split=lambda s: []))\n", + "import dataclasses\nm = dataclasses.types.ModuleType('x')\n", + ], + ids=[ + "dateutil's get_data", + "the dateutil re-export", + "six's spec_from_loader", + "get_data", + "module_from_spec", + "a class built by types.new_class", + "types.ModuleType", + ], +) +def test_the_package_data_and_spec_routes_are_refused(code: str) -> None: + assert [f for f in validate_security(code, library="matplotlib") if f.rule != "savefig-target"] + + # --- plotnine and altair (step 1b of agentic/specs/261024-agent-all-libraries.md) ------------------------------ PLOTNINE_HEADER = """import os diff --git a/tests/unit/agents/renderer/helpers.py b/tests/unit/agents/renderer/helpers.py index 1e02233e1b6..c999d4deff9 100644 --- a/tests/unit/agents/renderer/helpers.py +++ b/tests/unit/agents/renderer/helpers.py @@ -147,12 +147,17 @@ def __init__(self) -> None: self.stuck = 0 self.available = True self.toolchains: dict[str, bool] = {"python": True, "javascript": False, "r": False, "julia": False} + self.missing_rows: set[tuple[str, str]] = set() + """`(language, library)` rows whose own binaries are missing although their language's are there.""" self.page: PageRun | None = None """A page build's answer; None builds a small page that names the library and the theme.""" def languages(self) -> dict[str, bool]: return dict(self.toolchains) + def serves(self, language: str, library: str) -> bool: + return self.toolchains.get(language, False) and (language, library) not in self.missing_rows + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: self.active += 1 self.max_active = max(self.max_active, self.active) diff --git a/tests/unit/agents/renderer/test_figure_mode.py b/tests/unit/agents/renderer/test_figure_mode.py new file mode 100644 index 00000000000..4c6af35f23d --- /dev/null +++ b/tests/unit/agents/renderer/test_figure_mode.py @@ -0,0 +1,1062 @@ +"""The harness's figure mode (`harness.py --figure plot.py -- `, PB-1 of step 4 in +agentic/specs/261024-agent-all-libraries.md): the plot step forked before anything is armed, the plot armed in the +child, the parent never armed, the figure's checks, the final error line of either step, and the renderer's +two-step rows. + +The harness runs in a subprocess of its own (an audit hook and a seccomp filter cannot be removed), loaded by a +driver that stands in for the package's lookup, so these cases need neither plotly nor bokeh; the screenshot command +is a Python script that reports what it ran under. `tests/unit/automation/test_js_render_figure.py` draws real +figures through the JavaScript harness. +""" + +import contextlib +import json +import os +import platform +import re +import resource +import subprocess +import sys +import time +from collections.abc import Callable, Iterator +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agents.anyplot.render import harness +from agents.anyplot.render.runtimes.python import HARNESS_SOURCE +from agents.renderer import commands +from agents.renderer import executor as executor_module +from agents.renderer.commands import ( + FIGURE_FLAG, + FIGURE_LIBRARIES, + JS_ENV, + JS_HARNESS, + NODE_BINARY, + RuntimeCommand, + command_for, +) +from agents.renderer.executor import SANDBOX_PATH, ExecutorConfig, SandboxExecutor, build_argv, sandbox_env +from agents.renderer.wire import RenderRequest + +from .conftest import FakeLauncher + + +KERNEL = pytest.mark.skipif( + not sys.platform.startswith("linux") or platform.machine() not in harness.EXEC_DENY_MACHINES, + reason="the filter is installed on Linux on x86_64 and aarch64 only", +) +ERROR_LINE = re.compile(r"^render: error: (plot|figure|crash|harness) ([A-Za-z_][A-Za-z0-9_]*) (\S+:\d+|-): (.*)$") +"""The final error line's grammar for the plot step (`harness.figure_error_line`), `render.mjs`'s own.""" +DRIVER = ( + "import importlib.util, sys\n" + "spec = importlib.util.spec_from_file_location('harness', sys.argv[1])\n" + "harness = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(harness)\n" + "package = sys.argv[2]\n" + "harness.find_figure_package = lambda name: package\n" + "{patch}" + "sys.exit(harness.main(['harness.py', *sys.argv[3:]]))\n" +) +"""The harness's `main` with the package's lookup answering a stand-in directory (the image has plotly and bokeh, a +test environment need not) and a case's `{patch}`.""" +STATUS = ( + "def status(pid='self'):\n" + " with open(f'/proc/{pid}/status') as handle:\n" + " pairs = (line.partition(':') for line in handle)\n" + " return {key: value.strip() for key, _, value in pairs if key in ('Seccomp_filters', 'NoNewPrivs')}\n" +) +"""A process's seccomp filters and its no-new-privileges bit. The count, not the mode: a test may already run under a +filter of its own sandbox, which every process below it inherits.""" +OBSERVING_PLOT = ( + "import json, os, resource\n" + STATUS + "seen = {'pid': os.getpid(), 'ppid': os.getppid(), 'status': status(),\n" + " 'as': resource.getrlimit(resource.RLIMIT_AS)[0], 'nproc': resource.getrlimit(resource.RLIMIT_NPROC)[0],\n" + " 'cpu': resource.getrlimit(resource.RLIMIT_CPU)[0]}\n" + "with open('plot-light.json', 'w') as handle:\n" + " json.dump({'data': [], 'layout': {}, 'seen': seen}, handle)\n" +) +"""A plot that writes what it ran under into its figure, the one file it may write.""" +SCREENSHOT = ( + "import json, os, resource, subprocess, sys\n" + STATUS + "figure = sys.argv[-1].split('=', 1)[1]\n" + "report = {'argv': sys.argv[1:], 'harness': os.getppid(), 'harness_status': status(os.getppid()),\n" + " 'status': status(), 'as': resource.getrlimit(resource.RLIMIT_AS)[0],\n" + " 'nproc': resource.getrlimit(resource.RLIMIT_NPROC)[0], 'cpu': resource.getrlimit(resource.RLIMIT_CPU)[0],\n" + " 'env': {key: value for key, value in os.environ.items() if key.startswith('ANYPLOT_')},\n" + " 'figure': json.load(open(figure)), 'starts': subprocess.run(['/bin/true']).returncode}\n" + "open('shot.json', 'w').write(json.dumps(report))\n" + "open('plot-light.png', 'wb').write(b'\\x89PNG\\r\\n\\x1a\\n')\n" + "code = int(os.environ.get('SHOT_EXIT', '0'))\n" + "if code:\n" + " print('render: error: page TypeError -: the screenshot step failed', file=sys.stderr)\n" + "sys.exit(code)\n" +) +"""The screenshot step's stand-in: reports its own and its parent's state, then writes a PNG's signature.""" + + +def run_figure( + tmp_path: Path, + plot: str, + *, + env: dict[str, str] | None = None, + package: str = "plotly", + command: list[str] | None = None, + patch: str = "", + timeout: float = 120, +) -> subprocess.CompletedProcess[str]: + """The harness in figure mode on `plot` in `tmp_path`, the stand-in screenshot (or `command`) after it, with the + driver's `patch` applied before `main`.""" + work = tmp_path / "work" + work.mkdir(exist_ok=True) + (work / "plot.py").write_text(plot, encoding="utf-8") + (tmp_path / "shot.py").write_text(SCREENSHOT, encoding="utf-8") + stand_in = tmp_path / "site-packages" / package + stand_in.mkdir(parents=True, exist_ok=True) + command = command or [sys.executable, "-I", str(tmp_path / "shot.py")] + argv = [FIGURE_FLAG, package, "plot.py", "--", *command] + driver = DRIVER.replace("{patch}", patch) + return subprocess.run( + [sys.executable, *harness.INTERPRETER_FLAGS, "-c", driver, str(HARNESS_SOURCE), str(stand_in), *argv], + cwd=work, + capture_output=True, + text=True, + timeout=timeout, + env={ + "PATH": "/usr/bin:/bin", + "HOME": str(tmp_path), + "ANYPLOT_THEME": "light", + "ANYPLOT_RLIMIT_CPU_S": "120", + **(env or {}), + }, + ) + + +def end_line(stdout: str) -> dict[str, object]: + lines = [line for line in stdout.splitlines() if line.startswith(f"{harness.MARKER} ")] + report = json.loads(lines[-1].split(" ", 1)[1]) + assert report["event"] == "end" + return report + + +def last_line(stderr: str) -> str: + lines = stderr.strip().splitlines() + return lines[-1] if lines else "" + + +class TestTwoSteps: + @KERNEL + def test_the_plot_runs_armed_in_a_fork_and_the_screenshot_after_it_unarmed(self, tmp_path: Path) -> None: + """Fork before arm: both steps are children of one harness process, which no filter ever reached; the plot + step runs under the filter and the plot's limits, the screenshot step under neither and without the plot's + variables, and starts a program.""" + address_space = 4096 + done = run_figure( + tmp_path, OBSERVING_PLOT, env={"ANYPLOT_DENY_EXEC": "required", "ANYPLOT_RLIMIT_AS_MB": str(address_space)} + ) + + assert done.returncode == 0, done.stderr + shot = json.loads((tmp_path / "work" / "shot.json").read_text(encoding="utf-8")) + plot = shot["figure"]["seen"] + assert plot["ppid"] == shot["harness"] # a child of the harness, forked, not a program it started anew + inherited = int(shot["harness_status"]["Seccomp_filters"]) # the test's own sandbox may hold one + # The kernel filter in the plot step alone; the harness never armed, and the screenshot starts a program. + assert plot["status"] == {"Seccomp_filters": str(inherited + 1), "NoNewPrivs": "1"} + assert shot["harness_status"]["NoNewPrivs"] == "0" + assert shot["status"] == shot["harness_status"] and shot["starts"] == 0 + assert plot["as"] == address_space * 1024 * 1024 and shot["as"] == resource.RLIM_INFINITY + assert plot["cpu"] == shot["cpu"] == 120 # the run's limits reach both steps + assert shot["argv"] == ["--figure=plot-light.json"] + assert shot["env"]["ANYPLOT_FIGURE_PACKAGE"].endswith("site-packages/plotly") + assert not set(harness.FIGURE_PLOT_ONLY_ENV) & set(shot["env"]) + end = end_line(done.stdout) + assert end["exec_deny"] == "on" and set(end["steps"]) == {"plot", "screenshot"} + assert end["max_rss_mb"] >= end["steps"]["plot"]["max_rss_mb"] + assert end["cpu_s"] >= end["steps"]["plot"]["cpu_s"] + end["steps"]["screenshot"]["cpu_s"] - 0.01 + + def test_imports_from_the_install_run_in_the_plot_step(self, tmp_path: Path) -> None: + """The plot step's guard trusts the interpreter's own loaders (F6 of PB-1's review): a site package with + extension modules, a stdlib module with a `lib-dynload` extension, a stdlib package and a frozen module, each + imported first with the guard armed.""" + plot = "import scipy.stats\nimport lzma\nimport zoneinfo\nimport __hello__\n" + OBSERVING_PLOT + done = run_figure(tmp_path, plot, env={"ANYPLOT_DENY_EXEC": "required"}) + + assert done.returncode == 0, done.stderr + assert (tmp_path / "work" / "shot.json").exists() + + def test_the_plots_process_limit_is_its_own(self, tmp_path: Path) -> None: + """`RLIMIT_NPROC` binds the plot step; the screenshot step keeps the harness's own, as the JavaScript row.""" + soft, hard = resource.getrlimit(resource.RLIMIT_NPROC) + wanted = hard if hard != resource.RLIM_INFINITY else 1_000_000 + done = run_figure(tmp_path, OBSERVING_PLOT, env={"ANYPLOT_RLIMIT_NPROC": str(wanted)}) + + assert done.returncode == 0, done.stderr + shot = json.loads((tmp_path / "work" / "shot.json").read_text(encoding="utf-8")) + assert shot["figure"]["seen"]["nproc"] == wanted and shot["nproc"] == soft + + def test_without_the_requirement_the_end_line_reports_the_state(self, tmp_path: Path) -> None: + done = run_figure(tmp_path, OBSERVING_PLOT) + + assert done.returncode == 0, done.stderr + assert end_line(done.stdout)["exec_deny"] in ("on", "unavailable", "unsupported") + + def test_a_stale_figure_never_reaches_the_screenshot(self, tmp_path: Path) -> None: + (tmp_path / "work").mkdir() + (tmp_path / "work" / "plot-light.json").write_text('{"data": [], "layout": {}}', encoding="utf-8") + done = run_figure(tmp_path, "x = 1\n") + + assert done.returncode == 1 + assert last_line(done.stderr) == "render: error: figure NoFigure -: the plot wrote no plot-light.json" + assert not (tmp_path / "work" / "shot.json").exists() + + +class TestThePlotStep: + @pytest.mark.parametrize( + ("body", "line", "event"), + [ + ("import subprocess\nsubprocess.run(['/bin/true'])\n", 2, "subprocess.Popen"), + ("import os\nos.posix_spawn('/bin/true', ['true'], {})\n", 2, "os.posix_spawn"), + ("import os\nos.fork()\n", 2, "os.fork"), + ("import os\nos.execv('/bin/true', ['true'])\n", 2, "os.exec"), + ("import socket\nsocket.socketpair()\n", 2, "socket"), + ("open('plot-light.png', 'wb').write(b'x')\n", 1, "open"), + ], + ids=["subprocess", "posix_spawn", "fork", "execv", "a socket pair", "the PNG itself"], + ) + def test_a_plot_that_starts_a_program_or_writes_the_png_fails_before_the_screenshot( + self, tmp_path: Path, body: str, line: int, event: str + ) -> None: + """No browser from plot code: kaleido's choreographer opens a socket pair, Selenium a socket to chromedriver + and both start a program; every such start, and the PNG the screenshot step alone writes, are refused.""" + done = run_figure(tmp_path, body) + + assert done.returncode == 1 + match = ERROR_LINE.match(last_line(done.stderr)) + assert match is not None, done.stderr + assert match.group(1) == "plot" and match.group(3) == f"plot.py:{line}" + assert match.group(2) == "PermissionError" and event in match.group(4) + assert not (tmp_path / "work" / "shot.json").exists() and not (tmp_path / "work" / "plot-light.png").exists() + assert set(end_line(done.stdout)["steps"]) == {"plot"} + + def test_nothing_of_the_plot_runs_after_it(self, tmp_path: Path) -> None: + """The child ends with `os._exit`: an `atexit` callback and a thread the plot left, each of which would + overwrite the figure (the one file the guard lets it write), never run on.""" + plot = ( + "import atexit, threading, time\n" + "atexit.register(lambda: open('plot-light.json', 'w').write('late-atexit'))\n" + "threading.Thread(target=lambda: (time.sleep(1), open('plot-light.json', 'w').write('late-thread'))).start()\n" + + OBSERVING_PLOT + ) + done = run_figure(tmp_path, plot) + time.sleep(1.5) + + assert done.returncode == 0, done.stderr + assert "seen" in json.loads((tmp_path / "work" / "plot-light.json").read_text(encoding="utf-8")) + + @KERNEL + def test_the_kernel_refuses_a_start_the_audit_hook_misses(self, tmp_path: Path) -> None: + """The plot step's second layer: with the hook made blind to `os.exec` (as it is to native code), the kernel + filter still refuses the start, so nothing after the arming starts a program in the plot step.""" + blind = "harness.PLOT_BLOCKED_PREFIXES = tuple(p for p in harness.PLOT_BLOCKED_PREFIXES if p != 'os.exec')\n" + plot = ( + "import json, os\n" + "try:\n" + " os.execv('/bin/true', ['true'])\n" + "except OSError as exc:\n" + " refused = type(exc).__name__\n" + "json.dump({'data': [], 'refused': refused}, open('plot-light.json', 'w'))\n" + ) + done = run_figure(tmp_path, plot, env={"ANYPLOT_DENY_EXEC": "required"}, patch=blind) + + assert done.returncode == 0, done.stderr + shot = json.loads((tmp_path / "work" / "shot.json").read_text(encoding="utf-8")) + assert shot["figure"]["refused"] == "PermissionError" and shot["starts"] == 0 + + @pytest.mark.parametrize( + ("plot", "code", "line"), + [ + ("x = 1\nraise ValueError('boom')\n", 1, "render: error: plot ValueError plot.py:2: boom"), + ("x = (\n", 1, None), + ("import sys\nsys.exit(4)\n", 4, "render: error: plot SystemExit plot.py:2: 4"), + ("x = 1\n", 1, "render: error: figure NoFigure -: the plot wrote no plot-light.json"), + ("import os\nos._exit(0)\n", 1, "render: error: figure NoFigure -: the plot wrote no plot-light.json"), + ( + "open('plot-light.json', 'w').close()\n", + 1, + "render: error: figure FigureError -: plot-light.json is empty", + ), + ], + ids=[ + "an exception", + "a syntax error", + "sys.exit", + "no figure", + "os._exit before the figure", + "an empty figure", + ], + ) + def test_a_failed_plot_step_ends_stderr_with_its_line_and_its_code( + self, tmp_path: Path, plot: str, code: int, line: str | None + ) -> None: + done = run_figure(tmp_path, plot) + + assert done.returncode == code + last = last_line(done.stderr) + assert ERROR_LINE.match(last), done.stderr + if line is not None: + assert last == line + else: + assert last.startswith("render: error: plot SyntaxError plot.py:1: ") + assert not (tmp_path / "work" / "shot.json").exists() + + def test_a_plot_that_succeeds_with_sys_exit_0_goes_on(self, tmp_path: Path) -> None: + done = run_figure(tmp_path, OBSERVING_PLOT + "import sys\nsys.exit(0)\n") + assert done.returncode == 0, done.stderr + assert (tmp_path / "work" / "shot.json").exists() + + def test_the_traceback_comes_before_the_final_line(self, tmp_path: Path) -> None: + done = run_figure(tmp_path, "def f():\n raise KeyError('k')\n\nf()\n") + + lines = done.stderr.strip().splitlines() + assert "Traceback (most recent call last):" in lines and 'File "plot.py", line 4, in ' in done.stderr + assert lines[-1] == "render: error: plot KeyError plot.py:2: 'k'" + + def test_the_error_record_runs_no_plot_code_paused(self, tmp_path: Path) -> None: + """The record's traceback walk pauses the guard; a `__traceback__` property of the plot's exception class is + not called there, and where the interpreter's traceback calls it, the armed guard refuses its write.""" + plot = ( + "class Escape(Exception):\n" + " @property\n" + " def __traceback__(self):\n" + " try:\n" + " open('escaped', 'w').write('x')\n" + " except OSError:\n" + " pass\n" + " return None\n" + "\n" + "raise Escape('boom')\n" + ) + done = run_figure(tmp_path, plot) + + assert done.returncode == 1 + assert last_line(done.stderr) == "render: error: plot Escape plot.py:10: boom" + assert not (tmp_path / "work" / "escaped").exists() + + def test_a_plot_step_killed_by_a_signal_is_a_crash(self, tmp_path: Path) -> None: + """`RLIMIT_CPU` ends a busy plot: soft and hard limit alike, the kernel kills it with SIGKILL.""" + done = run_figure(tmp_path, "while True:\n pass\n", env={"ANYPLOT_RLIMIT_CPU_S": "1"}, timeout=60) + + assert done.returncode == 128 + 9 + assert last_line(done.stderr) == "render: error: crash PlotCrashed -: the plot step died of SIGKILL" + assert set(end_line(done.stdout)["steps"]) == {"plot"} + + def test_the_figure_cap_refuses_one_byte_more(self, tmp_path: Path) -> None: + plot = f"open('plot-light.json', 'w').write(' ' * {harness.FIGURE_MAX_BYTES} + '{{}}')\n" + done = run_figure(tmp_path, plot, env={"ANYPLOT_RLIMIT_FSIZE_MB": "64"}) + + assert done.returncode == 1 + size = harness.FIGURE_MAX_BYTES + 2 + assert last_line(done.stderr) == ( + f"render: error: figure FigureTooLarge -: plot-light.json is {size} bytes, " + f"more than the {harness.FIGURE_MAX_BYTES}-byte limit" + ) + assert not (tmp_path / "work" / "shot.json").exists() + + def test_a_required_filter_that_is_not_in_place_stops_the_plot_step(self, tmp_path: Path) -> None: + done = run_figure( + tmp_path, + "open('plot-light.json', 'w').write('ran')\n" + OBSERVING_PLOT, + env={"ANYPLOT_DENY_EXEC": "required"}, + patch="harness.install_exec_deny = lambda: 'unavailable'\n", + ) + + assert done.returncode == harness.EXEC_DENY_EXIT + assert last_line(done.stderr) == ( + "render: error: harness ExecDenyUnavailable -: the sandbox could not refuse process starts (unavailable)" + ) + assert not (tmp_path / "work" / "plot-light.json").exists() and not (tmp_path / "work" / "shot.json").exists() + assert end_line(done.stdout)["exec_deny"] == "unavailable" + + @pytest.mark.parametrize( + ("plot", "env", "patch", "line", "state"), + [ + pytest.param( + "import sys\nsys.exit(3)\n", + {"ANYPLOT_DENY_EXEC": "required"}, + "", + "render: error: plot SystemExit plot.py:2: 3", + "on", + marks=KERNEL, + id="sys.exit(3) under the filter", + ), + pytest.param( + "import sys\nsys.exit(3)\n", + {}, + "harness.install_exec_deny = lambda: 'unavailable'\n", + "render: error: plot SystemExit plot.py:2: 3", + "unavailable", + id="sys.exit(3) without a filter it did not require", + ), + pytest.param( + "import os\nos._exit(3)\n", + {}, + "harness.install_exec_deny = lambda: 'unavailable'\n", + "render: error: plot Error -: the plot step ended with 3", + "unavailable", + id="os._exit(3), no record", + ), + ], + ) + def test_a_plot_that_ends_with_the_filters_exit_code_is_a_plot_error( + self, tmp_path: Path, plot: str, env: dict[str, str], patch: str, line: str, state: str + ) -> None: + """`EXEC_DENY_EXIT` is the harness's refusal only when the report's first line says `refused`: a plot that ran + armed and chose the same code is the plot's failure, whatever the filter's state.""" + done = run_figure(tmp_path, plot, env=env, patch=patch) + + assert done.returncode == harness.EXEC_DENY_EXIT + assert last_line(done.stderr) == line + assert end_line(done.stdout)["exec_deny"] == state + assert not (tmp_path / "work" / "shot.json").exists() + + +class TestTheScreenshotStep: + def test_its_failure_is_the_runs_and_its_line_stays_last(self, tmp_path: Path) -> None: + done = run_figure(tmp_path, OBSERVING_PLOT, env={"SHOT_EXIT": "5"}) + + assert done.returncode == 5 + assert last_line(done.stderr) == "render: error: page TypeError -: the screenshot step failed" + assert set(end_line(done.stdout)["steps"]) == {"plot", "screenshot"} + + def test_a_command_that_cannot_start_is_127(self, tmp_path: Path) -> None: + missing = str(tmp_path / "no-such-node") + done = run_figure(tmp_path, OBSERVING_PLOT, command=[missing, "render.mjs"]) + + assert done.returncode == 127 + assert ( + last_line(done.stderr) == f"render: error: harness SpawnError -: cannot run {missing} (FileNotFoundError)" + ) + + +class TestHarnessUsage: + @pytest.mark.parametrize( + "argv", + [[FIGURE_FLAG], [FIGURE_FLAG, "plotly", "plot.py"], [FIGURE_FLAG, "matplotlib", "plot.py", "--", "x"]], + ids=["nothing", "no command", "another package"], + ) + def test_a_bad_call_is_a_usage_error_after_the_start_line( + self, argv: list[str], capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(harness, "set_run_limits", lambda: None) + assert harness.main(["harness.py", *argv]) == 2 + captured = capsys.readouterr() + assert captured.out.splitlines()[0] == executor_module.HARNESS_START + assert last_line(captured.err).startswith("render: error: harness UsageError -: ") + + def test_a_missing_package_stops_before_the_plot( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setattr(harness, "set_run_limits", lambda: None) + monkeypatch.setattr(harness, "find_figure_package", lambda name: None) + monkeypatch.setattr(harness, "plot_step", lambda *args: pytest.fail("the plot step started")) + monkeypatch.chdir(tmp_path) + assert harness.main(["harness.py", FIGURE_FLAG, "bokeh", "plot.py", "--", "x"]) == 2 + assert last_line(capsys.readouterr().err) == ( + "render: error: harness PackageMissing -: bokeh is not installed where the harness runs" + ) + + +class TestHelpers: + def test_the_package_is_found_without_importing_it(self) -> None: + found = harness.find_figure_package("json") + assert found is not None and Path(found, "__init__.py").is_file() + assert harness.find_figure_package("anyplot_no_such_package") is None + assert harness.find_figure_package("os") is None # a module, not a package + + def test_check_figure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + assert harness.check_figure("plot-light.json") == ("NoFigure", None, "the plot wrote no plot-light.json") + Path("plot-light.json").write_text("{}", encoding="utf-8") + assert harness.check_figure("plot-light.json") is None + Path("real.json").write_text("{}", encoding="utf-8") + Path("plot-dark.json").symlink_to("real.json") + name, _, message = harness.check_figure("plot-dark.json") + assert name == "FigureError" and "could not be opened as a regular file" in message + Path("dir.json").mkdir() + assert harness.check_figure("dir.json") == ("FigureError", None, "dir.json is not a regular file") + with open("at-cap.json", "wb") as handle: + handle.truncate(harness.FIGURE_MAX_BYTES) + assert harness.check_figure("at-cap.json") is None + + @pytest.mark.parametrize( + ("written", "state", "refused", "record"), + [ + (b"", "off", False, None), + (b"armed on\n", "on", False, None), + (b"armed unavailable\n", "unavailable", False, None), + (b"refused unavailable\n", "unavailable", True, None), + (b"armed forged\n", "unavailable", False, None), + (b"exec_deny on\n", "off", False, None), + (b"armed on\nrefused unavailable\n", "on", False, None), + (b'error ["SyntaxError", 3, "bad"]\n', "off", False, ("SyntaxError", 3, "bad")), + (b'armed on\nerror ["ValueError", 0, "x"]\n', "on", False, ("ValueError", None, "x")), + (b'armed on\nerror ["not a class", 2, "x"]\nerror [1, 2]\n', "on", False, None), + (b'armed on\nerror ["A", 2, "x"]\nnoise\n', "on", False, ("A", 2, "x")), + (b'armed on\nerror ["A", true, "x"]\nerror {"a": 1}\n', "on", False, ("A", None, "x")), + ], + ids=[ + "nothing", + "on", + "unavailable and not required", + "refused", + "a forged state", + "an unknown first line", + "a refusal after the arming", + "before arming", + "line 0", + "malformed", + "noise after", + "a flag for a line", + ], + ) + def test_the_report_pipe(self, written: bytes, state: str, refused: bool, record: tuple | None) -> None: + """The first line is the harness's own: a later `refused` line is the plot's, and changes nothing.""" + read_end, write_end = os.pipe() + os.write(write_end, written) + os.close(write_end) + os.set_blocking(read_end, False) + report = harness._read_figure_report(read_end) + assert (report.state, report.refused, report.record) == (state, refused, record) + + def test_an_empty_open_pipe_reads_as_nothing(self) -> None: + """A write end a stray process of the plot kept open: the reader takes what is there and does not wait.""" + read_end, write_end = os.pipe() + os.set_blocking(read_end, False) + try: + report = harness._read_figure_report(read_end) + finally: + os.close(write_end) + assert (report.state, report.refused, report.record) == ("off", False, None) + + def test_the_final_line_is_one_line_within_its_cap(self, capsys: pytest.CaptureFixture[str]) -> None: + harness.figure_error_line("plot", "ValueError", "plot.py:3", "a\nb\t c" + "x" * 500) + line = capsys.readouterr().err.rstrip("\n") + assert "\n" not in line and ERROR_LINE.match(line) + assert line.startswith("render: error: plot ValueError plot.py:3: a b c") + assert len(line.split(": ", 3)[3]) == harness.FIGURE_ERROR_CHARS and line.endswith("…") + + def test_the_constants_the_javascript_harness_shares(self) -> None: + render = (Path(__file__).resolve().parents[4] / "automation" / "js-render" / "render.mjs").read_text() + cap = re.search(r"^const FIGURE_MAX_BYTES = (\d+) \* (\d+) \* (\d+);$", render, re.MULTILINE) + assert cap is not None and int(cap[1]) * int(cap[2]) * int(cap[3]) == harness.FIGURE_MAX_BYTES + assert re.search(r"^const ERROR_MESSAGE_CHARS = (\d+);", render, re.MULTILINE)[1] == str( + harness.FIGURE_ERROR_CHARS + ) + assert re.search(r"process\.env\.(ANYPLOT_FIGURE_PACKAGE)", render)[1] == harness.FIGURE_PACKAGE_ENV + figures = render[render.index("const FIGURES = {") : render.index("\n};", render.index("const FIGURES = {"))] + assert set(re.findall(r"^ (\w+): \{", figures, re.MULTILINE)) == set(harness.FIGURE_PACKAGES) + assert harness.EXEC_DENY_ENV in harness.FIGURE_PLOT_ONLY_ENV + + def test_the_figure_mode_flag_and_packages_are_the_renderers(self) -> None: + assert FIGURE_FLAG == harness.FIGURE_FLAG and FIGURE_LIBRARIES == harness.FIGURE_PACKAGES + + def test_plotlys_validator_builder_is_a_code_builder_and_nothing_else_of_it_is(self) -> None: + def frame(path: str, name: str) -> object: + code = type("Code", (), {"co_filename": path, "co_name": name}) + return type("Frame", (), {"f_code": code, "f_back": None}) + + site = "/app/.venv/lib/python3.14/site-packages" + assert harness.builds_code(frame(f"{site}/_plotly_utils/basevalidators.py", "build_validator")) + assert not harness.builds_code(frame(f"{site}/_plotly_utils/basevalidators.py", "__init__")) + assert not harness.builds_code(frame(f"{site}/plotly/basedatatypes.py", "build_validator")) + + +class Exited(Exception): + """`os._exit` in an in-process call of the plot step's child: the code, instead of the end of the test process.""" + + def __init__(self, code: int) -> None: + super().__init__(code) + self.code = code + + +class StandInGuard: + """`PlotGuard`'s part the plot step's child uses: the filter's state and `paused`.""" + + def __init__(self, allowed: tuple[str, ...], exec_deny: str) -> None: + self.allowed = allowed + self.exec_deny = exec_deny + + @contextlib.contextmanager + def paused(self) -> Iterator[None]: + yield + + +def wait_status(code: int = 0, signal_number: int = 0) -> int: + """A wait status as `os.wait4` returns it: an exit code, or a death by a signal.""" + return signal_number or code << 8 + + +def raise_in_plot(source: str) -> None: + """Run `source` as code of `plot.py`, so a traceback through it has the plot's frames.""" + exec(compile(source, "plot.py", "exec"), {}) + + +class TestInProcess: + """The figure mode's logic in the test process, where the coverage of a run counts: the parent's judgement, the + child's exit codes and report, the records, the screenshot step's start and the steps' order, with `os.fork`, + `os._exit`, the guard and the plot stood in. The fork, the arming and the kernel filter themselves run only in the + subprocess cases above (an audit hook and a seccomp filter cannot be removed from the test process).""" + + @pytest.mark.parametrize( + ("status", "report", "verdict"), + [ + (wait_status(0), harness.FigureReport("on"), (0, None)), + ( + wait_status(3), + harness.FigureReport("on", record=("SystemExit", 2, "3")), + (3, ("plot", "SystemExit", "plot.py:2", "3")), + ), + ( + wait_status(3), + harness.FigureReport("unavailable"), + (3, ("plot", "Error", None, "the plot step ended with 3")), + ), + ( + wait_status(3), + harness.FigureReport("unavailable", refused=True), + ( + 3, + ( + "harness", + "ExecDenyUnavailable", + None, + "the sandbox could not refuse process starts (unavailable)", + ), + ), + ), + ( + wait_status(1), + harness.FigureReport(record=("SyntaxError", 1, "bad")), + (1, ("plot", "SyntaxError", "plot.py:1", "bad")), + ), + (wait_status(1), harness.FigureReport(), (1, ("plot", "Error", None, "the plot step ended with 1"))), + ( + wait_status(signal_number=9), + harness.FigureReport("unavailable", refused=True), + (137, ("crash", "PlotCrashed", None, "the plot step died of SIGKILL")), + ), + ], + ids=[ + "success", + "sys.exit(3) armed", + "exit 3 without a filter it did not require", + "refused", + "a syntax error before the arming", + "the harness's own failure", + "a signal", + ], + ) + def test_the_verdict(self, status: int, report: harness.FigureReport, verdict: tuple) -> None: + assert harness.plot_step_verdict(status, report, "work/plot.py") == verdict + + def test_the_parent_reads_the_report_after_the_wait( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + read_end, write_end = os.pipe() + os.write(write_end, b'armed on\nerror ["SystemExit", 2, "3"]\n') + waited: list[int] = [] + + def wait4(pid: int, options: int) -> tuple[int, int, SimpleNamespace]: + waited.append(pid) + return pid, wait_status(3), SimpleNamespace(ru_maxrss=204800, ru_utime=0.5, ru_stime=0.25) + + monkeypatch.setattr(os, "pipe", lambda: (read_end, write_end)) + monkeypatch.setattr(os, "fork", lambda: 4242) + monkeypatch.setattr(os, "wait4", wait4) + run: dict = {"exec_deny": "off", "steps": {}} + + assert harness.plot_step("plot.py", "plot-light.json", run) == 3 + assert waited == [4242] and run["exec_deny"] == "on" + assert run["steps"]["plot"]["max_rss_mb"] == 200.0 and run["steps"]["plot"]["cpu_s"] == 0.75 + assert last_line(capsys.readouterr().err) == "render: error: plot SystemExit plot.py:2: 3" + + @staticmethod + def child( + monkeypatch: pytest.MonkeyPatch, run_plot: Callable[..., None], *, exec_deny: str = "on" + ) -> tuple[int, harness.FigureReport]: + """`plot_child` with the plot `run_plot(path, guard, armed)`: its exit code and its report.""" + monkeypatch.setattr(harness, "set_plot_limits", lambda: None) + monkeypatch.setattr(harness, "seed_mplconfig", lambda: None) + monkeypatch.setattr(harness, "PlotGuard", lambda allowed: StandInGuard(allowed, exec_deny)) + monkeypatch.setattr(harness, "run_plot", run_plot) + + def exit_(code: int) -> None: + raise Exited(code) + + monkeypatch.setattr(os, "_exit", exit_) + read_end, write_end = os.pipe() + try: + with pytest.raises(Exited) as exited: + harness.plot_child("plot.py", "plot-light.json", write_end) + finally: + os.close(write_end) + os.set_blocking(read_end, False) + return exited.value.code, harness._read_figure_report(read_end) + + @pytest.mark.parametrize( + ("raised", "code", "record"), + [ + (None, 0, None), + ("raise SystemExit(3)", 3, ("SystemExit", 1, "3")), + ("raise SystemExit(259)", 3, ("SystemExit", 1, "259")), + ("raise SystemExit(256)", 0, None), + ("raise SystemExit(None)", 0, None), + ("raise SystemExit('no data')", 1, ("SystemExit", 1, "no data")), + ("x = 1\nraise ValueError('boom')", 1, ("ValueError", 2, "boom")), + ], + ids=["success", "sys.exit(3)", "a code past 255", "256", "None", "a message", "an exception"], + ) + def test_the_childs_code_and_report_after_the_arming( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + raised: str | None, + code: int, + record: tuple | None, + ) -> None: + def run_plot(path: str, guard: StandInGuard, armed: Callable[[str], None]) -> None: + assert guard.allowed == ("plot-light.json",) + armed(guard.exec_deny) + if raised is not None: + raise_in_plot(raised) + + exit_code, report = self.child(monkeypatch, run_plot) + + assert exit_code == code + assert (report.state, report.refused, report.record) == ("on", False, record) + stderr = capsys.readouterr().err + assert ("Traceback (most recent call last):" in stderr) == (record is not None and record[0] == "ValueError") + assert ("no data" in stderr) == (record is not None and record[2] == "no data") + + def test_a_refused_arming_is_reported_before_any_plot_code( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + def run_plot(path: str, guard: StandInGuard, armed: Callable[[str], None]) -> None: + raise harness.ExecDenyUnavailable(guard.exec_deny) + + code, report = self.child(monkeypatch, run_plot, exec_deny="unavailable") + + assert code == harness.EXEC_DENY_EXIT + assert (report.state, report.refused, report.record) == ("unavailable", True, None) + assert "could not refuse process starts (unavailable)" in capsys.readouterr().err + assert harness.plot_step_verdict(wait_status(code), report, "plot.py")[1][:2] == ( + "harness", + "ExecDenyUnavailable", + ) + + def test_the_harnesss_own_failure_in_the_child_is_1_without_a_report(self, monkeypatch: pytest.MonkeyPatch) -> None: + def run_plot(path: str, guard: StandInGuard, armed: Callable[[str], None]) -> None: + pytest.fail("the plot ran") + + def limits() -> None: + raise OSError("setrlimit") + + monkeypatch.setattr(harness, "set_plot_limits", limits) + monkeypatch.setattr(harness, "seed_mplconfig", lambda: None) + monkeypatch.setattr(harness, "run_plot", run_plot) + + def exit_(code: int) -> None: + raise Exited(code) + + monkeypatch.setattr(os, "_exit", exit_) + read_end, write_end = os.pipe() + with pytest.raises(Exited) as exited: + harness.plot_child("plot.py", "plot-light.json", write_end) + os.close(write_end) + os.set_blocking(read_end, False) + report = harness._read_figure_report(read_end) + + assert exited.value.code == 1 and (report.state, report.refused, report.record) == ("off", False, None) + + def test_the_error_record(self) -> None: + guard = StandInGuard((), "on") + try: + raise_in_plot("def f():\n raise KeyError('k')\n\nf()\n") + except KeyError as exc: + assert harness._error_record("plot.py", exc, guard) == ["KeyError", 2, "'k'"] + try: + compile("x = (\n", "plot.py", "exec") + except SyntaxError as exc: + name, line, _ = harness._error_record("plot.py", exc, guard) + assert (name, line) == ("SyntaxError", 1) + odd = type("not a class name", (Exception,), {})("x" * 1000) + assert harness._error_record("plot.py", odd, guard) == ["Error", None, "x" * harness.FIGURE_ERROR_CHARS] + + class Unprintable(Exception): + def __str__(self) -> str: + raise RuntimeError("no text") + + assert harness._error_record("plot.py", Unprintable(), guard) == ["Unprintable", None, ""] + assert harness._error_record("plot.py", SystemExit(4), guard, "4") == ["SystemExit", None, "4"] + + def test_the_screenshot_step_starts_without_the_plots_variables( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + for key in harness.FIGURE_PLOT_ONLY_ENV: + monkeypatch.setenv(key, "1") + monkeypatch.setenv("ANYPLOT_THEME", "dark") + script = ( + "import json, os, sys\n" + "json.dump({'argv': sys.argv[1:], 'env': dict(os.environ)}, open('seen.json', 'w'))\n" + "sys.exit(5)\n" + ) + run: dict = {"steps": {}} + + code = harness.screenshot_step([sys.executable, "-I", "-c", script], "plot-dark.json", "/pkg/plotly", run) + + assert code == 5 + seen = json.loads((tmp_path / "seen.json").read_text(encoding="utf-8")) + assert seen["argv"] == ["--figure=plot-dark.json"] + assert not set(harness.FIGURE_PLOT_ONLY_ENV) & set(seen["env"]) + assert seen["env"][harness.FIGURE_PACKAGE_ENV] == "/pkg/plotly" and seen["env"]["ANYPLOT_THEME"] == "dark" + assert set(run["steps"]["screenshot"]) == {"max_rss_mb", "cpu_s", "wall_s"} + + def test_a_screenshot_step_killed_by_a_signal_is_128_plus_it(self, tmp_path: Path) -> None: + script = "import os, signal\nos.kill(os.getpid(), signal.SIGTERM)\n" + run: dict = {"steps": {}} + assert harness.screenshot_step([sys.executable, "-I", "-c", script], "plot-light.json", "/pkg", run) == 143 + + def test_a_screenshot_step_that_cannot_start_is_127( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + missing = str(tmp_path / "no-such-node") + assert harness.screenshot_step([missing], "plot-light.json", "/pkg", {"steps": {}}) == 127 + assert last_line(capsys.readouterr().err) == ( + f"render: error: harness SpawnError -: cannot run {missing} (FileNotFoundError)" + ) + + @pytest.fixture + def steps(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, list]: + """`run_figure` with the limits, the package's lookup and both steps stood in: what each step was given.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANYPLOT_THEME", "light") + monkeypatch.setattr(harness, "set_run_limits", lambda: None) + monkeypatch.setattr(harness, "find_figure_package", lambda name: f"/pkg/{name}") + given: dict[str, list] = {"plot": [], "screenshot": []} + + def plot_step(path: str, figure: str, run: dict) -> int: + given["plot"].append((path, figure, Path(figure).exists())) + if "TEST_FIGURE" in os.environ: + Path(figure).write_text(os.environ["TEST_FIGURE"], encoding="utf-8") + return int(os.environ.get("TEST_PLOT_EXIT", "0")) + + def screenshot_step(command: list[str], figure: str, package_dir: str, run: dict) -> int: + given["screenshot"].append((command, figure, package_dir)) + return 7 + + monkeypatch.setattr(harness, "plot_step", plot_step) + monkeypatch.setattr(harness, "screenshot_step", screenshot_step) + return given + + def test_run_figure_takes_the_steps_in_order( + self, steps: dict[str, list], tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "plot-light.json").write_text("stale", encoding="utf-8") + monkeypatch.setenv("TEST_FIGURE", "{}") + + assert harness.run_figure(["bokeh", "plot.py", "--", "node", "render.mjs"], {"steps": {}}) == 7 + assert steps["plot"] == [("plot.py", "plot-light.json", False)] # the stale figure removed before + assert steps["screenshot"] == [(["node", "render.mjs"], "plot-light.json", "/pkg/bokeh")] + + @pytest.mark.parametrize( + ("env", "code", "line"), + [ + ({"TEST_PLOT_EXIT": "3"}, 3, None), + ({}, 1, "render: error: figure NoFigure -: the plot wrote no plot-light.json"), + ({"TEST_FIGURE": ""}, 1, "render: error: figure FigureError -: plot-light.json is empty"), + ({"ANYPLOT_THEME": "blue"}, 2, "render: error: harness UsageError -: ANYPLOT_THEME must be light or dark"), + ], + ids=["the plot step failed", "no figure", "an empty figure", "a bad theme"], + ) + def test_run_figure_stops_at_the_first_failure( + self, + steps: dict[str, list], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + env: dict[str, str], + code: int, + line: str | None, + ) -> None: + for key, value in env.items(): + monkeypatch.setenv(key, value) + + assert harness.run_figure(["plotly", "plot.py", "--", "node"], {"steps": {}}) == code + assert steps["screenshot"] == [] + if line is not None: + assert last_line(capsys.readouterr().err) == line + + def test_the_end_line_carries_the_steps_even_when_a_step_raises( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + def run_figure(argv: list[str], run: dict) -> int: + run["exec_deny"] = "on" + run["steps"]["plot"] = {"max_rss_mb": 1.0, "cpu_s": 0.1, "wall_s": 0.2} + raise RuntimeError("the screenshot step broke") + + monkeypatch.setattr(harness, "run_figure", run_figure) + with pytest.raises(RuntimeError): + harness.main_figure(["plotly", "plot.py", "--", "node"]) + lines = capsys.readouterr().out.splitlines() + + assert lines[0] == executor_module.HARNESS_START + end = end_line("\n".join(lines)) + assert end["exec_deny"] == "on" and end["steps"] == {"plot": {"max_rss_mb": 1.0, "cpu_s": 0.1, "wall_s": 0.2}} + + def test_signal_names(self) -> None: + assert harness._signal_name(9) == "SIGKILL" and harness._signal_name(250) == "signal 250" + + +class TestTheRows: + @pytest.mark.parametrize("library", FIGURE_LIBRARIES) + def test_plotly_and_bokeh_run_in_two_steps(self, library: str) -> None: + row = command_for("python", library) + assert row.figure == library and not row.in_process and row.code_file == "plot.py" + assert row.argv == (NODE_BINARY, JS_HARNESS, f"--library={library}") and row.toolchain == (NODE_BINARY,) + assert row.env == JS_ENV + # The plot step's: the filter and the Python limits; the screenshot step drops them. + assert row.exec_deny and row.rlimit_as and row.rlimit_nproc and row.rlimit_as_mb is None + + @pytest.mark.parametrize("node", [True, False], ids=["with Node", "without Node"]) + def test_a_two_step_row_is_served_only_where_node_is( + self, config: ExecutorConfig, monkeypatch: pytest.MonkeyPatch, node: bool + ) -> None: + """The Python language's toolchain names no Node, so the row's own does: without it plotly and bokeh are + `runtime_unavailable` (`Renderer._render`), matplotlib still runs, and `GET /status` still lists Python.""" + + def which(binary: str, mode: int = os.F_OK | os.X_OK, path: str | None = None) -> str | None: + return None if binary == NODE_BINARY and not node else binary + + monkeypatch.setattr(executor_module.shutil, "which", which) + executor = SandboxExecutor(config) + + assert executor.languages()["python"] is True + assert executor.serves("python", "matplotlib") is True + assert all(executor.serves("python", library) is node for library in FIGURE_LIBRARIES) + # A language without its toolchain serves none of its rows. + monkeypatch.setattr(executor, "languages", lambda: {"python": False}) + assert executor.serves("python", "matplotlib") is False + + def test_no_other_row_runs_in_two_steps(self) -> None: + for library, language in commands.LIBRARY_LANGUAGES.items(): + assert bool(command_for(language, library).figure) == (library in FIGURE_LIBRARIES), library + + def test_a_two_step_row_is_a_python_plot_with_a_command(self) -> None: + with pytest.raises(ValueError, match="two-step"): + RuntimeCommand("javascript", "plot.js", ("node",), figure="plotly") + with pytest.raises(ValueError, match="two-step"): + RuntimeCommand("python", "plot.py", (), figure="plotly") + + def test_the_argv_runs_the_harness_in_figure_mode(self, config: ExecutorConfig, tmp_path: Path) -> None: + row = command_for("python", "bokeh") + argv = build_argv(config, "r-a-dark", tmp_path, "dark", 60, command=row) + + assert argv[argv.index(config.harness) :] == [ + config.harness, + FIGURE_FLAG, + "bokeh", + "plot.py", + "--", + NODE_BINARY, + JS_HARNESS, + "--library=bokeh", + ] + + def test_the_environment_is_a_python_rows_plus_the_javascript_rows(self, config: ExecutorConfig) -> None: + limited = replace(config, rlimit_as_mb=1024, rlimit_nproc=64) + env = sandbox_env(limited, "dark", 60, command=command_for("python", "plotly")) + + assert env["ANYPLOT_THEME"] == "dark" and env["MPLBACKEND"] == "Agg" + assert {key: env[key] for key in JS_ENV} == dict(JS_ENV) + assert env["ANYPLOT_DENY_EXEC"] == "required" + assert env["ANYPLOT_RLIMIT_AS_MB"] == "1024" and env["ANYPLOT_RLIMIT_NPROC"] == "64" + assert env["PATH"] == SANDBOX_PATH and env["HOME"] == "/tmp" + # Every other row's environment is what it was. + matplotlib = sandbox_env(limited, "dark", 60, command=command_for("python", "matplotlib")) + assert "ANYPLOT_SANDBOX" not in matplotlib and "PLAYWRIGHT_BROWSERS_PATH" not in matplotlib + + +WRAPPER = ( + "import importlib.util, sys\n" + "spec = importlib.util.spec_from_file_location('harness', {harness!r})\n" + "harness = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(harness)\n" + "harness.find_figure_package = lambda name: {package!r}\n" + "sys.exit(harness.main(sys.argv))\n" +) +"""The harness for the executor's runs: the image's, with the package's lookup standing in.""" + + +class TestTheRenderersTimeout: + """The job's one timeout covers both steps: the executor kills the launcher's process group, the harness and both + of its children with it, whichever step runs.""" + + @pytest.fixture + def figure_executor( + self, config: ExecutorConfig, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> SandboxExecutor: + wrapper = tmp_path / "harness_wrapper.py" + stand_in = tmp_path / "site-packages" / "plotly" + stand_in.mkdir(parents=True) + wrapper.write_text(WRAPPER.format(harness=str(HARNESS_SOURCE), package=str(stand_in)), encoding="utf-8") + shot = tmp_path / "shot.py" + shot.write_text( + "import os, sys, time\n" + "if os.environ.get('ANYPLOT_THEME') == 'dark':\n" + " time.sleep(60)\n" + "open('plot-' + os.environ['ANYPLOT_THEME'] + '.png', 'wb').write(b'\\x89PNG\\r\\n\\x1a\\n')\n", + encoding="utf-8", + ) + row = replace(command_for("python", "plotly"), argv=(sys.executable, "-I", str(shot)), env={}) + monkeypatch.setattr(executor_module, "command_for", lambda language, library: row) + return SandboxExecutor(replace(config, harness=str(wrapper))) + + @staticmethod + def job(source: str, timeout_s: float) -> RenderRequest: + return RenderRequest( + job_id="figure1", + language="python", + library="plotly", + source=source, + data_csv="x\n1\n", + themes=["light"], + timeout_s=timeout_s, + ) + + async def test_a_plot_step_that_runs_on_is_stopped( + self, figure_executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig + ) -> None: + started = time.monotonic() + run = await figure_executor.run_theme(self.job("import time\ntime.sleep(60)\n", 1.5), "light") + + assert run.timed_out and run.reason == "timeout" and run.exit_code is None + assert time.monotonic() - started < 15 + assert launcher.deletes() == launcher.launches() + assert list(config.runs_dir.iterdir()) == [] + + async def test_a_screenshot_step_that_runs_on_is_stopped( + self, figure_executor: SandboxExecutor, launcher: FakeLauncher + ) -> None: + started = time.monotonic() + plot = "open('plot-dark.json', 'w').write('{}')\n" + run = await figure_executor.run_theme(self.job(plot, 2.5), "dark") + + assert run.timed_out and run.reason == "timeout" and run.png_base64 is None + assert time.monotonic() - started < 15 + assert launcher.deletes() == launcher.launches() + + async def test_both_steps_within_the_timeout_render(self, figure_executor: SandboxExecutor) -> None: + run = await figure_executor.run_theme(self.job("open('plot-light.json', 'w').write('{}')\n", 30), "light") + + assert run.exit_code == 0 and not run.timed_out, run.stderr_tail + assert run.png_base64 is not None and run.probe is None diff --git a/tests/unit/agents/renderer/test_service.py b/tests/unit/agents/renderer/test_service.py index e1785ba3708..6c87806c909 100644 --- a/tests/unit/agents/renderer/test_service.py +++ b/tests/unit/agents/renderer/test_service.py @@ -687,6 +687,19 @@ async def test_a_language_without_its_toolchain_is_refused_before_the_slot( response = await client.post("/render", json=body(language="r", library="ggplot2"), headers=auth()) assert response.status_code == 200 and fake.calls == [("job123", "light"), ("job123", "dark")] + async def test_a_row_without_its_own_binaries_is_refused_though_its_language_has_them( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + """plotly's row is Python's language but needs Node for its screenshot step (`RuntimeCommand.figure`).""" + fake.missing_rows.add(("python", "plotly")) + + response = await client.post("/render", json=body(language="python", library="plotly"), headers=auth()) + + assert response.status_code == 503 and response.json() == {"detail": "runtime_unavailable"} + assert fake.calls == [] + response = await client.post("/render", json=body(), headers=auth()) + assert response.status_code == 200 + def test_the_default_renderer_uses_the_sandbox_executor(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("RENDERER_ALLOWED_CALLERS", f"{AGENTS_SA}, owner@example.com") monkeypatch.setenv("RENDERER_AUDIENCES", URL) diff --git a/tests/unit/agents/runtime/test_harness_guard.py b/tests/unit/agents/runtime/test_harness_guard.py index af966c55bd6..6991e0fac24 100644 --- a/tests/unit/agents/runtime/test_harness_guard.py +++ b/tests/unit/agents/runtime/test_harness_guard.py @@ -85,6 +85,53 @@ def run_data(tmp_path: Path, code: str) -> tuple[int, str | None, str]: ) +TEXT_PATH = ( + "class Text(str):\n" + " pass\n" + "def _tries(name):\n" + " method = getattr(str, name)\n" + " def override(self, *args, **kwargs):\n" + " try:\n" + f" {WRITE_MARKER}\n" + " except OSError:\n" + " pass\n" + " return method(self, *args, **kwargs)\n" + " return override\n" + "for _name in ('split', 'rsplit', 'startswith', 'endswith', 'partition', 'rpartition', 'rstrip', 'lstrip', 'strip',\n" + " '__getitem__', '__add__', '__eq__', '__hash__', '__contains__', '__len__', '__iter__', 'find', 'rfind',\n" + " 'join', 'replace', '__str__', '__format__', 'removesuffix', 'count', 'encode', '__fspath__'):\n" + " if hasattr(str, _name):\n" + " setattr(Text, _name, _tries(_name))\n" +) +"""A `str` subclass every method of which tries to write the marker: armed, the guard refuses the write; paused, it +would pass. A path of this class may reach the guard's own path handling.""" +DISARMING_PAYLOAD = ( + "import sys\n" + "frame = sys._getframe(0)\n" + "while frame is not None:\n" + " for value in list(frame.f_locals.values()):\n" + " if hasattr(value, 'armed') and hasattr(value, 'outputs'):\n" + " value.armed = False\n" + " frame = frame.f_back\n" + f"{WRITE_MARKER}\n" +) +"""Module source a plot hands the import machinery (F6): with an import's trust it reads the frames up to the guard, +disarms it and writes the marker.""" +STOCK_LOADER = ( + "import importlib, importlib.abc, os\n" + "import importlib.machinery as machinery\n" + "import importlib.util as util\n" + f"PAYLOAD = {DISARMING_PAYLOAD!r}\n" + "def write_source(name):\n" + " path = os.path.abspath(name)\n" + " open(path, 'w').write(PAYLOAD)\n" + " return path\n" +) +"""The F6 routes' preamble: the machinery's modules and `write_source`, which puts the payload into an output file the +guard lets the plot write and returns its absolute path.""" +RUN_LOADER = "loader.exec_module(util.module_from_spec(util.spec_from_loader('evil', loader)))\n" + + def forged_selector(payload: str) -> str: """A plot body that hands cssselect2's compiler Python objects in place of tokens: a `dimension` whose `int_value` is `payload` lands unquoted in the `eval` text `CompiledSelector.__init__` builds (`CODE_BUILDERS`), and the @@ -145,6 +192,100 @@ def forged_selector(payload: str) -> str: id="InterpreterPoolExecutor", marks=pytest.mark.skipif(sys.version_info < (3, 14), reason="InterpreterPoolExecutor is new in 3.14"), ), + # PB-1's review of the paused paths (2026-10-31): `writes_elsewhere` ran `realpath` on the event's path with the + # guard paused, and a `str` subclass's `split` and `startswith` wrote the marker there; every method of `Text` tries. + pytest.param(TEXT_PATH + "open(Text('/tmp/escape'), 'w')\n", id="a str subclass path to open"), + pytest.param( + TEXT_PATH + "import os\nos.open(Text('/tmp/escape'), os.O_WRONLY | os.O_CREAT)\n", id="... to os.open" + ), + pytest.param( + TEXT_PATH + "class P:\n def __fspath__(self):\n return Text('/tmp/escape')\nopen(P(), 'w')\n", + id="a path object that returns one", + ), + # F6 of PB-1's review (Fable, 2026-10-31): the import machinery driven by the plot. A module body the plot supplies + # would run with an import's trust, read the frames up to the guard, disarm it and write the marker. + pytest.param( + STOCK_LOADER + "loader = machinery.SourceFileLoader('evil', write_source('plot-light.html'))\n" + RUN_LOADER, + id="a stock source loader on a file the plot wrote", + ), + pytest.param( + STOCK_LOADER + "loader = type(np.__loader__)('evil', write_source('plot-light.html'))\n" + RUN_LOADER, + id="the loader class of an admitted module", + ), + pytest.param( + STOCK_LOADER + + "import marshal\npath = os.path.abspath('plot-light.png')\n" + + "code = marshal.dumps(compile(PAYLOAD, 'evil', 'exec'))\n" + + "open(path, 'wb').write(util.MAGIC_NUMBER + bytes(12) + code)\n" + + "loader = machinery.SourcelessFileLoader('evil', path)\n" + + RUN_LOADER, + id="a stock bytecode loader on a file the plot wrote", + ), + pytest.param( + STOCK_LOADER + + "import zipfile, zipimport\npath = os.path.abspath('plot-light.png')\n" + + "with zipfile.ZipFile(path, 'w') as archive:\n archive.writestr('evil.py', PAYLOAD)\n" + + "spec = zipimport.zipimporter(path).find_spec('evil')\nspec.loader.exec_module(util.module_from_spec(spec))\n", + id="zipimport on an archive the plot wrote", + ), + pytest.param( + STOCK_LOADER + + "class Loader(importlib.abc.SourceLoader):\n" + + " def get_filename(self, name):\n return os.__file__\n" + + " def get_data(self, path):\n return PAYLOAD.encode()\n" + + "loader = Loader()\n" + + RUN_LOADER, + id="a loader of the plot's own (Fable's proof of concept)", + ), + pytest.param( + STOCK_LOADER + + "loader = machinery.SourceFileLoader('evil', os.path.join(os.path.dirname(os.__file__), 'colorsys.py'))\n" + + "loader.get_code = lambda name: compile(PAYLOAD, loader.path, 'exec')\n" + + RUN_LOADER, + id="a stock loader whose instance overrides get_code", + ), + # Fable's review of the F6 fix: the method set on the stock class instead of the instance, on a stdlib file and with + # a clean instance dict. Patching the machinery deeper, its module globals (`_bootstrap._call_with_frames_removed`, + # `_frozen_importlib_external`'s helpers) or `builtins.compile`, is F8's kind and knowingly out of the in-process + # layer's reach: the validator, which refuses `sys`, every dunder name and the import machinery's helpers, is the + # barrier there (the harness's security model). + pytest.param( + STOCK_LOADER + + "path = os.path.join(os.path.dirname(os.__file__), 'colorsys.py')\n" + + "machinery.SourceFileLoader.get_code = lambda self, name: compile(PAYLOAD, path, 'exec')\n" + + "loader = machinery.SourceFileLoader('evil', path)\n" + + RUN_LOADER, + id="get_code set on the stock loader's class", + ), + pytest.param( + STOCK_LOADER + + "import sys\nsys.pycache_prefix = os.path.abspath('mpl')\n" # no cached bytecode: the source is compiled + + "path = os.path.join(os.path.dirname(os.__file__), 'colorsys.py')\n" + + "machinery.SourceFileLoader.source_to_code = lambda self, data, path, **kw: compile(PAYLOAD, path, 'exec')\n" + + "loader = machinery.SourceFileLoader('evil', path)\n" + + RUN_LOADER, + id="source_to_code set on the stock loader's class", + ), + pytest.param( + STOCK_LOADER + "import sys\nsys.path_hooks.insert(0, exec)\nsys.path.insert(0, PAYLOAD)\n" + "try:\n importlib.import_module('anyplot_no_such_module')\nexcept ImportError:\n pass\n", + id="exec as a path hook", + ), + pytest.param( + STOCK_LOADER + + "import numpy.random.mtrand as native\n" + + "open('plot-light.png', 'wb').write(open(native.__file__, 'rb').read())\n" + + "loader = machinery.ExtensionFileLoader('mtrand', os.path.abspath('plot-light.png'))\n" + + "util.module_from_spec(util.spec_from_file_location('mtrand', loader.path, loader=loader))\n", + id="an extension module the plot wrote", + ), + pytest.param( + "import pkgutil\nraw = pkgutil.get_data('matplotlib', '../../../../../../../../etc/hostname')\n", + id="pkgutil.get_data past its package", + ), + # Flash's review of PB-1: stdout and stderr are one open file description across figure mode's fork. + pytest.param("import fcntl, os\nfcntl.fcntl(1, fcntl.F_SETFL, os.O_NONBLOCK)\n", id="fcntl on stdout"), + pytest.param("import os\nos.set_blocking(1, False)\n", id="os.set_blocking on stdout"), pytest.param(f"{WRITE_MARKER}\n", id="open for writing"), pytest.param(f"import os\nos.open('{MARKER}', os.O_CREAT | os.O_WRONLY)\n", id="os.open for writing"), pytest.param("plt.figure().savefig('plot-dark.png')\n", id="a save to another name"), @@ -250,6 +391,15 @@ def test_route_fails_the_render_and_leaves_no_trace(self, tmp_path: Path, body: assert not (tmp_path / "plot-dark.png").exists() and not (tmp_path / "mpl" / "plot-light.png").exists() assert (tmp_path / "plot.py").exists() + def test_imports_from_the_install_still_run_under_the_guard(self, tmp_path: Path) -> None: + """F6's rules keep every import of the interpreter's own loaders: a site package with extension modules and an + import-time `ctypes.cast` (scipy.stats), a stdlib module with an extension of `lib-dynload` (lzma), a stdlib + package (zoneinfo) and a frozen module (`__hello__`), each imported first with the guard armed.""" + exit_code, stderr = render(tmp_path, "import scipy.stats\nimport lzma\nimport zoneinfo\nimport __hello__\n") + + assert exit_code == 0, stderr[-2000:] + assert (tmp_path / "plot-light.png").stat().st_size > 0 + @pytest.mark.parametrize("body", CALLBACK_ROUTES) def test_frame_callbacks_are_refused(self, tmp_path: Path, body: str) -> None: exit_code, stderr = render(tmp_path, DISARM + body + WRITE_AFTER) @@ -717,6 +867,37 @@ def test_frame_callbacks_are_refused(self, tmp_path: Path, body: str) -> None: assert exit_code in (DATA_EXIT["blocked"], DATA_EXIT["error"]), stderr assert out == "" and not (tmp_path / MARKER).exists() + def test_the_traceback_is_read_without_the_snippets_code(self, tmp_path: Path) -> None: + """`print_snippet_error` reads the traceback with the guard paused: through `exc.__traceback__`, a property of + the snippet's exception class ran there and wrote the marker under the armed guard (F1 of PB-1, 2026-10-31). + `BaseException`'s own descriptor never calls it.""" + code = ( + "class Escape(Exception):\n" + " @property\n" + " def __traceback__(self):\n" + " try:\n" + f" {WRITE_MARKER}\n" + " except RuntimeError:\n" + " pass\n" + " return None\n" + "\n" + "raise Escape('boom')\n" + ) + exit_code, out, stderr = run_data(tmp_path, code) + + assert exit_code == DATA_EXIT["error"], stderr + assert out == "" and not (tmp_path / MARKER).exists() + assert 'File "snippet.py", line 10, in ' in stderr + assert stderr.strip().splitlines()[-1] == "__main__.Escape: boom" + + def test_the_message_runs_armed(self, tmp_path: Path) -> None: + """The exception's message is the snippet's code too (`__str__`): it runs with the guard armed.""" + code = f"class Escape(Exception):\n def __str__(self):\n {WRITE_MARKER}\n return 'x'\n\nraise Escape()\n" + exit_code, out, stderr = run_data(tmp_path, code) + + assert exit_code == DATA_EXIT["blocked"], stderr + assert out == "" and not (tmp_path / MARKER).exists() + def test_unaudited_calls_are_replaced(self, tmp_path: Path) -> None: code = "import os\nimport pandas as pd\nos.mkfifo('fifo')\ndf = pd.DataFrame()\n" exit_code, out, stderr = run_data(tmp_path, code) @@ -770,6 +951,130 @@ def test_native_and_remote_routes_fail_the_run(self, tmp_path: Path, code: str) assert out == "" +class TestImportMachinery: + """What an import must look like for the plot guard to trust it (`harness.ImportMachinery`, F6 of PB-1's review); + the routes above run the same rules armed.""" + + STDLIB_FILE = str(Path(json.__file__).parent.parent / "colorsys.py") + + def test_the_install_holds_the_stdlib_and_the_site_packages_and_not_the_run(self, tmp_path: Path) -> None: + prefixes = harness.install_prefixes() + assert harness.under_prefixes(self.STDLIB_FILE, prefixes) + assert harness.under_prefixes(pytest.__file__, prefixes) + assert not harness.under_prefixes(str(tmp_path / "plot-light.html"), prefixes) + assert not harness.under_prefixes("/tmp/evil.py", prefixes) + (tmp_path / "link.py").symlink_to(self.STDLIB_FILE) + assert harness.under_prefixes(str(tmp_path / "link.py"), prefixes) # its real path is the stdlib's + (tmp_path / "out.py").write_text("x = 1\n", encoding="utf-8") + climbing = str(Path(self.STDLIB_FILE).parent) + "/.." * 20 + str(tmp_path / "out.py") + assert not harness.under_prefixes(climbing, prefixes) # starts with a prefix, resolves outside + + def test_only_an_exact_stock_loader_of_the_install_is_trusted(self, tmp_path: Path) -> None: + import importlib.abc + import importlib.machinery + import zipimport + + machinery = harness.ImportMachinery() + stock = importlib.machinery.SourceFileLoader("colorsys", self.STDLIB_FILE) + assert machinery.trusted_loader(stock) + + class Subclass(importlib.machinery.SourceFileLoader): + pass + + class Own(importlib.abc.SourceLoader): + def get_filename(self, name: str) -> str: + return "colorsys.py" + + def get_data(self, path: str) -> bytes: + return b"" + + overridden = importlib.machinery.SourceFileLoader("colorsys", self.STDLIB_FILE) + overridden.get_code = lambda name: None # type: ignore[method-assign] + assert not machinery.trusted_loader(Subclass("colorsys", self.STDLIB_FILE)) + assert not machinery.trusted_loader(Own()) + assert not machinery.trusted_loader(overridden) + assert not machinery.trusted_loader(importlib.machinery.SourceFileLoader("x", str(tmp_path / "x.py"))) + assert not machinery.trusted_loader(object()) + archive = tmp_path / "evil.zip" + import zipfile + + with zipfile.ZipFile(archive, "w") as handle: + handle.writestr("evil.py", "x = 1\n") + assert not machinery.trusted_loader(zipimport.zipimporter(str(archive))) + + def test_a_stock_loader_whose_class_changed_is_not_trusted(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Fable's review of the F6 fix: a class attribute changes with no audit event, so the stock loaders' classes + are read before arming and a loader is trusted only while its class resolves every name as it did then. Each + patch is undone at once: the test process imports through these classes.""" + import importlib.machinery + import typing + import zipimport + + external = sys.modules["_frozen_importlib_external"] + machinery = harness.ImportMachinery() + stock = importlib.machinery.SourceFileLoader("colorsys", self.STDLIB_FILE) + typing.get_type_hints(importlib.machinery.SourceFileLoader) # Python's own lazy entries keep the trust + assert machinery.trusted_loader(stock) + + def stand_in(*args: object, **kwargs: object) -> None: + return None + + changes = [ + (importlib.machinery.SourceFileLoader, "get_code"), # inherited: a new entry of the class itself + (external.SourceLoader, "source_to_code"), + (external.FileLoader, "get_data"), + (importlib.machinery.SourceFileLoader, "__getattr__"), # a name none of the classes had + ] + for cls, name in changes: + with monkeypatch.context() as patch: + patch.setattr(cls, name, stand_in, raising=False) + assert not machinery.trusted_loader(stock), name + assert machinery.trusted_loader(stock), name + with monkeypatch.context() as patch: + patch.delattr(external.SourceLoader, "path_stats") + assert not machinery.trusted_loader(stock) + assert machinery.trusted_loader(stock) + + class Shim(external._LoaderBasics): # type: ignore[name-defined] + pass + + sourceless = importlib.machinery.SourcelessFileLoader + bases = sourceless.__bases__ + try: # monkeypatch would undo `__bases__` with a `delattr`, which a class refuses + sourceless.__bases__ = (external.FileLoader, Shim) + assert not machinery.intact(sourceless) # the same entries, another resolution order + finally: + sourceless.__bases__ = bases + assert machinery.intact(sourceless) + for cls, name in [ + (importlib.machinery.ExtensionFileLoader, "create_module"), + (zipimport.zipimporter, "get_code"), + ]: + with monkeypatch.context() as patch: + patch.setattr(cls, name, stand_in) + assert not machinery.intact(cls), name + assert machinery.intact(cls), name + + def test_only_the_interpreters_own_import_frames_earn_trust(self) -> None: + machinery = harness.ImportMachinery() + + def frame(code: object, back: object = None, local_self: object = None) -> object: + return type("Frame", (), {"f_code": code, "f_back": back, "f_locals": {"self": local_self}}) + + import importlib.machinery + + stock = importlib.machinery.SourceFileLoader("colorsys", self.STDLIB_FILE) + exec_module = frame(next(iter(machinery.loader_entries)), local_self=stock) + assert machinery.trusts(frame(machinery.trampoline, back=exec_module)) + assert machinery.trusts(frame(machinery.frozen_exec)) + assert machinery.trusts(frame(machinery.trampoline, back=frame(next(iter(machinery.static_entries))))) + assert not machinery.trusts(frame(machinery.trampoline, back=frame((lambda: 0).__code__))) + assert not machinery.trusts(frame(machinery.trampoline)) + assert not machinery.trusts(frame((lambda: 0).__code__)) # `_path_hooks`, `_find_spec` and the like + untrusted = frame(next(iter(machinery.loader_entries)), local_self=object()) + assert not machinery.trusts(frame(machinery.trampoline, back=untrusted)) + + def test_ctypes_helper_addresses_are_empty_without_ctypes(monkeypatch: pytest.MonkeyPatch) -> None: """An interpreter without `_ctypes` gives the plot guard no address to let through.""" monkeypatch.setitem(sys.modules, "_ctypes", None) # `import _ctypes` raises ImportError diff --git a/tests/unit/automation/test_js_render_figure.py b/tests/unit/automation/test_js_render_figure.py new file mode 100644 index 00000000000..e43fda2e7f7 --- /dev/null +++ b/tests/unit/automation/test_js_render_figure.py @@ -0,0 +1,447 @@ +"""The JavaScript render harness's figure mode (`automation/js-render/render.mjs --library= +--figure=`, PB-1 of step 4 in agentic/specs/261024-agent-all-libraries.md), run through Node and Chromium, and +the whole two-step render through the Python harness. + +Each case draws a figure with the package's own bundled JavaScript, taken from the installed Python package as the +renderer takes it, so the module runs where Node, the lock's packages, Playwright's browser and plotly or bokeh are +present and is skipped elsewhere; the renderer image's CI step (`ci-image.yml`) renders through the built image. The +harness is copied into a checkout's layout, as in `test_js_render_sandbox.py`. +""" + +import importlib.util +import json +import os +import re +import shutil +import struct +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from agents.anyplot.render import harness as python_harness +from agents.anyplot.render.runtimes.python import HARNESS_SOURCE + + +REPO_ROOT = Path(__file__).resolve().parents[3] +HARNESS = REPO_ROOT / "automation" / "js-render" / "render.mjs" +SANDBOX_ENV = {"ANYPLOT_SANDBOX": "1", "ANYPLOT_NO_HTML": "1"} +ERROR_LINE = re.compile( + r"^render: error: (page|import|build|draw|crash|data|timeout|harness|figure) ([A-Za-z_$][\w$]*) (\S+:\d+|-): (.*)$" +) +"""The final error line's grammar, with figure mode's `figure` kind.""" +BAR = {"data": [{"type": "bar", "x": ["a", "b", "c"], "y": [3, 1, 2]}], "layout": {"title": {"text": "bars"}}} + + +def package_dir(name: str) -> Path | None: + spec = importlib.util.find_spec(name) + return Path(next(iter(spec.submodule_search_locations))) if spec and spec.submodule_search_locations else None + + +def plotly_figure(width: float | None = None, height: float | None = None, **layout: object) -> dict[str, object]: + figure = json.loads(json.dumps(BAR)) + if width is not None: + figure["layout"]["width"] = width + if height is not None: + figure["layout"]["height"] = height + figure["layout"].update(layout) + return figure + + +@dataclass +class Run: + returncode: int + stdout: str + stderr: str + work: Path + + @property + def last_line(self) -> str: + lines = self.stderr.strip().splitlines() + return lines[-1] if lines else "" + + def png_size(self, theme: str = "light") -> tuple[int, int]: + png = (self.work / f"plot-{theme}.png").read_bytes() + assert png[:8] == b"\x89PNG\r\n\x1a\n" + return struct.unpack(">II", png[16:24]) + + +@dataclass +class Harness: + root: Path + node: str + + @property + def render_mjs(self) -> Path: + return self.root / "automation" / "js-render" / "render.mjs" + + def environment(self, extra: dict[str, str] | None = None, *, sandbox: bool = True) -> dict[str, str]: + environment = {key: value for key, value in os.environ.items() if not key.startswith("ANYPLOT_")} + environment |= {"ANYPLOT_THEME": "light", "ANYPLOT_NODE_MODULES": str(self.root / "node_modules")} + if sandbox: + environment |= SANDBOX_ENV + return environment | (extra or {}) + + def draw( + self, + tmp_path: Path, + figure: object, + *, + library: str = "plotly", + args: list[str] | None = None, + env: dict[str, str] | None = None, + sandbox: bool = True, + package: bool = True, + ) -> Run: + work = tmp_path / "work" + work.mkdir(parents=True, exist_ok=True) + if figure is not None: + text = figure if isinstance(figure, str) else json.dumps(figure) + (work / "plot-light.json").write_text(text, encoding="utf-8") + extra = dict(env or {}) + found = package_dir(library) if library in ("plotly", "bokeh") else None + if package and found is not None: + extra["ANYPLOT_FIGURE_PACKAGE"] = str(found) + argv = args if args is not None else [f"--library={library}", "--figure=plot-light.json"] + done = subprocess.run( + [self.node, str(self.render_mjs), *argv], + cwd=work, + env=self.environment(extra, sandbox=sandbox), + capture_output=True, + text=True, + timeout=120, + check=False, + ) + return Run(done.returncode, done.stdout, done.stderr, work) + + +def _node_modules() -> Path | None: + root = Path(os.environ.get("ANYPLOT_NODE_MODULES", REPO_ROOT / "node_modules")) + return root if (root / "playwright" / "package.json").is_file() else None + + +@pytest.fixture(scope="module") +def harness(tmp_path_factory: pytest.TempPathFactory) -> Harness: + node = shutil.which("node") + modules = _node_modules() + if node is None or modules is None: + pytest.skip("needs node on PATH and the lock's node_modules (npm ci, or ANYPLOT_NODE_MODULES)") + if package_dir("plotly") is None: + pytest.skip("needs plotly, whose bundled plotly.js draws the figures (the lib-plotly extra)") + root = tmp_path_factory.mktemp("checkout") + (root / "automation" / "js-render").mkdir(parents=True) + shutil.copy(HARNESS, root / "automation" / "js-render" / "render.mjs") + (root / "node_modules").symlink_to(modules.resolve(), target_is_directory=True) + probe = Harness(root, node).draw(tmp_path_factory.mktemp("probe"), plotly_figure(800, 450)) + if probe.returncode != 0: + pytest.skip(f"the harness cannot render here (no browser?): {probe.stderr.strip()[-300:]}") + return Harness(root, node) + + +def bokeh_or_skip(): + if package_dir("bokeh") is None: + pytest.skip("needs bokeh, whose bundled BokehJS draws the figures (the lib-bokeh extra)") + import bokeh.embed + import bokeh.layouts + import bokeh.models + import bokeh.plotting + + return bokeh + + +def bokeh_item(width: int, height: int, **kwargs: object) -> dict[str, object]: + bokeh = bokeh_or_skip() + plot = bokeh.plotting.figure(width=width, height=height, toolbar_location=None, **kwargs) + plot.line([1, 2, 3], [3, 1, 2], line_width=4) + return bokeh.embed.json_item(plot) + + +def assert_final_line(run: Run, kind: str, cls: str) -> str: + assert run.returncode != 0, run.stderr + match = ERROR_LINE.match(run.last_line) + assert match is not None, run.stderr + assert (match.group(1), match.group(2), match.group(3)) == (kind, cls, "-"), run.last_line + return match.group(4) + + +class TestTheCanvas: + @pytest.mark.parametrize( + ("size", "pixels", "scale"), + [ + ((800, 450), (3200, 1800), "4.000"), + ((1600, 900), (3200, 1800), "2.000"), + ((600, 600), (2400, 2400), "4.000"), + ((1200, 1200), (2400, 2400), "2.000"), + ((3600, 3600), (2400, 2400), "0.667"), + ((1000, 700), (3200, 1800), "2.564"), + ((700, 1000), (2400, 2400), "2.400"), + ((None, None), (3200, 1800), "2.000"), + ((1200, None), (3200, 1800), "2.667"), + ((None, 600), (3200, 1800), "2.985"), + ], + ids=["800x450", "1600x900", "600x600", "1200x1200", "3600x3600", "wide", "tall", "no size", "width", "height"], + ) + def test_a_plotly_figure_keeps_its_size_and_the_scale_makes_the_canvas( + self, harness: Harness, tmp_path: Path, size: tuple, pixels: tuple[int, int], scale: str + ) -> None: + """The catalogue writes plotly at 800x450 scale 4 (146 files), 1600x900 scale 3 (116), 600x600 scale 4 (43) + and 1200x1200 scale 3 (9): the layout stays, the scale follows the canvas.""" + run = harness.draw(tmp_path, plotly_figure(*size)) + + assert run.returncode == 0, run.stderr + assert run.png_size() == pixels + assert f"at scale {scale}" in run.stdout + assert not (run.work / "plot-light.html").exists() + + @pytest.mark.parametrize( + ("size", "drawn"), [((1200, None), "1200x675"), ((None, 450), "800x450")], ids=["width", "height"] + ) + def test_a_plotly_figure_with_one_side_keeps_it_and_gets_the_other_at_16_to_9( + self, harness: Harness, tmp_path: Path, size: tuple, drawn: str + ) -> None: + """The side the code named is drawn, not replaced by 1600 x 900; the other follows the landscape aspect.""" + run = harness.draw(tmp_path, plotly_figure(*size)) + + assert run.returncode == 0, run.stderr + assert f"figure {drawn} at scale" in run.stdout + + def test_a_bokeh_figure_at_the_catalogues_4800_wide_draws_at_two_thirds( + self, harness: Harness, tmp_path: Path + ) -> None: + run = harness.draw(tmp_path, bokeh_item(4800, 2700), library="bokeh") + + assert run.returncode == 0, run.stderr + assert run.png_size() == (3200, 1800) and "figure 4800x2700 at scale 0.667" in run.stdout + + def test_a_bokeh_layout_is_measured_first(self, harness: Harness, tmp_path: Path) -> None: + bokeh = bokeh_or_skip() + plots = [] + for _ in range(2): + plot = bokeh.plotting.figure(width=800, height=450, toolbar_location=None) + plot.line([1, 2], [3, 4]) + plots.append(plot) + run = harness.draw(tmp_path, bokeh.embed.json_item(bokeh.layouts.row(*plots)), library="bokeh") + + assert run.returncode == 0, run.stderr + assert run.png_size() == (3200, 1800) and "figure 1600x450 at scale 2.000" in run.stdout + + def test_the_dark_theme_draws_too(self, harness: Harness, tmp_path: Path) -> None: + run = harness.draw(tmp_path, plotly_figure(800, 450), env={"ANYPLOT_THEME": "dark"}, args=None) + assert run.returncode == 0, run.stderr + assert run.png_size("dark") == (3200, 1800) + + +class TestTheFigureIsData: + def test_markup_in_a_text_stays_text(self, harness: Harness, tmp_path: Path) -> None: + """A title that would close the JSON element and open a script draws as text and runs nothing.""" + title = "