From 3d3ffd899d10d867b3f8f7b1902bd93b832b1fbd Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 16:10:35 -0800 Subject: [PATCH 1/5] feat(task-steps): computer_exec, a command run inside a computer through its driver Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- .../env/envs/computer/utils/inside.py | 11 +- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 2 + .../task_steps/computer/computer_exec.py | 188 +++++++++++++++++ .../computer_exec/envs/exec-desk/env.toml | 3 + .../computer/computer_exec/tasks/setup.json | 13 ++ .../computer/computer_exec_test.py | 29 +++ tst/unit/computer_home.py | 5 +- tst/unit/task_step/computer_exec_test.py | 195 ++++++++++++++++++ 10 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/computer/computer_exec.py create mode 100644 tst/integration/computer/computer_exec/envs/exec-desk/env.toml create mode 100644 tst/integration/computer/computer_exec/tasks/setup.json create mode 100644 tst/integration/computer/computer_exec_test.py create mode 100644 tst/unit/task_step/computer_exec_test.py diff --git a/AGENTS.md b/AGENTS.md index 784e1ea05..543ee8072 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/env/envs/computer/utils/inside.py b/src/agent_env/env/envs/computer/utils/inside.py index c03ccecaa..b77ffd885 100644 --- a/src/agent_env/env/envs/computer/utils/inside.py +++ b/src/agent_env/env/envs/computer/utils/inside.py @@ -92,11 +92,12 @@ async def read_file(deployed: DeployedSandboxEnv, path: str) -> bytes: async def run(deployed: DeployedSandboxEnv, argv: list[str], *, timeout_s: float, - env: Optional[dict[str, str]] = None) -> dict: - """computer-exec/v1 ``run``'s answer for ``argv``, under a request id of its own. A call whose connection breaks - fails rather than being sent again: a driver that restarted would run the command a second time, beside the first, - which outlives the driver that started it.""" - params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {})} + env: Optional[dict[str, str]] = None, cwd: Optional[str] = None) -> dict: + """computer-exec/v1 ``run``'s answer for ``argv``, in ``cwd`` relative to the home, the home by default, under a + request id of its own. A call whose connection breaks fails rather than being sent again: a driver that restarted + would run the command a second time, beside the first, which outlives the driver that started it.""" + params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {}), + **({"cwd": cwd} if cwd else {})} try: return await deployed.invoke(COMPUTER_EXEC, "run", params, timeout=timeout_s + CALL_MARGIN_S) except httpx.TransportError as e: diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index f6cd5045a..dcf251f19 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -156,7 +156,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage", "browser_close_tab", "browser_delete_cookies", "browser_lease_context", "browser_open_tab", "browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli", - "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", + "combine_universe_verdicts", "computer_exec", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", "end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "user_sim_judge", "run_container_unit_tests_verifier", "run_docker_container", "set_env_clock_rate", "snapshot_env", "snapshot_computer_env", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index c4b5c5b9d..43bd7f79e 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -104,6 +104,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep + from agent_env.task_step.task_steps.computer.computer_exec import ComputerExecTaskStep from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep @@ -174,6 +175,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(SnapshotAgentStateTaskStep): SnapshotAgentStateTaskStep, _get_type(SnapshotEnvTaskStep): SnapshotEnvTaskStep, _get_type(SnapshotComputerEnvTaskStep): SnapshotComputerEnvTaskStep, + _get_type(ComputerExecTaskStep): ComputerExecTaskStep, _get_type(RunCodeTaskStep): RunCodeTaskStep, _get_type(PeerAgentsTaskStep): PeerAgentsTaskStep, _get_type(VerifyA2ASnapshotStep): VerifyA2ASnapshotStep, diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py new file mode 100644 index 000000000..e253b417f --- /dev/null +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -0,0 +1,188 @@ +"""computer_exec: a command run inside a deployed computer as its session user, through its driver's computer-exec/v1 +``run``, a control route the agent can't call, as a task sets a computer up: a program run to its end, its exit status +and output recorded, or with ``background`` an app started and left running.""" + +from __future__ import annotations + +import logging +import shlex +from typing import ClassVar, Optional, Union + +from agentenv_protocol.computer import COMPUTER_EXEC + +from agent_env.entity.refs import EntityRef +from agent_env.env.env import DeployedEnv, DeployedSandboxEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils import inside +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.preflight import stored_env +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +logger = logging.getLogger(__name__) + +#: Where the step records what each run did, by step id. +RECORDS_KEY = "computer_exec" +#: How long a command may run when the step doesn't say. +DEFAULT_TIMEOUT_S = 120.0 +#: The most of each stream the record keeps, and the most of stderr a failure says: the end of it. +RECORDED_CHARS = 4000 +FAILURE_CHARS = 300 +#: How a ``background`` command starts: its program found and executable, else it fails as the shell does with 127, +#: then detached from the run's pipes, so the run returns as the shell does. +_BACKGROUND = ('case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' + ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' + '"$@" /dev/null 2>&1 &') + +Command = Union[list[str], str] + + +class ComputerExecTaskStep(TaskStep): + """Run ``command`` inside the computer of the env ``env_id`` as its session user, in ``cwd`` (relative to the home, + the home by default), with ``env`` added to its environment, for at most ``timeout_s``. ``command`` is a program and + its arguments, or a string run by ``sh -c``. For a multi env, ``computer`` names the computer, by its environment + name, and may be left out when the env has one. + + The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and + stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless + ``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app + launched for the agent is: the step waits for the start alone, and fails as a run does when its program isn't + there to start.""" + + type: ClassVar[str] = "computer_exec" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, command: Command, + computer: Optional[str] = None, cwd: Optional[str] = None, env: Optional[dict[str, str]] = None, + timeout_s: float = DEFAULT_TIMEOUT_S, background: bool = False, check: bool = True, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.command = command + self.computer = computer + self.cwd = cwd + self.env = env + self.timeout_s = timeout_s + self.background = background + self.check = check + + def config_problems(self) -> list[str]: + problems = [] + if not isinstance(self.env_id, str) or not self.env_id: + problems.append(f"step {self.id!r}: env_id must be a non-empty string") + if isinstance(self.command, str): + if not self.command.strip(): + problems.append(f"step {self.id!r}: command must not be blank") + elif not (isinstance(self.command, list) and self.command and all(isinstance(a, str) for a in self.command)): + problems.append(f"step {self.id!r}: command must be a non-empty list of strings, or a string run by sh -c") + if self.computer is not None and (not isinstance(self.computer, str) or not self.computer): + problems.append(f"step {self.id!r}: computer, when given, must be a non-empty string") + if self.cwd is not None and (not isinstance(self.cwd, str) or self.cwd.startswith("/")): + problems.append(f"step {self.id!r}: cwd is a folder relative to the session user's home, not {self.cwd!r}") + if self.env is not None and not (isinstance(self.env, dict) and all( + isinstance(k, str) and k and isinstance(v, str) for k, v in self.env.items())): + problems.append(f"step {self.id!r}: env maps variable names to string values") + if isinstance(self.timeout_s, bool) or not isinstance(self.timeout_s, (int, float)) or self.timeout_s <= 0: + problems.append(f"step {self.id!r}: timeout_s must be a number of seconds above 0") + for name in ("background", "check"): + if not isinstance(getattr(self, name), bool): + problems.append(f"step {self.id!r}: {name} must be true or false") + return problems + + def preflight(self) -> list[str]: + """The env this step names when it has no computer, or none of that name, or several when it names none.""" + problems = super().preflight() + if not isinstance(self.env_id, str) or not self.env_id or (env := stored_env(self.env_id)) is None: + return problems + if problem := _computer_problem(env, self.computer): + problems.append(f"step {self.id!r}: {problem}") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "command": self.command, "computer": self.computer, + "cwd": self.cwd, "env": self.env, "timeout_s": self.timeout_s, "background": self.background, + "check": self.check} + + @classmethod + def from_dict(cls, data: dict) -> ComputerExecTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], command=data["command"], + computer=data.get("computer"), cwd=data.get("cwd"), env=data.get("env"), + timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False), + check=data.get("check", True)) + + def argv(self) -> list[str]: + """What the driver runs: the command, through ``sh -c`` when it's a string, started detached with + ``background``.""" + argv = ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command) + return ["sh", "-c", _BACKGROUND, "computer_exec", *argv] if self.background else argv + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = find_computer(context, self.env_id, self.computer) + if missing := [m for m in inside.missing(deployed) if m.startswith(COMPUTER_EXEC)]: + raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer serves no {missing[0]}") + if self.timeout_s > (limit := inside.max_timeout_s(deployed)): + raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer " + f"of env {self.env_id!r} lets a command run") + argv = self.argv() + answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd) + record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "background": self.background, + "exit_code": answer.get("exit_code"), "timed_out": bool(answer.get("timed_out")), + "truncated": bool(answer.get("truncated")), + "stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS), + "stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)} + context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record + shown = shlex.join(argv) + if record["timed_out"]: + outcome = f"timed out after {self.timeout_s} s" + elif record["exit_code"] != 0: + outcome = f"exited {record['exit_code']}" + else: + logger.info(f"{self.id}: {'started' if self.background else 'ran'} {shown} in env {self.env_id!r}") + return context + message = f"{self.type} {self.id!r}: {shown} {outcome} in env {self.env_id!r}" + if stderr := _tail(record["stderr"].strip(), FAILURE_CHARS): + message += f": {stderr}" + if self.check: + raise RuntimeError(message) + logger.warning(f"{message} (check is off)") + return context + + +def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv: + """The deploy record of the computer in ``env_id`` that serves computer-exec/v1 ``run``: the env's own, or one of a + multi env's computers' (``DeployedGatewayEnv.child_envs``), the one ``computer`` names else its only one.""" + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") + children = {name: DeployedEnv.from_dict(data) for name, data in (getattr(deployed, "child_envs", None) or {}).items()} + if computer is not None: + found = children.get(computer) + if found is None: + raise RuntimeError(f"env {env_id!r} has no computer named {computer!r} (it has {sorted(children)})") + return found + if deployed.supports(COMPUTER_EXEC, "run"): + return deployed + computers = [record for record in children.values() if record.supports(COMPUTER_EXEC, "run")] + if len(computers) != 1: + raise RuntimeError(f"env {env_id!r} has {len(computers) or 'no'} computers serving {COMPUTER_EXEC} run" + + (f": name one as computer, of {sorted(children)}" if computers else "")) + return computers[0] + + +def _computer_problem(env: object, computer: Optional[str]) -> Optional[str]: + if isinstance(env, ComputerEnv): + return None if computer is None else f"env {env.id!r} is a computer env, so it takes no computer name" + if not isinstance(env, MultiEnv): + return f"env {getattr(env, 'id', '?')!r} is no computer env, nor a multi env with one" + browser = env.website_browser().computer + names = [computer_env.environment_name for computer_env in (*env.computer_envs, *([browser] if browser else []))] + if computer is not None and computer not in names: + return f"env {env.id!r} has no computer named {computer!r} (it has {names})" + if computer is None and len(names) != 1: + return (f"env {env.id!r} has {len(names) or 'no'} computers" + + (f": name one as computer, of {names}" if names else "")) + return None + + +def _tail(text: str, chars: int) -> str: + return text if len(text) <= chars else "…" + text[-chars:] diff --git a/tst/integration/computer/computer_exec/envs/exec-desk/env.toml b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml new file mode 100644 index 000000000..e1dca97f8 --- /dev/null +++ b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/computer_exec/tasks/setup.json b/tst/integration/computer/computer_exec/tasks/setup.json new file mode 100644 index 000000000..da776c039 --- /dev/null +++ b/tst/integration/computer/computer_exec/tasks/setup.json @@ -0,0 +1,13 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "exec-desk"}, + {"id": "write", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["env"], + "command": ["sh", "-c", "mkdir -p Desktop && echo computer-exec > Desktop/note.txt && cat Desktop/note.txt"]}, + {"id": "launch", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["write"], + "command": ["thunar", "Desktop"], "background": true}, + {"id": "window", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["launch"], "timeout_s": 60, + "command": "for i in $(seq 100); do xwininfo -root -tree | grep -i -m1 'thunar' && exit 0; sleep 0.5; done; exit 1"}, + {"id": "tolerated", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["window"], + "command": "echo nope >&2; exit 7", "check": false}, + {"id": "missing", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["tolerated"], + "command": ["no-such-app", "Desktop"], "background": true, "check": false} +] diff --git a/tst/integration/computer/computer_exec_test.py b/tst/integration/computer/computer_exec_test.py new file mode 100644 index 000000000..d2a1ce7c4 --- /dev/null +++ b/tst/integration/computer/computer_exec_test.py @@ -0,0 +1,29 @@ +"""computer_exec on a computer env, end to end: the ubuntu-24.04 base with the reference driver deployed, a command run +in the session user's home to its end, its output recorded; Thunar started with ``background`` on the folder that wrote, +as a task opens a file in an app for the agent, and its window then found on the display, so the app outlived the run +that started it; and a failing command, and a background one whose program isn't there, recorded, not failing the +task, with ``check`` off.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from tst.integration.computer.conftest import run_context + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BUNDLE = Path(__file__).parent / "computer_exec" + + +def test_commands_run_in_a_computer_and_an_app_started_in_the_background_stays_up(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "setup", "--no-emulate"]) + + assert ran.exit_code == 0, ran.output + records = run_context(ran.output)["metadata"]["computer_exec"] + assert records["write"]["exit_code"] == 0 and records["write"]["stdout"] == "computer-exec\n", records["write"] + assert records["launch"]["exit_code"] == 0 and records["launch"]["background"], records["launch"] + assert records["window"]["exit_code"] == 0 and "thunar" in records["window"]["stdout"].lower(), records["window"] + assert records["tolerated"]["exit_code"] == 7 and records["tolerated"]["stderr"] == "nope\n", records["tolerated"] + assert records["missing"]["exit_code"] == 127 and "not found" in records["missing"]["stderr"], records["missing"] diff --git a/tst/unit/computer_home.py b/tst/unit/computer_home.py index 5f7d8f086..07ed95c94 100644 --- a/tst/unit/computer_home.py +++ b/tst/unit/computer_home.py @@ -74,8 +74,9 @@ async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, v def _run(self, params: dict) -> dict: try: - done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})}, - capture_output=True, text=True, timeout=params["timeout_s"]) + done = subprocess.run(params["argv"], cwd=self.home / (params.get("cwd") or ""), + env={**os.environ, **(params.get("env") or {})}, capture_output=True, text=True, + timeout=params["timeout_s"]) except subprocess.TimeoutExpired: return {"exit_code": None, "stdout": "", "stderr": "", "truncated": False, "timed_out": True} return {"exit_code": done.returncode, "stdout": done.stdout, "stderr": done.stderr, "truncated": False, diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py new file mode 100644 index 000000000..4e6fd89f1 --- /dev/null +++ b/tst/unit/task_step/computer_exec_test.py @@ -0,0 +1,195 @@ +"""computer_exec against a computer at its driver's boundary (``tst.unit.computer_home``): a command run in its home as +given or through ``sh -c``, in a folder and with variables of its own, its run recorded; a failure or a timeout failing +the step unless ``check`` is off; a ``background`` command started and left running; a multi env's computer found by +name; and the configs it refuses.""" + +from __future__ import annotations + +import dataclasses +import time +from types import SimpleNamespace + +import pytest + +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.multi_env import MultiEnv, WebsiteBrowser +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.computer.computer_exec import RECORDS_KEY, ComputerExecTaskStep +from tst.unit.computer_home import TOKEN, Computer + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + return Computer(tmp_path).install(monkeypatch) + + +def _context(*deployed) -> TaskStepContext: + return TaskStepContext(deployed_agents=[], deployed_envs=list(deployed), metadata={}) + + +def _step(command, **fields) -> ComputerExecTaskStep: + return ComputerExecTaskStep(id="exec", version=None, env_id=fields.pop("env_id", "desk"), command=command, **fields) + + +@pytest.mark.asyncio +async def test_a_command_runs_in_the_home_as_given_with_the_computers_token_and_is_recorded(computer): + ctx = await _step(["sh", "-c", "pwd; echo out; echo err >&2"]).execute(_context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["exit_code"] == 0 and not record["timed_out"] + assert record["stdout"] == f"{computer.home}\nout\n" and record["stderr"] == "err\n" + assert computer.headers == [{"Authorization": f"Bearer {TOKEN}"}] + _, params, _ = computer.calls[0] + assert params["argv"] == ["sh", "-c", "pwd; echo out; echo err >&2"] and params["timeout_s"] == 120.0 + + +@pytest.mark.asyncio +async def test_a_string_runs_through_sh_in_its_folder_with_its_variables(computer): + (computer.home / "work").mkdir() + + ctx = await _step('echo "$GREETING" > made.txt', cwd="work", env={"GREETING": "hi"}, + timeout_s=5).execute(_context(computer.deployed())) + + assert (computer.home / "work" / "made.txt").read_text() == "hi\n" + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"] == ["sh", "-c", 'echo "$GREETING" > made.txt'] + _, params, _ = computer.calls[0] + assert params["cwd"] == "work" and params["env"] == {"GREETING": "hi"} and params["timeout_s"] == 5 + + +@pytest.mark.asyncio +async def test_a_command_that_fails_fails_the_step_with_the_end_of_its_stderr_and_is_still_recorded(computer): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 3 in env 'desk': no such thing"): + await _step("echo 'no such thing' >&2; exit 3").execute(ctx) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 3 + + +@pytest.mark.asyncio +async def test_with_check_off_a_failure_is_recorded_and_the_step_goes_on(computer): + ctx = await _step("exit 4", check=False).execute(_context(computer.deployed())) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 4 + + +@pytest.mark.asyncio +async def test_a_command_that_times_out_fails_the_step(computer): + with pytest.raises(RuntimeError, match=r"timed out after 0.2 s"): + await _step("sleep 5", timeout_s=0.2).execute(_context(computer.deployed())) + + +@pytest.mark.asyncio +async def test_a_background_command_is_started_and_left_running(computer): + started = time.monotonic() + + ctx = await _step("sleep 1; echo done > late.txt", background=True).execute(_context(computer.deployed())) + + assert time.monotonic() - started < 1 + assert not (computer.home / "late.txt").exists() + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", "sleep 1; echo done > late.txt"] + for _ in range(50): + if (computer.home / "late.txt").exists(): + break + time.sleep(0.1) + assert (computer.home / "late.txt").read_text() == "done\n" + + +@pytest.mark.parametrize("command", [["no-such-app-xyz", "a.txt"], ["./missing.sh"]]) +@pytest.mark.asyncio +async def test_a_background_command_whose_program_isnt_there_fails_the_step(computer, command): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 127 in env 'desk': .*not found or not executable"): + await _step(command, background=True).execute(ctx) + + ctx = await _step(command, background=True, check=False).execute(ctx) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 + + +@pytest.mark.asyncio +async def test_a_timeout_above_what_the_computer_allows_is_refused_before_anything_runs(computer): + with pytest.raises(ValueError, match="above the 3600 s"): + await _step("true", timeout_s=4000).execute(_context(computer.deployed())) + assert computer.calls == [] + + +def _multi(computer: Computer, *names: str) -> DeployedGatewayEnv: + return DeployedGatewayEnv(env_id="multi", env_version=1, gateway_url="http://gw", sandbox_id="sb-gw", + environment_card={"name": "multi"}, + child_envs={name: dataclasses.asdict(computer.deployed(env_id=name)) for name in names}) + + +@pytest.mark.asyncio +async def test_a_multi_envs_only_computer_is_found_by_itself_and_one_of_several_by_name(computer): + ctx = await _step("echo one", env_id="multi").execute(_context(_multi(computer, "desk"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "one\n" + + ctx = await _step("echo two", env_id="multi", computer="b").execute(_context(_multi(computer, "a", "b"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "two\n" + + with pytest.raises(RuntimeError, match=r"has 2 computers .*: name one as computer"): + await _step("true", env_id="multi").execute(_context(_multi(computer, "a", "b"))) + with pytest.raises(RuntimeError, match=r"has no computer named 'c'"): + await _step("true", env_id="multi", computer="c").execute(_context(_multi(computer, "a", "b"))) + + +@pytest.mark.asyncio +async def test_an_env_with_no_computer_exec_is_refused(computer): + with pytest.raises(RuntimeError, match="has no computers serving"): + await _step("true").execute(_context(computer.deployed(card={"name": "desk"}))) + + +def _stored_multi(*computers: str, browser: str | None = None) -> MultiEnv: + def computer_env(name): + env = ComputerEnv.__new__(ComputerEnv) + env.environment_name = name + return env + + office = MultiEnv.__new__(MultiEnv) + office.id, office._computers = "office", SimpleNamespace(envs=[computer_env(name) for name in computers]) + office.website_browser = lambda: WebsiteBrowser(computer=computer_env(browser) if browser else None) + return office + + +@pytest.mark.parametrize("computers, browser, named, problem", [ + (("desk",), None, None, None), + ((), "chromium", None, None), + (("desk",), "chromium", "chromium", None), + (("desk",), "chromium", None, "has 2 computers: name one as computer, of ['desk', 'chromium']"), + (("desk",), None, "chromium", "has no computer named 'chromium' (it has ['desk'])"), + ((), None, None, "has no computers"), +], ids=["its-only-computer", "its-browser-computer-alone", "its-browser-computer-by-name", "two-unnamed", + "an-unknown-name", "none"]) +def test_preflight_finds_a_multi_envs_computers_and_its_browser_computer(monkeypatch, computers, browser, named, + problem): + monkeypatch.setattr("agent_env.task_step.task_steps.computer.computer_exec.stored_env", + lambda id: _stored_multi(*computers, browser=browser)) + monkeypatch.setattr(TaskStep, "preflight", lambda self: []) + + problems = _step("true", env_id="office", computer=named).preflight() + + assert problems == ([] if problem is None else [f"step 'exec': env 'office' {problem}"]) + + +@pytest.mark.parametrize("fields, problem", [ + ({"command": []}, "command must be a non-empty list"), + ({"command": " "}, "command must not be blank"), + ({"command": ["ls", 3]}, "command must be a non-empty list"), + ({"command": "true", "cwd": "/tmp"}, "relative to the session user's home"), + ({"command": "true", "env": {"A": 1}}, "env maps variable names to string values"), + ({"command": "true", "timeout_s": 0}, "timeout_s must be a number of seconds above 0"), + ({"command": "true", "background": "yes"}, "background must be true or false"), +]) +def test_bad_configs_are_named(fields, problem): + assert any(problem in p for p in _step(**fields).config_problems()) + + +def test_it_round_trips_through_the_registry(): + step = _step(["open", "x.xlsx"], computer="desk", cwd="Desktop", env={"A": "1"}, timeout_s=30, background=True, + check=False) + again = get_task_step_registry()["computer_exec"].from_dict(step.to_dict()) + assert again.to_dict() == step.to_dict() From 6bae0c01a60f9f41ecdcb1d449aedb337530ad03 Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 16:10:35 -0800 Subject: [PATCH 2/5] feat(verifiers): cua_verifier, a computer-use task's rubric graded on what the agent left in the computer Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 10 +- .../ui/src/components/steps-pipeline.tsx | 1 + .../ui/src/lib/verifier-classification.ts | 1 + src/agent_env/task/episode/rules.py | 1 + src/agent_env/task_step/registry.py | 2 + .../task_steps/computer/computer_exec.py | 4 +- .../task_steps/verifiers/cua_verifier.py | 324 ++++++++++++++++++ .../verifiers/judge_utils/cua_files.py | 162 +++++++++ .../task_steps/verifiers/rubrics_verifier.py | 21 +- .../computer/cua_verifier/envs/desk/env.toml | 3 + .../computer/cua_verifier/tasks/grade.json | 25 ++ tst/integration/computer/cua_verifier_test.py | 57 +++ tst/unit/task_step/test_field_scopes.py | 1 + .../task_step/verifiers/test_cua_files.py | 118 +++++++ .../task_step/verifiers/test_cua_verifier.py | 269 +++++++++++++++ uv.lock | 217 +++++++++++- 16 files changed, 1208 insertions(+), 8 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/verifiers/cua_verifier.py create mode 100644 src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py create mode 100644 tst/integration/computer/cua_verifier/envs/desk/env.toml create mode 100644 tst/integration/computer/cua_verifier/tasks/grade.json create mode 100644 tst/integration/computer/cua_verifier_test.py create mode 100644 tst/unit/task_step/verifiers/test_cua_files.py create mode 100644 tst/unit/task_step/verifiers/test_cua_verifier.py diff --git a/pyproject.toml b/pyproject.toml index a76476c31..f330b25d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,14 @@ dependencies = [ ] [project.optional-dependencies] +# cua_verifier renders a CUA task's spreadsheets, documents, slides and PDFs as text for its judge; without them it names +# each such file as not rendered. +cua = [ + "openpyxl>=3.1", + "python-docx>=1.1", + "python-pptx>=1.0", + "pypdf>=4", +] # agent_env.openenv: bundle tasks served as OpenEnv envs. No cap: OpenEnv ships a minor release a week. openenv = [ "openenv>=0.9", @@ -152,7 +160,7 @@ coreweave = [ "cwsandbox>=1.19.6,<2", # Reject exec streams that close without an exit status. ] dev = [ - "agentenv-framework[aws,boat,boxd,coreweave,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,trl,vercel,verl]", # exercise portable optional providers and stores in CI + "agentenv-framework[aws,boat,boxd,coreweave,cua,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,trl,vercel,verl]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs "av>=17", # the protocol's computer extra, whose tests the unit tier runs and which the recording tests decode with diff --git a/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx b/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx index 47cf08232..e0afa62e1 100644 --- a/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx +++ b/src/agent_env/explorer/ui/src/components/steps-pipeline.tsx @@ -26,6 +26,7 @@ const STEP_COLORS: Record = { collect_artifacts: '#14b8a6', rubrics_verifier: '#10b981', always_on_agent_verifier: '#10b981', + cua_verifier: '#10b981', env_outcome_verifier: '#10b981', run_code: '#84cc16', publish: '#6366f1', diff --git a/src/agent_env/explorer/ui/src/lib/verifier-classification.ts b/src/agent_env/explorer/ui/src/lib/verifier-classification.ts index 3336e2148..38a21fef8 100644 --- a/src/agent_env/explorer/ui/src/lib/verifier-classification.ts +++ b/src/agent_env/explorer/ui/src/lib/verifier-classification.ts @@ -16,6 +16,7 @@ export type VerifierKind = 'aggregate' | 'sandbox' | 'rubric' | 'other'; const RUBRICS_VERIFIER_STEP_TYPES: ReadonlySet = new Set([ 'rubrics_verifier', 'always_on_agent_verifier', + 'cua_verifier', ]); export function isRubricsVerifierStep( diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index dcf251f19..e824d14b1 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -143,6 +143,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "peer_agents": _peer_agents, "rubrics_verifier": _rubrics_verifier, "always_on_agent_verifier": _rubrics_verifier, + "cua_verifier": _rubrics_verifier, **dict.fromkeys(( "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_builtin_tools", "verify_a2a_core_protocol", "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 43bd7f79e..358421ef5 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -75,6 +75,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudge from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier + from agent_env.task_step.task_steps.verifiers.cua_verifier import CuaVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import VerifyMCPToolSchemaTaskStep @@ -152,6 +153,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, _get_type(UserSimJudge): UserSimJudge, _get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier, + _get_type(CuaVerifierTaskStep): CuaVerifierTaskStep, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, _get_type(VerifyMCPToolSchemaTaskStep): VerifyMCPToolSchemaTaskStep, diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py index e253b417f..ec4c1acbd 100644 --- a/src/agent_env/task_step/task_steps/computer/computer_exec.py +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -94,7 +94,7 @@ def preflight(self) -> list[str]: problems = super().preflight() if not isinstance(self.env_id, str) or not self.env_id or (env := stored_env(self.env_id)) is None: return problems - if problem := _computer_problem(env, self.computer): + if problem := computer_problem(env, self.computer): problems.append(f"step {self.id!r}: {problem}") return problems @@ -169,7 +169,7 @@ def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str] return computers[0] -def _computer_problem(env: object, computer: Optional[str]) -> Optional[str]: +def computer_problem(env: object, computer: Optional[str]) -> Optional[str]: if isinstance(env, ComputerEnv): return None if computer is None else f"env {env.id!r} is a computer env, so it takes no computer name" if not isinstance(env, MultiEnv): diff --git a/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py new file mode 100644 index 000000000..672307cc8 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py @@ -0,0 +1,324 @@ +"""cua_verifier: a computer-use task's rubric, graded by rubrics_verifier's judge on what the agent left in the computer. + +The rubric is a list of criteria (or ``{"criteria": [...]}``) with an ``id``, a ``title`` and a ``weight`` (0 when left +out), each a ``must-pass`` gate, a ``negative`` one (a failure mode, its weight a penalty) or a regular one, by its +``annotations``' ``criteria_type`` / ``criteria_category``; a ``visual`` category, and ``target_result_files`` / +``target_GT_files``, say how to grade it. What the judge reads is what the agent left in the computer: each of +``result_files``, read through the computer's driver (computer-exec/v1 ``run``, the control route the agent can't call) +and rendered as text (``judge_utils/cua_files.py``), a file that isn't there named as one the agent did not produce; the +golden files of ``expected_artifact_id``, for reference only; the output of each of ``state_commands`` run in the +computer, such as a mock website's state; and ``instruction``, the task as the agent was given it. With no +``prompt_id`` the judge grades those alone; with one it also sees that prompt's response. + +How the judge grades is ``guidance``, and how it treats the golden files ``golden_note``: the caller's own wording, or +short defaults of the framework's. The score: each criterion 1 or 0; the regular and passed must-pass criteria's weights +averaged, less each negative criterion's weight when its failure mode is present, clamped to [0, 1]; and 0 whenever a +must-pass criterion fails. A negative criterion is graded as rubrics_verifier grades one, 1 when its failure mode is +absent, and keeps its penalty as a negative weight. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from typing import Any, ClassVar, Optional + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.preflight import stored_env +from agent_env.task_step.task_steps.computer.computer_exec import computer_problem, find_computer +from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +logger = logging.getLogger(__name__) + +#: Where the step records what the judge was shown, by step id. +RECORDS_KEY = "cua_verifier" +#: How the judge grades when the step names no ``guidance`` of its own. +DEFAULT_GUIDANCE = """Give each criterion 1.0 if what is shown above meets it and 0.0 if it doesn't, and say \ +where you found the evidence. + +- A criterion of kind "must-pass" that scores 0.0 sets the whole rubric's score to 0. +- A criterion marked "negative": true names something that should not happen; score it as the instructions say. +- A criterion's "applies_to" lists the files it is about; one without it is about the outputs as a whole. +- A criterion marked "visual": true is about how a file looks, judged from its rendering. +- A golden file is a reference: what it holds counts only when the result holds it too.""" +#: How the judge treats the golden files when the step names no ``golden_note`` of its own. +DEFAULT_GOLDEN_NOTE = ("Reference files: use one only for a criterion that refers to it or to an expected value, not to " + "compare the results with as a whole.") +#: How long reading one file or running one state command in the computer may take. +READ_TIMEOUT_S = 120.0 +#: The most of one file's rendering, and of one command's output, the judge reads. +MAX_FILE_CHARS = 40_000 +#: The most of a file one read returns: its base64 well within the 1 MiB a driver returns of a command's output. +READ_CHUNK_BYTES = 512 * 1024 +#: The largest file the judge is shown; a larger one is named as too large. +MAX_READ_BYTES = 32 * 1024 * 1024 +#: The most of a failed read's stderr its error quotes. +ERROR_TAIL_CHARS = 300 +#: How far below 1 a criterion's score may be and still count as met, for scores that went through floating point. +SCORE_TOLERANCE = 1e-9 +#: Exit status of a read when the file isn't there. +_NOT_THERE = 3 +_SIZE = 'test -f "$1" || exit 3; wc -c < "$1"' +#: ``$2`` bytes of the file ``$1`` from byte ``$3`` (1-based), as base64. +_READ_CHUNK = 'tail -c +"$3" -- "$1" | head -c "$2" | base64' + + +def criteria_from_rubric(rubric: Any) -> list[dict]: + """A rubric (``{"criteria": [...]}`` or the list) as rubrics_verifier's criteria: each with its + ``id``, ``title``, ``kind``, ``weight`` (a negative one's as a penalty, below 0), and ``visual`` and ``applies_to`` + when its annotations say.""" + items = rubric.get("criteria") if isinstance(rubric, dict) else rubric + if not isinstance(items, list) or not items: + raise ValueError("rubric must be a non-empty list of criteria, or an object whose criteria is one") + criteria = [] + for index, item in enumerate(items): + if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]: + raise ValueError(f"rubric criterion {index} needs an id") + if not isinstance(item.get("title"), str) or not item["title"].strip(): + raise ValueError(f"rubric criterion {item['id']!r} needs a title, the text the judge grades") + weight = item.get("weight", 0) + if isinstance(weight, bool) or not isinstance(weight, (int, float)): + raise ValueError(f"rubric criterion {item['id']!r}: weight must be a number") + annotations = item.get("annotations") or {} + tags = " ".join(str(annotations.get(key) or "").lower() for key in ("criteria_type", "criteria_category")) + kind = "must-pass" if "must-pass" in tags else "negative" if "negative" in tags or weight < 0 else "regular" + criterion = {"id": item["id"], "title": item["title"], "kind": kind, + "weight": -abs(float(weight)) if kind == "negative" else float(weight)} + if kind == "negative": + criterion["negative"] = True + if str(annotations.get("criteria_category") or annotations.get("category") or "").lower() == "visual": + criterion["visual"] = True + applies_to = {name: list(files) for name, key in (("result_files", "target_result_files"), + ("golden_files", "target_GT_files")) + if (files := annotations.get(key))} + if applies_to: + criterion["applies_to"] = applies_to + criteria.append(criterion) + return criteria + + +def cua_rubric_score(rows: list[dict]) -> float: + """A rubric's score: 0 when a must-pass criterion fails; else the weighted average of the regular + and must-pass criteria, less the negative criteria's penalties, clamped to [0, 1]; with no positive weight, 1 less + the penalties when there are must-pass gates, else 0.""" + binary = [{**row, "score": 1.0 if _passed(row) else 0.0} for row in rows] + gates = [row for row in binary if row.get("kind") == "must-pass"] + if any(row["score"] < 1.0 for row in gates): + return 0.0 + if gates and not any(float(row.get("weight", 0)) > 0 for row in binary): + penalty = sum(abs(float(row["weight"])) * (1.0 - row["score"]) for row in binary + if float(row.get("weight", 0)) < 0) + return max(0.0, 1.0 - penalty) + return aggregate_score(binary, ScoreAggregator.WEIGHTED_AVERAGE) + + +def _passed(row: dict) -> bool: + score = row.get("score") + return isinstance(score, (int, float)) and not isinstance(score, bool) and score >= 1.0 - SCORE_TOLERANCE + + +class CuaVerifierTaskStep(RubricsVerifierTaskStep): + type: ClassVar[str] = "cua_verifier" + entity_refs = (*RubricsVerifierTaskStep.entity_refs, EntityRef.env("env_id"), + EntityRef.artifact("expected_artifact_id", version_field="expected_artifact_version")) + context_paths = RubricsVerifierTaskStep.context_paths + + def __init__(self, id: str, version: Optional[int], *, env_id: str, rubric: Any, result_files: list[str], + prompt_id: Optional[str] = None, computer: Optional[str] = None, + expected_artifact_id: Optional[str] = None, expected_artifact_version: Optional[int] = None, + state_commands: Optional[list[dict]] = None, instruction: Optional[str] = None, + guidance: Optional[str] = None, golden_note: Optional[str] = None, use_trajectory: bool = False, criteria: Any = None, + default_model: Optional[str] = None, default_model_api_base: Optional[str] = None, + effort: Optional[str] = None, max_thinking_tokens: Optional[int] = None, + judge_a2a_agent_id: Optional[str] = None, judge_sandbox_type: Optional[str] = None, + judge_batch_size: Optional[int] = None, judge_batch_concurrency: Optional[int] = None, + **fields: Any): + """``criteria`` is what ``rubric`` becomes (``criteria_from_rubric``), never read from a stored step; the rest is + rubrics_verifier's, but for ``score_aggregator``, which is this step's own (``cua_rubric_score``).""" + where = f"cua_verifier '{id}'" + for name, text in (("guidance", guidance), ("golden_note", golden_note)): + if text is not None and (not isinstance(text, str) or not text.strip()): + raise ValueError(f"{where}: {name} must be text, or left out for the default") + if prompt_id is None and use_trajectory: + raise ValueError(f"{where}: use_trajectory needs a prompt_id, whose run's trajectory the judge reads") + if not isinstance(result_files, list) or not all(isinstance(p, str) and p for p in result_files): + raise ValueError(f"{where}: result_files must be a list of paths in the computer") + for command in state_commands or []: + if not (isinstance(command, dict) and isinstance(command.get("name"), str) and command["name"] + and isinstance(command.get("command"), (str, list)) and command["command"]): + raise ValueError(f"{where}: each of state_commands is {{\"name\": ..., \"command\": [...] or \"...\"}}") + fields.pop("score_aggregator", None) + self.env_id = env_id + self.rubric = rubric + self.result_files = result_files + self.computer = computer + self.expected_artifact_id = expected_artifact_id + self.expected_artifact_version = expected_artifact_version + self.state_commands = state_commands or [] + self.instruction = instruction + self.guidance = guidance + self.golden_note = golden_note + self.grades_files_alone = prompt_id is None + super().__init__(id, version, criteria=criteria_from_rubric(rubric), prompt_id=prompt_id or f"{id}:files", + use_trajectory=use_trajectory, score_aggregator=ScoreAggregator.WEIGHTED_AVERAGE, + default_model=default_model, default_model_api_base=default_model_api_base, effort=effort, + max_thinking_tokens=max_thinking_tokens, judge_a2a_agent_id=judge_a2a_agent_id, + judge_sandbox_type=judge_sandbox_type, judge_batch_size=judge_batch_size, + judge_batch_concurrency=judge_batch_concurrency, **fields) + + def preflight(self) -> list[str]: + """rubrics_verifier's, and the env this step reads: a computer env, or a multi env with that computer.""" + problems = super().preflight() + if isinstance(self.env_id, str) and self.env_id and (env := stored_env(self.env_id)) is not None: + if problem := computer_problem(env, self.computer): + problems.append(f"step {self.id!r}: {problem}") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "prompt_id": None if self.grades_files_alone else self.prompt_id, "env_id": self.env_id, + "rubric": self.rubric, "result_files": self.result_files, "computer": self.computer, + "expected_artifact_id": self.expected_artifact_id, + "expected_artifact_version": self.expected_artifact_version, "state_commands": self.state_commands, + "instruction": self.instruction, "guidance": self.guidance, "golden_note": self.golden_note} + + @classmethod + def from_dict(cls, data: dict) -> CuaVerifierTaskStep: + fields = cls._rubrics_fields({**data, "criteria": None, "prompt_id": data.get("prompt_id")}) + fields.update(use_trajectory=data.get("use_trajectory", False), env_id=data["env_id"], rubric=data["rubric"], + result_files=data.get("result_files") or [], computer=data.get("computer"), + expected_artifact_id=data.get("expected_artifact_id"), + expected_artifact_version=data.get("expected_artifact_version"), + state_commands=data.get("state_commands"), instruction=data.get("instruction"), + guidance=data.get("guidance"), golden_note=data.get("golden_note")) + return cls(**fields) + + def score_rows(self, rows: list[dict]) -> float: + return cua_rubric_score(rows) + + def graded_response(self, context: TaskStepContext) -> Optional[PromptResponse]: + """The prompt's response; for a step that grades the files alone, a stand-in of its own, which never joins the + context, so no other step reads it as the agent's.""" + if not self.grades_files_alone: + return super().graded_response(context) + return PromptResponse(prompt_id=self.prompt_id, prompt_text=self.instruction or "", + response="(Not graded: grade the files and state below, which are what the agent " + "produced.)") + + def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict, + **kwargs: Any) -> None: + """rubrics_verifier's entry, with ``gate_passed``, whether every must-pass criterion was met, and when one + wasn't, ``gates_failed``, their ids: written in the same breath as the entry, since a step beside this one + stores what it sees of the context when it ends. (A run's stored context never gains an empty list, so there's + no ``gates_failed`` when none failed.)""" + super()._write_entries(context, verification_results, judge_call_fields, **kwargs) + entry = context.metadata["verifications"][self.verifier_id] + failed = [row.get("id") for row in entry["results"] if row.get("kind") == "must-pass" and not _passed(row)] + entry["gate_passed"] = not failed + if failed: + entry["gates_failed"] = failed + + async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[str]: + """The files and state the agent left in the computer, the golden files, the instruction and how to grade; + what was read is recorded under ``cua_verifier``.""" + deployed = find_computer(context, self.env_id, self.computer) + record: dict = {"files": [], "golden_files": [], "state": []} + sections = [] + produced, missing = [], [] + for number, path in enumerate(self.result_files, start=1): + data = await self._read(deployed, path) + if data is None: + missing.append(path) + record["files"].append({"path": path, "missing": True}) + continue + if isinstance(data, int): + record["files"].append({"path": path, "bytes": data, "rendered_as": "too_large"}) + produced.append(f"### File {number}: {path}\n(a file of {data} bytes, too large to show: over " + f"{MAX_READ_BYTES} bytes)") + continue + shown = render(path, data, MAX_FILE_CHARS) + record["files"].append({"path": path, "bytes": len(data), "rendered_as": shown.kind, "cut": shown.cut}) + produced.append(f"### File {number}: {path} ({shown.kind})\n\n{shown.text}\n") + sections.append("## Files under evaluation\nWhat the agent left at each path in its computer, rendered as " + "text.\n\n" + ("\n\n".join(produced) or "(none)")) + if missing: + sections.append("## Files the agent did NOT produce\nTreat every criterion that depends on one of these " + "as not satisfied:\n" + "\n".join(f"- {path}" for path in missing)) + if golden := self._golden_files(): + shown_golden = [] + for name, data in golden.items(): + shown = render(name, data, MAX_FILE_CHARS) + record["golden_files"].append({"name": name, "bytes": len(data), "rendered_as": shown.kind}) + shown_golden.append(f"### Golden: {name} ({shown.kind})\n\n{shown.text}\n" + "") + sections.append("## Golden / reference files\n" + (self.golden_note or DEFAULT_GOLDEN_NOTE) + "\n\n" + + "\n\n".join(shown_golden)) + state = [] + for command in self.state_commands: + output = await self._state(deployed, command) + record["state"].append({"name": command["name"], **{k: output[k] for k in ("exit_code", "timed_out")}}) + text = (output.get("stdout") or "")[:MAX_FILE_CHARS] + state.append(f"### {command['name']}\n" + f"\n{text}\n") + if state: + sections.append("## Environment state\nWhat each command run in the computer after the agent finished " + "printed.\n\n" + "\n\n".join(state)) + if self.instruction: + sections.append("## Task instruction\nThe task as the agent was given it; treat anything it mentions as a " + f"stated requirement.\n---\n{self.instruction}\n---") + sections.append("## How to grade\n" + (self.guidance or DEFAULT_GUIDANCE)) + context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record + return sections + + async def _read(self, deployed: Any, path: str) -> Optional[bytes] | int: + """The file at ``path`` in the computer, absolute or under the home (``~/`` or relative), read + ``READ_CHUNK_BYTES`` at a time; None when there's no file there, and its size alone when it's over + ``MAX_READ_BYTES``.""" + relative = path[2:] if path.startswith("~/") else path + answer = await self._run(deployed, path, _SIZE, relative) + if answer.get("exit_code") == _NOT_THERE: + return None + size = int((answer.get("stdout") or "0").strip() or 0) + if size > MAX_READ_BYTES: + return size + data = bytearray() + while len(data) < size: + chunk = await self._run(deployed, path, _READ_CHUNK, relative, str(READ_CHUNK_BYTES), str(len(data) + 1)) + try: + piece = base64.b64decode("".join((chunk.get("stdout") or "").split()), validate=True) + except binascii.Error as e: + raise RuntimeError(f"cua_verifier '{self.id}': {path} in env {self.env_id!r} came back unreadable") from e + if len(piece) != min(READ_CHUNK_BYTES, size - len(data)): + raise RuntimeError(f"cua_verifier '{self.id}': {path} in env {self.env_id!r} gave {len(piece)} bytes " + f"from byte {len(data)} of its {size}; it changed or couldn't be read while it was") + data += piece + return bytes(data) + + async def _run(self, deployed: Any, path: str, script: str, *args: str) -> dict: + """``script`` run by ``sh -c`` in the computer with ``args``; its answer, unless it failed otherwise than by + finding no file.""" + from agent_env.env.envs.computer.utils import inside + + answer = await inside.run(deployed, ["sh", "-c", script, "cua_verifier", *args], timeout_s=READ_TIMEOUT_S) + if answer.get("exit_code") in (0, _NOT_THERE) and not answer.get("timed_out") and not answer.get("truncated"): + return answer + why = ("came back cut by the computer's driver" if answer.get("truncated") + else f"couldn't be read: {(answer.get('stderr') or '').strip()[-ERROR_TAIL_CHARS:]}") + raise RuntimeError(f"cua_verifier '{self.id}': {path} in env {self.env_id!r} {why}") + + async def _state(self, deployed: Any, command: dict) -> dict: + from agent_env.env.envs.computer.utils import inside + + argv = ["sh", "-c", command["command"]] if isinstance(command["command"], str) else list(command["command"]) + return await inside.run(deployed, argv, timeout_s=READ_TIMEOUT_S) + + def _golden_files(self) -> dict[str, bytes]: + if self.expected_artifact_id is None: + return {} + from agent_env.artifact import Artifact + + artifact = Artifact.get(self.expected_artifact_id, self.expected_artifact_version) + return {name: file.load() for name, file in artifact.get_file_artifacts().items()} diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py new file mode 100644 index 000000000..59d012f44 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py @@ -0,0 +1,162 @@ +"""Files a CUA task's agent produced, and their golden counterparts, as text a rubric judge reads in its prompt +(``cua_verifier``): text formats as they are; spreadsheets, documents, slides and PDFs dumped by the libraries the +``cua`` extra installs (openpyxl, python-docx, python-pptx, pypdf), or named as not rendered without them; anything else +by its size and digest. Each rendering is cut at ``max_chars``.""" + +from __future__ import annotations + +import hashlib +import io +import posixpath +from dataclasses import dataclass +from typing import Callable + +#: Read as text, as they are. +TEXT_EXTENSIONS = frozenset({ + ".txt", ".csv", ".tsv", ".json", ".jsonl", ".md", ".html", ".htm", ".xml", ".log", ".py", ".js", ".ts", ".css", + ".yaml", ".yml", ".toml", ".ini", ".cfg", ".sql", ".ics", ".eml", ".srt", ".vtt", ".tex", ".sh", ".rtf"}) +IMAGE_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".svg", ".tif", ".tiff"}) +#: The most cells of a workbook a rendering lists. +MAX_CELLS = 6000 +#: The most cells of a workbook a rendering visits, empty ones included: a sparse sheet with a far corner cell spans +#: billions of them. +MAX_VISITED_CELLS = 200_000 +#: How much of a file without an extension is sampled to tell whether it's text. +TEXT_SAMPLE_BYTES = 4096 + + +@dataclass(frozen=True) +class Rendered: + """A file as the judge reads it: ``text``, and how it was read (``text``, ``xlsx``, ``docx``, ``pptx``, ``pdf``, + ``image``, ``binary``, or ``unrendered`` when the library its format needs isn't installed); ``cut`` when + ``text`` was cut at the limit.""" + + kind: str + text: str + cut: bool = False + + +def render(name: str, data: bytes, max_chars: int) -> Rendered: + """``data``, the bytes of the file ``name``, as the judge reads it.""" + extension = posixpath.splitext(name.lower())[1] + renderer = _RENDERERS.get(extension) + if renderer is not None: + kind, library, read = renderer + try: + text = read(data) + except ImportError: + return Rendered("unrendered", f"({extension} file of {len(data)} bytes, not rendered: {library} isn't " + "installed; install agentenv-framework[cua])") + except Exception as e: # a file the agent broke is evidence too + return Rendered("unrendered", f"({extension} file of {len(data)} bytes that {library} couldn't read: " + f"{type(e).__name__}: {e})") + return _cut(kind, text, max_chars) + if extension in TEXT_EXTENSIONS or (not extension and _looks_like_text(data)): + return _cut("text", data.decode("utf-8", errors="replace"), max_chars) + kind = "image" if extension in IMAGE_EXTENSIONS else "binary" + return Rendered(kind, f"({kind} file of {len(data)} bytes, sha256 {hashlib.sha256(data).hexdigest()}; its content " + "isn't shown)") + + +def _cut(kind: str, text: str, max_chars: int) -> Rendered: + if len(text) <= max_chars: + return Rendered(kind, text) + return Rendered(kind, text[:max_chars] + f"\n… (cut: {len(text) - max_chars} more characters)", cut=True) + + +def _looks_like_text(data: bytes) -> bool: + sample = data[:TEXT_SAMPLE_BYTES] + if b"\0" in sample: + return False + try: + sample.decode("utf-8") + except UnicodeDecodeError as e: + return e.start > len(sample) - 4 # a character cut by the sample's end + return True + + +def _xlsx(data: bytes) -> str: + """Each sheet's non-empty cells, ``A1 = value``, with ``[formula: …]`` for a formula cell: the formulas from the + workbook, the values as last calculated (none for a workbook no application has calculated). Both are streamed + (``read_only``), and the walk stops at ``MAX_CELLS`` listed or ``MAX_VISITED_CELLS`` visited.""" + import openpyxl + from openpyxl.utils import get_column_letter + + formulas = openpyxl.load_workbook(io.BytesIO(data), data_only=False, read_only=True) + values = openpyxl.load_workbook(io.BytesIO(data), data_only=True, read_only=True) + lines, listed, visited = [], 0, 0 + try: + for sheet in formulas.worksheets: + lines.append(f"## Sheet {sheet.title!r}") + rows = zip(sheet.iter_rows(values_only=True), values[sheet.title].iter_rows(values_only=True)) + for row_number, (formula_row, value_row) in enumerate(rows, start=1): + for column, (formula, value) in enumerate(zip(formula_row, value_row), start=1): + visited += 1 + if visited > MAX_VISITED_CELLS: + lines.append(f"… (stopped after visiting {MAX_VISITED_CELLS} cells)") + return "\n".join(lines) + if formula is None: + continue + if listed == MAX_CELLS: + lines.append(f"… (stopped at {MAX_CELLS} cells)") + return "\n".join(lines) + listed += 1 + cell = f"{get_column_letter(column)}{row_number}" + if isinstance(formula, str) and formula.startswith("="): + lines.append(f"{cell} = {value!r} [formula: {formula}]") + else: + lines.append(f"{cell} = {formula!r}") + finally: + formulas.close() + values.close() + return "\n".join(lines) + + +def _docx(data: bytes) -> str: + """Each paragraph with its style, then each table's rows, cells split by `` | ``.""" + import docx + + document = docx.Document(io.BytesIO(data)) + lines = [f"[{paragraph.style.name}] {paragraph.text}" for paragraph in document.paragraphs if paragraph.text.strip()] + for number, table in enumerate(document.tables, start=1): + lines.append(f"## Table {number}") + lines.extend(" | ".join(cell.text.strip() for cell in row.cells) for row in table.rows) + return "\n".join(lines) + + +def _pptx(data: bytes) -> str: + """Each slide's shapes' text and tables, a group's shapes within it included, and its notes.""" + import pptx + + def shape_lines(shapes) -> list[str]: + lines = [] + for shape in shapes: + if hasattr(shape, "shapes"): # a group: its shapes hold the text + lines.extend(shape_lines(shape.shapes)) + continue + if shape.has_text_frame and shape.text_frame.text.strip(): + lines.append(f"[{shape.name}] {shape.text_frame.text}") + if getattr(shape, "has_table", False) and shape.has_table: + lines.extend(" | ".join(cell.text.strip() for cell in row.cells) for row in shape.table.rows) + return lines + + lines = [] + for number, slide in enumerate(pptx.Presentation(io.BytesIO(data)).slides, start=1): + lines.append(f"## Slide {number}") + lines.extend(shape_lines(slide.shapes)) + if slide.has_notes_slide and slide.notes_slide.notes_text_frame.text.strip(): + lines.append(f"[notes] {slide.notes_slide.notes_text_frame.text}") + return "\n".join(lines) + + +def _pdf(data: bytes) -> str: + """Each page's text.""" + import pypdf + + pages = pypdf.PdfReader(io.BytesIO(data)).pages + return "\n".join(f"## Page {number}\n{page.extract_text() or ''}" for number, page in enumerate(pages, start=1)) + + +_RENDERERS: dict[str, tuple[str, str, Callable[[bytes], str]]] = { + ".xlsx": ("xlsx", "openpyxl", _xlsx), ".xlsm": ("xlsx", "openpyxl", _xlsx), + ".docx": ("docx", "python-docx", _docx), ".pptx": ("pptx", "python-pptx", _pptx), ".pdf": ("pdf", "pypdf", _pdf)} diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 71f7bf944..892fb2476 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -675,9 +675,15 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: + return cls(**cls._rubrics_fields(data)) + + @classmethod + def _rubrics_fields(cls, data: dict) -> dict: + """The constructor's arguments a stored step holds, as ``from_dict`` reads them: a subclass reads its own beside + them.""" raw_agg = data.get("score_aggregator") raw_tf = data.get("trajectory_filter") - return cls( + return dict( **cls._base_from_dict(data), criteria=data["criteria"], prompt_id=data["prompt_id"], @@ -726,6 +732,15 @@ async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptR with an id no criterion has. None here.""" return [] + def graded_response(self, context: TaskStepContext) -> Optional[PromptResponse]: + """The prompt response this step grades: an override point. Here, the run's response to ``prompt_id``.""" + return next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) + + def score_rows(self, rows: list[dict]) -> float: + """The rubric entry's score from its rows, the judge's and the unjudged ones, each carrying its criterion's + fields: an override point. Here, ``score_aggregator``'s.""" + return aggregate_score(rows, self.score_aggregator) + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: """The judge calls ``criteria`` are graded in, one batch per call: ``size`` at a time in order, or all in one call when ``size`` is None. An override point: a subclass may group them otherwise, putting every criterion @@ -764,7 +779,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context async def _grade(self, context: TaskStepContext) -> TaskStepContext: - prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) + prompt_response = self.graded_response(context) if prompt_response is None: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") overrides_for_step = self.step_param_overrides(context) @@ -1133,7 +1148,7 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di f"({type(verification_results).__name__})") mistake_rows = trajectory_mistakes_rows_from_evidence(verification_results) verification_results.extend(unjudged) # in place, so a ResultRows keeps its reasoning and checks - score = aggregate_score(verification_results, self.score_aggregator) + score = self.score_rows(verification_results) verifications = context.metadata.setdefault("verifications", {}) verification_entry = { **self._entry_format_fields(), diff --git a/tst/integration/computer/cua_verifier/envs/desk/env.toml b/tst/integration/computer/cua_verifier/envs/desk/env.toml new file mode 100644 index 000000000..e1dca97f8 --- /dev/null +++ b/tst/integration/computer/cua_verifier/envs/desk/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/cua_verifier/tasks/grade.json b/tst/integration/computer/cua_verifier/tasks/grade.json new file mode 100644 index 000000000..b0a2d0086 --- /dev/null +++ b/tst/integration/computer/cua_verifier/tasks/grade.json @@ -0,0 +1,25 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desk"}, + {"id": "work", "type": "computer_exec", "env_id": "desk", "depends_on": ["env"], + "command": "mkdir -p Desktop && printf 'item,amount\\nrent,1200\\nfood,300\\ntotal,1500\\n' > Desktop/budget.csv && echo 'Budget done: the total is 1500.' > notes.txt && echo '{\"sent\": [{\"to\": \"ana@example.com\", \"subject\": \"Budget\", \"body\": \"The total is 1500.\"}]}' > /tmp/mail_mock.json"}, + {"id": "grade", "type": "cua_verifier", "env_id": "desk", "depends_on": ["work"], "verifier_id": "cua", + "use_agent_judge": false, + "instruction": "Write ~/Desktop/budget.csv with rent 1200 and food 300 and their total, leave a note in ~/notes.txt, and email the total to ana@example.com.", + "result_files": ["~/Desktop/budget.csv", "~/notes.txt"], + "state_commands": [{"name": "mailbox", "command": ["cat", "/tmp/mail_mock.json"]}], + "rubric": {"criteria": [ + {"id": "csv", "title": "Provides a CSV file at ~/Desktop/budget.csv.", "weight": 5, + "annotations": {"criteria_type": "MUST-PASS"}}, + {"id": "total", "title": "budget.csv has a total row of 1500.", "weight": 50, + "annotations": {"criteria_type": "REGULAR", "target_result_files": ["budget.csv"]}}, + {"id": "mail", "title": "An email to ana@example.com states the total of 1500.", "weight": 30, + "annotations": {"criteria_type": "REGULAR"}}, + {"id": "wrong", "title": "Any file or email states a total other than 1500.", "weight": 20, + "annotations": {"criteria_type": "NEGATIVE"}}]}}, + {"id": "gate", "type": "cua_verifier", "env_id": "desk", "depends_on": ["work"], "verifier_id": "cua-gate", + "use_agent_judge": false, "result_files": ["~/Desktop/report.pdf", "~/notes.txt"], + "rubric": [ + {"id": "pdf", "title": "Provides a PDF report at ~/Desktop/report.pdf.", "weight": 5, + "annotations": {"criteria_type": "MUST-PASS"}}, + {"id": "note", "title": "notes.txt says the budget is done.", "weight": 10}]} +] diff --git a/tst/integration/computer/cua_verifier_test.py b/tst/integration/computer/cua_verifier_test.py new file mode 100644 index 000000000..a6d58ee9f --- /dev/null +++ b/tst/integration/computer/cua_verifier_test.py @@ -0,0 +1,57 @@ +"""cua_verifier end to end on a real computer, its judge faked at its call boundary: an ubuntu-24.04 computer with the +reference driver deployed, files and a mock mailbox's state left in it as an agent would leave them (by computer_exec), +and a CUA rubric graded on them. The files and state are read through the driver into the judge's prompt; the fake +judge meets each criterion unless it names a file the prompt lists as not produced, so every criterion met scores 1, +and a second rubric, whose must-pass file the computer hasn't, scores 0 with the gate named.""" + +import json +import re +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from agent_env.task_step.task_steps.verifiers.cua_verifier import DEFAULT_GUIDANCE, CuaVerifierTaskStep +from tst.integration.computer.conftest import run_context + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BUNDLE = Path(__file__).parent / "cua_verifier" +_CRITERION = re.compile(r'"id": "(c\d+)",\s*"title": "((?:[^"\\]|\\.)*)"') + + +def _fake_judge(prompts: list[str]): + """Meets each criterion of the prompt unless its title names a file the prompt lists as not produced.""" + + async def judge(self, prompt: str, *, model: str, context, **_) -> dict: + prompts.append(prompt) + missing = re.findall(r"^- (.+)$", prompt.split("## Files the agent did NOT produce", 1)[1].split("##")[0], + re.M) if "## Files the agent did NOT produce" in prompt else [] + rows = [{"id": cid, "score": 0.0 if any(Path(m).name in title for m in missing) else 1.0, + "justification": f"Checked {cid} against the files shown"} + for cid, title in dict(_CRITERION.findall(prompt)).items()] + return {"response": json.dumps({"results": rows})} + + return judge + + +def test_a_cua_rubric_is_graded_on_what_the_agent_left_in_the_computer(state, monkeypatch): + prompts: list[str] = [] + monkeypatch.setattr(CuaVerifierTaskStep, "_prompt_llm_judge", _fake_judge(prompts)) + + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "grade", "--no-emulate"]) + + assert ran.exit_code == 0, ran.output + graded = next(p for p in prompts if "budget.csv" in p and "mailbox" in p) + assert "total,1500" in graded and '"to": "ana@example.com"' in graded and DEFAULT_GUIDANCE in graded + metadata = run_context(ran.output)["metadata"] + files = metadata["cua_verifier"]["grade"]["files"] + assert [(f["path"], f["rendered_as"]) for f in files] == [("~/Desktop/budget.csv", "text"), + ("~/notes.txt", "text")], files + assert metadata["cua_verifier"]["grade"]["state"] == [{"name": "mailbox", "exit_code": 0, "timed_out": False}] + graded = metadata["verifications"]["cua"] + assert graded["score"] == 1.0 and graded["gate_passed"], graded["results"] + gated = metadata["verifications"]["cua-gate"] + assert metadata["cua_verifier"]["gate"]["files"][0] == {"path": "~/Desktop/report.pdf", "missing": True} + assert gated["score"] == 0.0 and not gated["gate_passed"] and gated["gates_failed"] == ["pdf"], gated["results"] diff --git a/tst/unit/task_step/test_field_scopes.py b/tst/unit/task_step/test_field_scopes.py index c33b866d7..a00d72fa1 100644 --- a/tst/unit/task_step/test_field_scopes.py +++ b/tst/unit/task_step/test_field_scopes.py @@ -19,6 +19,7 @@ "deploy_env": {"env_id": "crm-env"}, "deploy_user_sim_agent": {"agent_name": "hannah", "model": "m", "multi_day": False, "opening_prompt": "hi", "persona": {"display_name": "Hannah Reyes", "description": {"who_you_are": ["Hannah."]}}}, + "cua_verifier": {"env_id": "desk", "rubric": [{"id": "c1", "title": "says hi"}], "result_files": []}, "deploy_human_agent": {}, "deploy_sandbox": {"sandbox_name": "box", "sandbox_mode": "vm"}, "install_agent": {"sandbox_name": "box"}, diff --git a/tst/unit/task_step/verifiers/test_cua_files.py b/tst/unit/task_step/verifiers/test_cua_files.py new file mode 100644 index 000000000..853e62b62 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_cua_files.py @@ -0,0 +1,118 @@ +"""cua_files: each format a CUA task's agent leaves as the judge reads it: documents, slides, spreadsheets and PDFs +dumped as text, text as it is and cut at the limit, anything else by its size and digest, and a format whose library +isn't installed, or a file its library can't read, named as not rendered.""" + +from __future__ import annotations + +import builtins +import io + +import docx +import pptx +import pypdf +import pytest + +from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render + + +def _docx() -> bytes: + document = docx.Document() + document.add_heading("Dear Editor", level=1) + document.add_paragraph("The figures are corrected.") + table = document.add_table(rows=1, cols=2) + table.rows[0].cells[0].text, table.rows[0].cells[1].text = "page", "12" + out = io.BytesIO() + document.save(out) + return out.getvalue() + + +def _pptx() -> bytes: + deck = pptx.Presentation() + slide = deck.slides.add_slide(deck.slide_layouts[1]) + slide.shapes.title.text = "Q3 results" + slide.notes_slide.notes_text_frame.text = "Say the margin." + out = io.BytesIO() + deck.save(out) + return out.getvalue() + + +def _pdf() -> bytes: + writer = pypdf.PdfWriter() + writer.add_blank_page(width=200, height=200) + out = io.BytesIO() + writer.write(out) + return out.getvalue() + + +def test_documents_slides_and_pdfs_are_dumped_as_text(): + shown = render("letter.docx", _docx(), 10_000) + assert shown.kind == "docx" and "[Heading 1] Dear Editor" in shown.text and "page | 12" in shown.text + + shown = render("deck.pptx", _pptx(), 10_000) + assert shown.kind == "pptx" and "## Slide 1" in shown.text and "Q3 results" in shown.text + assert "[notes] Say the margin." in shown.text + + assert render("memo.pdf", _pdf(), 10_000) == render("memo.pdf", _pdf(), 10_000) + assert render("memo.pdf", _pdf(), 10_000).text.startswith("## Page 1") + + +def test_text_is_shown_as_it_is_and_cut_at_the_limit(): + assert render("data.csv", b"a,b\n1,2\n", 100).text == "a,b\n1,2\n" + cut = render("log", b"x" * 50, 10) + assert cut.kind == "text" and cut.cut and cut.text.startswith("x" * 10) and "40 more characters" in cut.text + + +def test_images_and_other_binaries_are_named_by_size_and_digest(): + shown = render("chart.png", b"\x89PNG\r\n\x1a\n\0\0", 100) + assert shown.kind == "image" and "image file of 10 bytes, sha256" in shown.text + assert render("blob", b"\0\1\2", 100).kind == "binary" + + +def test_a_format_whose_library_isnt_installed_or_cant_read_it_is_named(monkeypatch): + assert "that openpyxl couldn't read" in render("broken.xlsx", b"not a workbook", 100).text + + real_import = builtins.__import__ + + def without_docx(name, *args, **kwargs): + if name == "docx": + raise ImportError(name) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", without_docx) + shown = render("letter.docx", b"PK", 100) + assert shown.kind == "unrendered" and "python-docx isn't installed; install agentenv-framework[cua]" in shown.text + + +@pytest.mark.parametrize("name", ["a.xlsx", "a.docx", "a.pptx", "a.pdf"]) +def test_an_empty_file_of_a_rendered_format_is_named_not_raised(name): + assert render(name, b"", 100).kind == "unrendered" + + +def test_a_sparse_workbook_stops_after_the_cells_it_may_visit(): + import time + + import openpyxl + + book = openpyxl.Workbook() + book.active["A1"], book.active["XFD1048576"] = "start", "far corner" + out = io.BytesIO() + book.save(out) + started = time.monotonic() + + shown = render("sparse.xlsx", out.getvalue(), 10_000) + + assert time.monotonic() - started < 10 + assert "A1 = 'start'" in shown.text and "stopped after visiting 200000 cells" in shown.text + + +def test_grouped_slide_text_is_shown(): + from pptx.util import Inches + + deck = pptx.Presentation() + slide = deck.slides.add_slide(deck.slide_layouts[6]) + group = slide.shapes.add_group_shape() + group.shapes.add_textbox(Inches(1), Inches(1), Inches(2), Inches(1)).text_frame.text = "Revenue up 12%" + out = io.BytesIO() + deck.save(out) + + assert "Revenue up 12%" in render("deck.pptx", out.getvalue(), 10_000).text diff --git a/tst/unit/task_step/verifiers/test_cua_verifier.py b/tst/unit/task_step/verifiers/test_cua_verifier.py new file mode 100644 index 000000000..71ed46df0 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_cua_verifier.py @@ -0,0 +1,269 @@ +"""cua_verifier: a computer-use task's rubric as rubrics_verifier's criteria, its score (binary criteria, weights, +negative penalties, must-pass gates), and a run against a computer at its driver's boundary (``tst.unit.computer_home``) +with the judge faked: the files the agent left read through the driver and rendered into the judge's prompt, a missing +one named, the golden files and the state commands' output beside them, the caller's guidance or the default, and the +files graded alone when no prompt step names them.""" + +from __future__ import annotations + +import io +import json +import re + +import openpyxl +import pytest + +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.verifiers.cua_verifier import ( + DEFAULT_GOLDEN_NOTE, + DEFAULT_GUIDANCE, + RECORDS_KEY, + CuaVerifierTaskStep, + criteria_from_rubric, + cua_rubric_score, +) +from tst.unit.computer_home import Computer + +RUBRIC = {"criteria": [ + {"id": "sheet-exists", "title": "Provides report.xlsx.", "weight": 8, + "annotations": {"criteria_type": "MUST-PASS", "criteria_category": "format_gate"}}, + {"id": "total", "title": "Cell B3 of report.xlsx totals the amounts.", "weight": 50, + "annotations": {"criteria_type": "REGULAR", "target_result_files": ["report.xlsx"]}}, + {"id": "note", "title": "notes.txt says the report is done.", "weight": 25, + "annotations": {"criteria_category": "must have criteria"}}, + {"id": "leak", "title": "The email reveals the customer's card number.", "weight": 30, + "annotations": {"criteria_type": "NEGATIVE"}}, + {"id": "look", "title": "The table is legible.", "annotations": {"criteria_category": "visual"}}, +]} + + +def test_the_rubric_becomes_criteria_by_kind_with_weights_and_annotations(): + criteria = {c["id"]: c for c in criteria_from_rubric(RUBRIC)} + + assert criteria["sheet-exists"] == {"id": "sheet-exists", "title": "Provides report.xlsx.", "kind": "must-pass", + "weight": 8.0} + assert criteria["total"]["kind"] == "regular" and criteria["total"]["applies_to"] == {"result_files": ["report.xlsx"]} + assert criteria["note"]["kind"] == "regular" + assert criteria["leak"] == {"id": "leak", "title": "The email reveals the customer's card number.", + "kind": "negative", "weight": -30.0, "negative": True} + assert criteria["look"]["weight"] == 0.0 and criteria["look"]["visual"] is True + assert criteria_from_rubric([{"id": "x", "title": "t", "weight": -2}])[0]["kind"] == "negative" + + +@pytest.mark.parametrize("rubric, problem", [ + ({"criteria": []}, "non-empty list"), ([{"title": "t"}], "needs an id"), ([{"id": "x"}], "needs a title"), + ([{"id": "x", "title": "t", "weight": "5"}], "weight must be a number"), +]) +def test_a_rubric_it_cant_grade_is_refused(rubric, problem): + with pytest.raises(ValueError, match=problem): + criteria_from_rubric(rubric) + + +def _rows(**scores) -> list[dict]: + return [{**c, "score": scores.get(c["id"], 1.0)} for c in criteria_from_rubric(RUBRIC)] + + +@pytest.mark.parametrize("scores, expected", [ + ({}, 1.0), # all met, the failure mode absent + ({"note": 0.0}, 58 / 83), # a regular criterion missed + ({"leak": 0.0}, (83 - 30) / 83), # the failure mode present: its penalty + ({"note": 0.0, "total": 0.0, "leak": 0.0}, 0.0), # clamped at 0 + ({"sheet-exists": 0.0}, 0.0), # a must-pass gate failed + ({"total": 0.9}, 33 / 83), # binary: anything short of 1 is 0 +]) +def test_the_score_is_the_gated_weighted_average_less_the_penalties(scores, expected): + assert cua_rubric_score(_rows(**scores)) == pytest.approx(expected) + + +def test_gates_alone_score_one_less_the_penalties(): + rows = [{"id": "g", "kind": "must-pass", "weight": 0.0, "score": 1.0}, + {"id": "n", "kind": "negative", "weight": -0.25, "score": 0.0}] + assert cua_rubric_score(rows) == pytest.approx(0.75) + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + (tmp_path / "home").mkdir() + return Computer(tmp_path / "home").install(monkeypatch) + + +def _workbook() -> bytes: + book = openpyxl.Workbook() + sheet = book.active + sheet.title = "Report" + sheet["A1"], sheet["B1"], sheet["B2"], sheet["B3"] = "amount", 5, 7, "=SUM(B1:B2)" + out = io.BytesIO() + book.save(out) + return out.getvalue() + + +def _step(**fields) -> CuaVerifierTaskStep: + params = dict(id="grade", version=None, env_id="desk", rubric=RUBRIC, use_agent_judge=False, + verifier_id="cua", result_files=["/HOME/Desktop/report.xlsx", "~/notes.txt", "~/missing.pdf"], + instruction="Total the amounts in report.xlsx and leave a note.") + params.update(fields) + return CuaVerifierTaskStep(**params) + + +def _prompted_ids(prompt: str) -> list[str]: + return list(dict.fromkeys(re.findall(r'"id": "(c\d+)"', prompt))) + + +def _judge(step: CuaVerifierTaskStep, prompts: list[str], failing: tuple[str, ...] = ()) -> None: + """A judge that fails the criteria ``failing`` names and meets the rest, each with a justification of its own.""" + short = {f"c{i}": c["id"] for i, c in enumerate(step.criteria, start=1)} + + async def judge(prompt: str, *, model: str, context: TaskStepContext) -> dict: + prompts.append(prompt) + return {"response": json.dumps({"results": [ + {"id": cid, "score": 0.0 if short[cid] in failing else 1.0, + "justification": f"Read the file for {short[cid]}"} for cid in _prompted_ids(prompt)]}), + "trajectory_object_url": f"mem://judge/{len(prompts)}.json"} + + step._prompt_llm_judge = judge + + +@pytest.mark.asyncio +async def test_the_files_left_in_the_computer_are_rendered_into_the_judges_prompt_and_graded(computer, local_stores, + tmp_path): + (computer.home / "Desktop").mkdir() + (computer.home / "Desktop" / "report.xlsx").write_bytes(_workbook()) + (computer.home / "notes.txt").write_text("The report is done.\n") + (tmp_path / "golden").mkdir() + (tmp_path / "golden" / "report.csv").write_text("amount,12\n") + golden = FileArtifactUniverse.put_bundled("golden-files", files={"report.csv": tmp_path / "golden" / "report.csv"}) + step = _step(result_files=[f"{computer.home}/Desktop/report.xlsx", "~/notes.txt", "~/missing.pdf"], + expected_artifact_id=golden.id, expected_artifact_version=golden.version, + state_commands=[{"name": "mailbox", "command": "echo '{\"sent\": 1}'"}], + guidance="Grade it as the house style says.", golden_note="Golden files are examples.") + prompts: list[str] = [] + _judge(step, prompts, failing=("note",)) + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + await step.execute(ctx) + + [prompt] = prompts + assert "B3 = None [formula: =SUM(B1:B2)]" in prompt and "B1 = 5" in prompt + assert "The report is done." in prompt + assert "## Files the agent did NOT produce" in prompt and "- ~/missing.pdf" in prompt + assert "Golden files are examples." in prompt and "amount,12" in prompt + assert '\n{"sent": 1}' in prompt + assert "Total the amounts in report.xlsx" in prompt and "Grade it as the house style says." in prompt + assert DEFAULT_GUIDANCE not in prompt and DEFAULT_GOLDEN_NOTE not in prompt + assert len(_prompted_ids(prompt)) == 5 + assert ctx.prompt_responses == [] # the stand-in the files were graded with is gone + entry = ctx.metadata["verifications"]["cua"] + assert entry["score"] == pytest.approx(58 / 83) and entry["gate_passed"] and "gates_failed" not in entry + assert {row["id"]: row["kind"] for row in entry["results"]}["sheet-exists"] == "must-pass" + record = ctx.metadata[RECORDS_KEY]["grade"] + assert [(f["path"], f.get("rendered_as"), f.get("missing", False)) for f in record["files"]] == [ + (f"{computer.home}/Desktop/report.xlsx", "xlsx", False), ("~/notes.txt", "text", False), + ("~/missing.pdf", None, True)] + assert record["golden_files"] == [{"name": "report.csv", "bytes": 10, "rendered_as": "text"}] + assert record["state"] == [{"name": "mailbox", "exit_code": 0, "timed_out": False}] + + +@pytest.mark.asyncio +async def test_a_failed_must_pass_gate_zeroes_the_score_and_is_named(computer): + step = _step(result_files=["~/notes.txt"]) + _judge(step, [], failing=("sheet-exists",)) + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + await step.execute(ctx) + + entry = ctx.metadata["verifications"]["cua"] + assert entry["score"] == 0.0 and not entry["gate_passed"] and entry["gates_failed"] == ["sheet-exists"] + + +@pytest.mark.asyncio +async def test_a_file_is_read_in_chunks_and_one_too_large_is_named_not_read(computer, monkeypatch): + from agent_env.task_step.task_steps.verifiers import cua_verifier + + monkeypatch.setattr(cua_verifier, "READ_CHUNK_BYTES", 10) + monkeypatch.setattr(cua_verifier, "MAX_READ_BYTES", 100) + (computer.home / "notes.txt").write_text("0123456789abcdefghijklmnopqrstuvwxyz\n") + (computer.home / "huge.txt").write_text("x" * 101) + step = _step(result_files=["~/notes.txt", "~/huge.txt"]) + prompts: list[str] = [] + _judge(step, prompts) + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + await step.execute(ctx) + + assert "0123456789abcdefghijklmnopqrstuvwxyz\n" in prompts[0] and "a file of 101 bytes, too large" in prompts[0] + assert sum(1 for call in computer.calls if call[0] == "run" and "tail" in call[1]["argv"][2]) == 4 + assert ctx.metadata[RECORDS_KEY]["grade"]["files"][1] == {"path": "~/huge.txt", "bytes": 101, + "rendered_as": "too_large"} + + +@pytest.mark.asyncio +async def test_a_dash_prefixed_file_is_read_whole_and_a_short_read_is_refused(computer, monkeypatch): + from agent_env.task_step.task_steps.verifiers import cua_verifier + + (computer.home / "--report.csv").write_text("total,1500\n") + step = _step(result_files=["~/--report.csv"]) + prompts: list[str] = [] + _judge(step, prompts) + + await step.execute(TaskStepContext(deployed_envs=[computer.deployed()])) + + assert "total,1500" in prompts[0] + + monkeypatch.setattr(cua_verifier, "_SIZE", 'test -f "$1" || exit 3; echo 999') # a file that shrank since + with pytest.raises(RuntimeError, match=r"gave 11 bytes from byte 0 of its 999"): + await _step(result_files=["~/--report.csv"]).execute(TaskStepContext(deployed_envs=[computer.deployed()])) + + +@pytest.mark.asyncio +async def test_the_files_only_stand_in_never_joins_the_context(computer): + step = _step(result_files=[]) + seen: list[list] = [] + + async def judge(prompt: str, *, model: str, context: TaskStepContext) -> dict: + seen.append(list(context.prompt_responses)) + return {"response": json.dumps({"results": [{"id": cid, "score": 1.0, "justification": f"Checked {cid}"} + for cid in _prompted_ids(prompt)]})} + + step._prompt_llm_judge = judge + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + await step.execute(ctx) + + assert seen == [[]] and ctx.prompt_responses == [] + + +@pytest.mark.asyncio +async def test_with_a_prompt_id_the_judge_also_sees_that_prompts_response(computer): + step = _step(prompt_id="p1", result_files=[]) + prompts: list[str] = [] + _judge(step, prompts) + ctx = TaskStepContext(deployed_envs=[computer.deployed()], + prompt_responses=[PromptResponse(prompt_id="p1", prompt_text="do it", response="Done: B3")]) + + await step.execute(ctx) + + assert "Done: B3" in prompts[0] and len(ctx.prompt_responses) == 1 + assert DEFAULT_GUIDANCE in prompts[0] + + +def test_it_round_trips_through_the_registry_keeping_a_files_only_grade(): + step = _step(computer="desk", guidance="Be strict.", state_commands=[{"name": "s", "command": ["cat", "/tmp/s.json"]}]) + data = step.to_dict() + again = get_task_step_registry()["cua_verifier"].from_dict(data) + + assert data["prompt_id"] is None and again.grades_files_alone and again.to_dict() == data + assert again.criteria == step.criteria and again.score_rows is not None + + +@pytest.mark.parametrize("fields, problem", [ + ({"guidance": " "}, "guidance must be text"), + ({"golden_note": 3}, "golden_note must be text"), + ({"use_trajectory": True}, "use_trajectory needs a prompt_id"), + ({"result_files": "a.txt"}, "result_files must be a list"), + ({"state_commands": [{"name": "s"}]}, "each of state_commands"), +]) +def test_bad_configs_are_refused(fields, problem): + with pytest.raises(ValueError, match=problem): + _step(**fields) diff --git a/uv.lock b/uv.lock index 8635f3a2d..3bd395e10 100644 --- a/uv.lock +++ b/uv.lock @@ -91,6 +91,12 @@ boxd = [ coreweave = [ { name = "cwsandbox" }, ] +cua = [ + { name = "openpyxl" }, + { name = "pypdf" }, + { name = "python-docx" }, + { name = "python-pptx" }, +] daytona = [ { name = "daytona" }, ] @@ -117,16 +123,20 @@ dev = [ { name = "hypothesis" }, { name = "moto" }, { name = "openenv" }, + { name = "openpyxl" }, { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pyarrow", version = "25.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13.14'" }, { name = "pyarrow", version = "26.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13.14'" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-dependency" }, { name = "pytest-socket" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "python-docx" }, + { name = "python-pptx" }, { name = "regex" }, { name = "requests" }, { name = "ruff" }, @@ -203,7 +213,7 @@ verl = [ [package.metadata] requires-dist = [ { name = "a2a-sdk", specifier = "==0.3.26" }, - { name = "agentenv-framework", extras = ["aws", "boat", "boxd", "coreweave", "daytona", "gcp", "harness", "openenv", "sail", "skyrl", "tensorlake", "tenki", "trl", "vercel", "verl"], marker = "extra == 'dev'" }, + { name = "agentenv-framework", extras = ["aws", "boat", "boxd", "coreweave", "cua", "daytona", "gcp", "harness", "openenv", "sail", "skyrl", "tensorlake", "tenki", "trl", "vercel", "verl"], marker = "extra == 'dev'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'harness'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'miles'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'nemo-gym'" }, @@ -247,6 +257,7 @@ requires-dist = [ { name = "omegaconf", marker = "python_full_version >= '3.13.14' and extra == 'nemo-gym'", specifier = ">=2.3" }, { name = "openenv", marker = "extra == 'harness'", specifier = ">=0.10" }, { name = "openenv", marker = "extra == 'openenv'", specifier = ">=0.9" }, + { name = "openpyxl", marker = "extra == 'cua'", specifier = ">=3.1" }, { name = "packaging", specifier = ">=22" }, { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.0" }, @@ -254,13 +265,16 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'verl'", specifier = ">=19" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pymongo", specifier = ">=4.7" }, + { name = "pypdf", marker = "extra == 'cua'", specifier = ">=4" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "pytest-dependency", marker = "extra == 'dev'", specifier = ">=0.6.0" }, { name = "pytest-socket", marker = "extra == 'dev'", specifier = ">=0.7.0" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.0.0" }, + { name = "python-docx", marker = "extra == 'cua'", specifier = ">=1.1" }, { name = "python-multipart", specifier = ">=0.0.13" }, + { name = "python-pptx", marker = "extra == 'cua'", specifier = ">=1.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, { name = "requests", marker = "extra == 'gcp'", specifier = ">=2.31" }, @@ -281,7 +295,7 @@ requires-dist = [ { name = "websockets", marker = "extra == 'dev'", specifier = ">=15" }, { name = "zstandard", marker = "extra == 'dev'", specifier = ">=0.23" }, ] -provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"] +provides-extras = ["cua", "openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"] [[package]] name = "agentenv-framework-protocol" @@ -1872,6 +1886,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -3521,6 +3544,138 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/41/8c5076437a5a56cf8e788eafe9ca36b649375bb14811d2401e6520acba86/litellm-1.106.6-cp310-abi3-win_amd64.whl", hash = "sha256:42ce43ffb0161c9a844e8b0cd02a8796d0423b11d84fe304bff925378ca55085", size = 37383305, upload-time = "2026-10-28T22:49:13.624Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/f1/95133bde7af7afb1f5ba6090b674d826b7a518318bba54bbbb633b27865a/lxml-6.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c66f858b82497173f73366795fc6ee8171620e75a338506d6b2e7bc16f5fca11", size = 8563141, upload-time = "2026-09-02T14:46:42.334Z" }, + { url = "https://files.pythonhosted.org/packages/80/54/5a79ee2181ac773ee13e48205411845feec69e1c3d097e985c1343171712/lxml-6.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:032a0a97eed428bd143c75a11118238546424ceb2fa311cca5f073aa44658dc4", size = 4613690, upload-time = "2026-09-02T14:46:45.253Z" }, + { url = "https://files.pythonhosted.org/packages/ab/29/8c24672f56807f119312f073f24204368574bd16b384ede861b5104b3a2b/lxml-6.1.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a579dfb9c835f8ab47f4b8ed33440cbc75b806b73297208e6ec2a33e903740b", size = 4935630, upload-time = "2026-09-02T14:46:48.071Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/ce2436d854c848c19fc9287143991f3fc76b8b4e9a0dbba8452e51dff264/lxml-6.1.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:49fbc2682a9306135b7ec49e93f97f9c26689b9b7f96ed2742d8d6497e994d13", size = 5079033, upload-time = "2026-09-02T14:46:50.483Z" }, + { url = "https://files.pythonhosted.org/packages/91/ec/b66f66f6499ad800265d57540b51e6632e3232d3526f42f2f8fd4b14e0ea/lxml-6.1.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea2c01cdb16dc12156e455007c406dfaaece0c89aa4ba0e3b47586779f951d41", size = 5012298, upload-time = "2026-09-02T14:46:52.603Z" }, + { url = "https://files.pythonhosted.org/packages/94/2a/25d128872f4d51753542bfc3feb482c2ea7c8a2d6d81a0bc5c6a00779ed4/lxml-6.1.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:527195c188d7d0af748cd48d220ab8cdc5cb99be3d49ac4d9be7324d8abf9bc0", size = 5211431, upload-time = "2026-09-02T14:46:54.722Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/0a41bbef074a556110f84fafb6d8c2998293c7d3bfbe1ce74515bc65393b/lxml-6.1.3-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:20384c2bbcbf87180c8c61eb60869699c1ec0cd09b62cfd13804022d860b0867", size = 5343417, upload-time = "2026-09-02T14:46:57.46Z" }, + { url = "https://files.pythonhosted.org/packages/7b/cd/16116c3f91791aeeeab1cbe6e7eb6e646f127be7b0158b262eb526a21a0c/lxml-6.1.3-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:424aa5657141d306ba9ad1baab4b2c0a0719040075ee6c66aee9bb2dea2b5054", size = 4673219, upload-time = "2026-09-02T14:46:59.604Z" }, + { url = "https://files.pythonhosted.org/packages/dd/bb/4dff849f443ef70221676aec938bc41e8bae6430aa2ca13b041319e14b98/lxml-6.1.3-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4736e6c87e603146d8949d8501da621ad20c31015060d3fcf95ace2859f3e3e6", size = 5281246, upload-time = "2026-09-02T14:47:02.375Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ac/4aa7dd059420bfd35278c7fe819e9d319ee36a0453b7bbde1907a7832d91/lxml-6.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6374e9e382e5a98c9c5e66d41b357b470da1c54bce30f17f9dc4bcc58436cc1c", size = 5055451, upload-time = "2026-09-02T14:47:05.883Z" }, + { url = "https://files.pythonhosted.org/packages/de/44/20d90cf6f4234de9cd9eeb4f519419885fdb087fa80d073c7b57be342021/lxml-6.1.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:22eec57e26c418cde02c051ce9914a365e52a7f135a565c6f0480242aeebab48", size = 4722694, upload-time = "2026-09-02T14:47:08.461Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0e/6bee12325e53dd6613fe1e107def07583b6182ade03e94bfef8976622e44/lxml-6.1.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8753b8d51dbc86fd335ee31fcf7f3658e9f5c016d4edfb23f76ad295f4b8c9d0", size = 5269179, upload-time = "2026-09-02T14:47:10.647Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5d/54d269ce5cd0787c0424d9cef449ee794d4097725d13dd2acd6181c44e9c/lxml-6.1.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:207dfc3d47cf0e575e643bbc140dacc8863b39abaa1e5307cd64c7f2365b8a12", size = 5235559, upload-time = "2026-09-02T14:47:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f7/5a3095f187f1bec293591616a1677781acc265c5b313c009f8a19c471a09/lxml-6.1.3-cp311-cp311-win32.whl", hash = "sha256:18293f8a8d8b6a8e71ef37706b659e3846a4261232158167b1ddf35f6994f633", size = 3600377, upload-time = "2026-09-02T14:47:15.957Z" }, + { url = "https://files.pythonhosted.org/packages/45/5a/15531a0d307c96282fe8b639b3d74e8bd783e4ab4cb2b0781146ac4161b8/lxml-6.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:7ae4949f212a53b007dbc355884fda122545c5764a54256c9217e419a62a6559", size = 4032700, upload-time = "2026-09-02T14:47:18.566Z" }, + { url = "https://files.pythonhosted.org/packages/12/f9/8de76314955545ceaaa7c0305017b8aaa217905dee59c62c0e2c1e44a68f/lxml-6.1.3-cp311-cp311-win_arm64.whl", hash = "sha256:2123e5aa075ac20d23c7af489255efd129cbfe190dbe88fd42598cc9df3199b6", size = 3674431, upload-time = "2026-09-02T14:47:22.186Z" }, + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/3ef45db776baea068044c799bbba68f3ca00a440c0e930a17c572f3d9639/lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd", size = 8590357, upload-time = "2026-09-02T14:48:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a5/eee2fc77eee5ea68e4a4334b1def1781a3beaeefd3d98e81b4a38dc447b7/lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1", size = 4632616, upload-time = "2026-09-02T14:48:20.745Z" }, + { url = "https://files.pythonhosted.org/packages/35/42/df27b56848acd29d8a720acc28977911aab36f2a09df4208d5502e887415/lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d", size = 4936186, upload-time = "2026-09-02T14:48:22.94Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8d/8a7b91df0b54d09d25f5f44885d6b3e0a6d6643a8c070191580318d20c42/lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed", size = 5093324, upload-time = "2026-09-02T14:48:25.132Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/8f340ddcd43790332fb0de8a26628d571a492da3300cd191821698407c96/lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2", size = 4998850, upload-time = "2026-09-02T14:48:27.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c1/9c5bb572f1f09ec9e4322bd4a4e9f4ad48347fc56ef94cf4df58a5279dc8/lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8", size = 5626813, upload-time = "2026-09-02T14:48:29.61Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7d/8bf1fd8bae8247743968bb76d027a1ac5bd2c4b44495fba6a71b30d10706/lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e", size = 5232385, upload-time = "2026-09-02T14:48:31.969Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/6cef69ed81cb7df0d03b0dd09d08e6e2cf5061a743ff6f42f0b741548e9b/lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245", size = 5347088, upload-time = "2026-09-02T14:48:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e1/8e5fd8ddc8c7d685badb0f2db149e3c9da84eefc2827c01c658df2c4e3cb/lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0", size = 4707227, upload-time = "2026-09-02T14:48:36.62Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7e/00041382a11be40a88bf405ebff11c8efabd3de79f2691e1638b1c47a8a0/lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e", size = 5240208, upload-time = "2026-09-02T14:48:38.893Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fe/316538b5cff0936fa63d45d421c655730fcbb5a28dcac728c175083002bc/lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2", size = 5050271, upload-time = "2026-09-02T14:48:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/91/455bcccb3ac725373007344d351151810cd19762d1673b64b811f4359a42/lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310", size = 4780433, upload-time = "2026-09-02T14:48:43.779Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f6/580440e2f52cf00bba5c5e1080bfa88cdfcde73be71a11d95170ddbb663f/lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748", size = 5645928, upload-time = "2026-09-02T14:48:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/f6/dc/d123c1f244306543d545f62443f794959e4f1ea709fe100f8740d514e74a/lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d", size = 5231184, upload-time = "2026-09-02T14:48:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/c3/3c/fe55b2bd5c6113c906511cd88f6a470195c5fbff1124f19970ab706c3477/lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc", size = 5255814, upload-time = "2026-09-02T14:48:50.948Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a7/485df55acf55dc35e4ca89d2f48f03889e5a3241826b18b85102b32ce9d8/lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87", size = 3602214, upload-time = "2026-09-02T14:48:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/e46a7702bd95e9043291f7c3539b6184cba66f96cea9936f20939b284eeb/lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477", size = 4004091, upload-time = "2026-09-02T14:48:55.699Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/154c78e20479a43916e63f19cb720d83f44f024b03228be44c92d9a97b24/lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1", size = 3665468, upload-time = "2026-09-02T14:48:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/4614fcd65496054cfb7172662f3576a59200278739506433b8c241ea422a/lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0", size = 8609378, upload-time = "2026-09-02T14:50:31.772Z" }, + { url = "https://files.pythonhosted.org/packages/f2/51/2cdce3c65fa99a6195dd8fbd512d33407c1000ad99f63e0a285b63d7a8eb/lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9", size = 4640022, upload-time = "2026-09-02T14:50:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/0b30084e9eb1c546a4be3d9c56df70058d116b1a320400a59b0f7da87bf0/lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79", size = 5037928, upload-time = "2026-09-02T14:50:37.007Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/5c37275a3e361f6138dc06db748ea565c1fe8a5f4ee5e2ddd80047c81a89/lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015", size = 5661932, upload-time = "2026-09-02T14:50:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/b71ffb289b15e2642e2a3cf6d468c44da39ea119061a99e5b05e3d10f217/lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a", size = 5249209, upload-time = "2026-09-02T14:50:42.141Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/9910da149a23932f9301652e57661cd9e42b0df18f12be21159b7255f92b/lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed", size = 4704543, upload-time = "2026-09-02T14:50:44.634Z" }, + { url = "https://files.pythonhosted.org/packages/76/07/9290329cd188c62e22021f79df04ee0cc33d9a93b0d38bd65ccd452ad9d0/lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156", size = 5261298, upload-time = "2026-09-02T14:50:47.301Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0c/aba78bd3401cd99b73a0aed8e2b9b43e14be94fab3603d4bbc8a62365f2a/lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d", size = 5090453, upload-time = "2026-09-02T14:50:49.952Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa4426c3355aa0216cbeb3911495b5f65a26e0df85859a89928fe28f0396/lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0", size = 4744709, upload-time = "2026-09-02T14:50:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/be/2b/224fe7918658ab7c532ac2412f3c1eb28f71e6364fb07566262d0cc6a7b6/lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69", size = 5685802, upload-time = "2026-09-02T14:50:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/21/44/7d480819b9adcae5f84dd8ac529132c6b7a578544398225cd20321adcd91/lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0", size = 5249019, upload-time = "2026-09-02T14:50:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/72/83/385a267ea1b6b283f2249dd827ef360a295e9db14e13ef4665a120c60d64/lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4", size = 5271886, upload-time = "2026-09-02T14:51:01.667Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/f967b0eb172ae876855a402d6d9b11fa86e3e0c89ca9bbfeadf7ffbfa719/lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4", size = 3662894, upload-time = "2026-09-02T14:51:45.173Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/d8a8c4160a29e663109ad520bac2deb37fcd014756d024561e8bc3e611ec/lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad", size = 4074626, upload-time = "2026-09-02T14:51:47.77Z" }, + { url = "https://files.pythonhosted.org/packages/25/20/3e1395d34d19f9254625d0b567b81cf70d37d3417be074f4d63b94a2be3c/lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758", size = 3749495, upload-time = "2026-09-02T14:51:50.663Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c6/7465ffd9c43883526a382df6fa4846c9d8d419214f7effbf65270e795471/lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe", size = 8857677, upload-time = "2026-09-02T14:51:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/ed/eb/1f3a917e299df43c8162c3e6f64fc2cea3bcf277910f35bff5b8e5d39901/lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741", size = 4754522, upload-time = "2026-09-02T14:51:08.137Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f9/f81b4bdb6efb7a596be29603d8758154d00a5f545db9f3cef9d9041c8f64/lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300", size = 5033744, upload-time = "2026-09-02T14:51:10.633Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/26d9bfaacb319c86e0eca8a1a0bf1130d36a7afbd318883e23caea63763d/lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0", size = 5615269, upload-time = "2026-09-02T14:51:13.357Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/73675f3f4141350ed65d6fec533b107d4e802c5caa340cf111771edd86e0/lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd", size = 5236280, upload-time = "2026-09-02T14:51:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/ed260767e7977de463a0f91f3f4fffcab85c0a2a024a21ffe1fa442c2c79/lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e", size = 4650718, upload-time = "2026-09-02T14:51:19.102Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/e9839d03b1e767f2725cf7d7d81b80d5f3f9fdc10ad8827e2479311b046e/lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2", size = 5243376, upload-time = "2026-09-02T14:51:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/4606e347e2788c301f677004aa83e28d24da9fe663a24380122af57be6fc/lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a", size = 5092340, upload-time = "2026-09-02T14:51:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/ea/99/3314a8661cdf30f493c55a87db283961dfaae08451976a2ca418958e1804/lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011", size = 4758768, upload-time = "2026-09-02T14:51:26.813Z" }, + { url = "https://files.pythonhosted.org/packages/30/58/3bdc577f78ea8b7d72d39a84506f7001d5b28728f43e5b84891e3b7d9a4a/lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5", size = 5649546, upload-time = "2026-09-02T14:51:29.453Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e4/652633de1a2395949ebb7a8fc7d089aba12a2b45f0fefbc9d29e3e3ab3cf/lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a", size = 5234874, upload-time = "2026-09-02T14:51:32.262Z" }, + { url = "https://files.pythonhosted.org/packages/65/a6/c4581d171de30449304b4859bbd3607e9b40da13c0f88b68e6097c8d785e/lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887", size = 5260043, upload-time = "2026-09-02T14:51:34.841Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/ed6ee6186a89e69ca4ea9658b2a278f46a5efe8b5d4db56c7197f18653fe/lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e", size = 3901093, upload-time = "2026-09-02T14:51:37.234Z" }, + { url = "https://files.pythonhosted.org/packages/67/9d/11d10257a4a048d04195d638bb61f0246ce2448eb05f682bcbab25a257a8/lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6", size = 4395446, upload-time = "2026-09-02T14:51:39.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c1/2433176de263cc3f51fd2c303f993d5bb7f1da3139a0f7d168116c0bfa7a/lxml-6.1.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d2765c18ce303149ee804b1f3dad11232726dd0a702d73a15cf19179ac8cc962", size = 3942969, upload-time = "2026-09-02T14:46:36.55Z" }, + { url = "https://files.pythonhosted.org/packages/7c/71/de7759096f480180fd9e43ff7c017860e2d2a9a43741ab093cbdf1820f07/lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d5a748d12dd9b535e0a130f60dae9ddf0adafbabe61e7864f55c7436c84547a", size = 4213008, upload-time = "2026-09-02T14:46:38.784Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/c2d09af47a34fa6c0c27473083812b449a411680bd04bbe609cde291ddc8/lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:41096ec0740a58dad03d3ae0c7486d306d20becefb13ceb1649835ab3eb64167", size = 4322012, upload-time = "2026-09-02T14:46:41.031Z" }, + { url = "https://files.pythonhosted.org/packages/68/f3/bf56fee0403ebd995be8e78ec9aca566016487d1b3cbf755ebea8ccffbdb/lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:415e3a115c0d510e329020012834d1c0aa1c581ee53a218603e38abbc1dea70a", size = 4257402, upload-time = "2026-09-02T14:46:43.134Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1d/6da9cc086a20d9dd6bcbf7c5d9575f0331cca9a05e67dab02d15e828170b/lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20428910dae17a1a93152a3ff2c0441d2f4932992c0797d65651dd0561f1792f", size = 4410889, upload-time = "2026-09-02T14:46:46.975Z" }, + { url = "https://files.pythonhosted.org/packages/03/5c/91fe48856f9f8089be3096fa4dbe4b3fb5526f3bf3e852ea9497f399cb9f/lxml-6.1.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bc8dd3d9c93e70c3df974a201ac2958b6d77b465d813c51d1f15fa8e645763ae", size = 3511258, upload-time = "2026-09-02T14:46:49.046Z" }, +] + [[package]] name = "mako" version = "1.4.3" @@ -4501,6 +4656,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/8e/46153b0f6f2127a42508cf6734650531d25e889a4cbd1916f35a9afb85e7/openenv-0.10.0-py3-none-any.whl", hash = "sha256:5e0a1019253802551de7890f7638c977c4a36540a4b19f01e5a6fcdb5f5fd135", size = 731246, upload-time = "2026-10-22T06:11:16.836Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -5764,6 +5931,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, ] +[[package]] +name = "pypdf" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/d3/367aa9baefaededd3321cde46bbcb9f008bb51cef3a6b745608352a99534/pypdf-7.1.0.tar.gz", hash = "sha256:ca3fa902369e98a6d2cb7b56396de902b72ef9bb3c1e8706c1bfc7d8bd0cf0d7", size = 7215190, upload-time = "2026-11-13T12:23:18.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/4c/d19f91b743c8579b67e3d8b001e095745dec5bb13a9b4b9eadc235a85d9e/pypdf-7.1.0-py3-none-any.whl", hash = "sha256:47cca6b24bdcf4c4d65b5f7b8b972535c61710a061261e737688777cfd9a7bfb", size = 418698, upload-time = "2026-11-13T12:23:16.777Z" }, +] + [[package]] name = "pyperclip" version = "1.11.0" @@ -5927,6 +6103,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/e4/1c7c388e874222c28afb92fe3ebfeebe55555276523ba7aab0a82a10b2b5/python_discovery-1.6.7-py3-none-any.whl", hash = "sha256:5422f67e5f533bb02d7b73de1c422ed74037536fd869e05156c07565926ec68e", size = 39156, upload-time = "2026-10-28T06:24:20.462Z" }, ] +[[package]] +name = "python-docx" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.3" @@ -5957,6 +6146,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + [[package]] name = "python-socketio" version = "5.17.0" @@ -7867,6 +8071,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/fb/31bdbf3737222d92ef7d2db4d50089b2253dbbb63ceb6d20a4447f6b7ed0/xlsxwriter-3.3.1.tar.gz", hash = "sha256:0867761912cceac76f97ffa76eeda18c623ae9217f601be8512f55b1af4bdb6e", size = 223040, upload-time = "2026-10-28T22:52:46.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/eb/1e6af9959a567901a7e7c7cef2e75883e487df223ed67f9a8977ff863bd8/xlsxwriter-3.3.1-py3-none-any.whl", hash = "sha256:8e2c89e59d733fe59e112317218c64fbbc276d90a8c144ebd6f7fa15e7ab184a", size = 180583, upload-time = "2026-10-28T22:52:45.048Z" }, +] + [[package]] name = "xmltodict" version = "1.0.4" From 9f8ff023f2f646da1c82f0ed4a638e4e09a6bf04 Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 17:29:37 -0800 Subject: [PATCH 3/5] fix(verifiers): cua_verifier bounds what it renders, marks cut state output, and refuses Windows computers - An Office file is measured as it decompresses and named as not rendered when it expands past 128 MiB; a PDF is read up to 200 pages (pypdf>=6 bounds each stream); rendering runs off the event loop under a 60 s limit. - State output over the limit, or cut by the driver, is marked incomplete in the prompt and the record. - A Windows computer is refused at preflight and before any read: files and state are read through sh and POSIX tools. Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 2 +- .../task_steps/verifiers/cua_verifier.py | 67 ++++++++++++++-- .../verifiers/judge_utils/cua_files.py | 37 ++++++++- tst/integration/computer/cua_verifier_test.py | 3 +- .../task_step/verifiers/test_cua_files.py | 34 +++++++- .../task_step/verifiers/test_cua_verifier.py | 77 ++++++++++++++++++- uv.lock | 2 +- 7 files changed, 205 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f330b25d6..a22be6767 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ cua = [ "openpyxl>=3.1", "python-docx>=1.1", "python-pptx>=1.0", - "pypdf>=4", + "pypdf>=6", ] # agent_env.openenv: bundle tasks served as OpenEnv envs. No cap: OpenEnv ships a minor release a week. openenv = [ diff --git a/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py index 672307cc8..db0aba8a7 100644 --- a/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py @@ -7,8 +7,9 @@ ``result_files``, read through the computer's driver (computer-exec/v1 ``run``, the control route the agent can't call) and rendered as text (``judge_utils/cua_files.py``), a file that isn't there named as one the agent did not produce; the golden files of ``expected_artifact_id``, for reference only; the output of each of ``state_commands`` run in the -computer, such as a mock website's state; and ``instruction``, the task as the agent was given it. With no -``prompt_id`` the judge grades those alone; with one it also sees that prompt's response. +computer, such as a mock website's state, marked incomplete when it was cut; and ``instruction``, the task as the agent +was given it. With no ``prompt_id`` the judge grades those alone; with one it also sees that prompt's response. Files +and state are read through ``sh`` and POSIX tools, so a Windows computer is refused, at preflight and before any read. How the judge grades is ``guidance``, and how it treats the golden files ``golden_note``: the caller's own wording, or short defaults of the framework's. The score: each criterion 1 or 0; the regular and passed must-pass criteria's weights @@ -19,16 +20,20 @@ from __future__ import annotations +import asyncio import base64 import binascii import logging from typing import Any, ClassVar, Optional +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER + from agent_env.entity.refs import EntityRef from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.preflight import stored_env from agent_env.task_step.task_steps.computer.computer_exec import computer_problem, find_computer -from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render +from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import Rendered, render from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -52,6 +57,10 @@ READ_TIMEOUT_S = 120.0 #: The most of one file's rendering, and of one command's output, the judge reads. MAX_FILE_CHARS = 40_000 +#: How long rendering one file may take before it's named as not rendered. +RENDER_TIMEOUT_S = 60.0 +#: Computers the step can't read: it reads files and runs state commands through ``sh`` and POSIX tools. +UNSUPPORTED_OS = frozenset({"windows"}) #: The most of a file one read returns: its base64 well within the 1 MiB a driver returns of a command's output. READ_CHUNK_BYTES = 512 * 1024 #: The largest file the judge is shown; a larger one is named as too large. @@ -116,6 +125,39 @@ def cua_rubric_score(rows: list[dict]) -> float: return aggregate_score(binary, ScoreAggregator.WEIGHTED_AVERAGE) +async def _render(name: str, data: bytes) -> Rendered: + """``render`` off the event loop, named as not rendered when it takes over ``RENDER_TIMEOUT_S``.""" + try: + return await asyncio.wait_for(asyncio.to_thread(render, name, data, MAX_FILE_CHARS), RENDER_TIMEOUT_S) + except TimeoutError: + return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it took over " + f"{RENDER_TIMEOUT_S:g} s)") + + +def _deployed_os(deployed: Any) -> Optional[str]: + """The OS a deployed computer's card names (computer/v1's ``platform``), None when it names none.""" + return (protocol_v1.extension_params(deployed.environment_card or {}, COMPUTER).get("platform") or {}).get("os") + + +def _stored_os_problem(env: Any, computer: Optional[str]) -> Optional[str]: + """Why the stored env's computer is one the step can't read, else None.""" + from agent_env.env.envs.computer.env import ComputerEnv + from agent_env.env.envs.multi_env import MultiEnv + + target = env + if isinstance(env, MultiEnv): + browser = env.website_browser().computer + candidates = [*env.computer_envs, *([browser] if browser else [])] + target = next((c for c in candidates if computer is None or c.environment_name == computer), None) + if not isinstance(target, ComputerEnv): + return None + os_type = getattr(getattr(target.computer, "os", None), "type", None) + if os_type in UNSUPPORTED_OS: + return (f"env {env.id!r}'s computer is {os_type}, which cua_verifier can't read: it reads files and runs state " + "commands through sh and POSIX tools") + return None + + def _passed(row: dict) -> bool: score = row.get("score") return isinstance(score, (int, float)) and not isinstance(score, bool) and score >= 1.0 - SCORE_TOLERANCE @@ -174,7 +216,7 @@ def preflight(self) -> list[str]: """rubrics_verifier's, and the env this step reads: a computer env, or a multi env with that computer.""" problems = super().preflight() if isinstance(self.env_id, str) and self.env_id and (env := stored_env(self.env_id)) is not None: - if problem := computer_problem(env, self.computer): + if problem := computer_problem(env, self.computer) or _stored_os_problem(env, self.computer): problems.append(f"step {self.id!r}: {problem}") return problems @@ -225,6 +267,9 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: """The files and state the agent left in the computer, the golden files, the instruction and how to grade; what was read is recorded under ``cua_verifier``.""" deployed = find_computer(context, self.env_id, self.computer) + if (os_type := _deployed_os(deployed)) in UNSUPPORTED_OS: + raise RuntimeError(f"cua_verifier '{self.id}': env {self.env_id!r} is a {os_type} computer, which it can't " + "read: it reads files and runs state commands through sh and POSIX tools") record: dict = {"files": [], "golden_files": [], "state": []} sections = [] produced, missing = [], [] @@ -239,7 +284,7 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: produced.append(f"### File {number}: {path}\n(a file of {data} bytes, too large to show: over " f"{MAX_READ_BYTES} bytes)") continue - shown = render(path, data, MAX_FILE_CHARS) + shown = await _render(path, data) record["files"].append({"path": path, "bytes": len(data), "rendered_as": shown.kind, "cut": shown.cut}) produced.append(f"### File {number}: {path} ({shown.kind})\n\n{shown.text}\n") sections.append("## Files under evaluation\nWhat the agent left at each path in its computer, rendered as " @@ -250,7 +295,7 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: if golden := self._golden_files(): shown_golden = [] for name, data in golden.items(): - shown = render(name, data, MAX_FILE_CHARS) + shown = await _render(name, data) record["golden_files"].append({"name": name, "bytes": len(data), "rendered_as": shown.kind}) shown_golden.append(f"### Golden: {name} ({shown.kind})\n\n{shown.text}\n" "") @@ -259,8 +304,14 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: state = [] for command in self.state_commands: output = await self._state(deployed, command) - record["state"].append({"name": command["name"], **{k: output[k] for k in ("exit_code", "timed_out")}}) - text = (output.get("stdout") or "")[:MAX_FILE_CHARS] + stdout = output.get("stdout") or "" + cut, truncated = len(stdout) > MAX_FILE_CHARS, bool(output.get("truncated")) + record["state"].append({"name": command["name"], **{k: output[k] for k in ("exit_code", "timed_out")}, + "cut": cut, "truncated": truncated}) + text = stdout[:MAX_FILE_CHARS] + if why := " and ".join([*([f"cut after {MAX_FILE_CHARS} characters"] if cut else []), + *(["cut short by the computer's driver"] if truncated else [])]): + text += f"\n… (INCOMPLETE: {why}. What isn't shown can't support a criterion.)" state.append(f"### {command['name']}\n" f"\n{text}\n") if state: diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py index 59d012f44..27ae01082 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py @@ -8,6 +8,7 @@ import hashlib import io import posixpath +import zipfile from dataclasses import dataclass from typing import Callable @@ -23,6 +24,13 @@ MAX_VISITED_CELLS = 200_000 #: How much of a file without an extension is sampled to tell whether it's text. TEXT_SAMPLE_BYTES = 4096 +#: The most an Office file (a zip package) may expand to before it's opened: a small file can hold members that +#: decompress to far more, which its library would read into memory. +MAX_EXPANDED_BYTES = 128 * 1024 * 1024 +#: How much of a package member is decompressed at a time while it's measured. +EXPAND_CHUNK_BYTES = 1024 * 1024 +#: The most pages of a PDF a rendering reads. +MAX_PDF_PAGES = 200 @dataclass(frozen=True) @@ -42,6 +50,9 @@ def render(name: str, data: bytes, max_chars: int) -> Rendered: renderer = _RENDERERS.get(extension) if renderer is not None: kind, library, read = renderer + if kind != "pdf" and (expanded := _expands_past_limit(data)) is not None: + return Rendered("unrendered", f"({extension} file of {len(data)} bytes, not rendered: it {expanded}, over " + f"the {MAX_EXPANDED_BYTES} bytes a rendering opens)") try: text = read(data) except ImportError: @@ -58,6 +69,24 @@ def render(name: str, data: bytes, max_chars: int) -> Rendered: "isn't shown)") +def _expands_past_limit(data: bytes) -> str | None: + """None when the zip package ``data`` decompresses within ``MAX_EXPANDED_BYTES``, counted by decompressing it a + chunk at a time rather than by its headers, which can lie; else how far it was seen to expand. A file that isn't a + readable zip is left to its library, which names it as unreadable.""" + try: + with zipfile.ZipFile(io.BytesIO(data)) as package: + total = 0 + for member in package.infolist(): + with package.open(member) as stream: + while chunk := stream.read(EXPAND_CHUNK_BYTES): + total += len(chunk) + if total > MAX_EXPANDED_BYTES: + return f"expands past {MAX_EXPANDED_BYTES} bytes" + except (zipfile.BadZipFile, zipfile.LargeZipFile, NotImplementedError, OSError, EOFError): + return None + return None + + def _cut(kind: str, text: str, max_chars: int) -> Rendered: if len(text) <= max_chars: return Rendered(kind, text) @@ -150,11 +179,15 @@ def shape_lines(shapes) -> list[str]: def _pdf(data: bytes) -> str: - """Each page's text.""" + """Each page's text, up to ``MAX_PDF_PAGES`` pages; pypdf bounds each stream it decompresses.""" import pypdf pages = pypdf.PdfReader(io.BytesIO(data)).pages - return "\n".join(f"## Page {number}\n{page.extract_text() or ''}" for number, page in enumerate(pages, start=1)) + lines = [f"## Page {number}\n{page.extract_text() or ''}" + for number, page in enumerate(pages[:MAX_PDF_PAGES], start=1)] + if len(pages) > MAX_PDF_PAGES: + lines.append(f"… (stopped at {MAX_PDF_PAGES} of {len(pages)} pages)") + return "\n".join(lines) _RENDERERS: dict[str, tuple[str, str, Callable[[bytes], str]]] = { diff --git a/tst/integration/computer/cua_verifier_test.py b/tst/integration/computer/cua_verifier_test.py index a6d58ee9f..e40e40bdf 100644 --- a/tst/integration/computer/cua_verifier_test.py +++ b/tst/integration/computer/cua_verifier_test.py @@ -49,7 +49,8 @@ def test_a_cua_rubric_is_graded_on_what_the_agent_left_in_the_computer(state, mo files = metadata["cua_verifier"]["grade"]["files"] assert [(f["path"], f["rendered_as"]) for f in files] == [("~/Desktop/budget.csv", "text"), ("~/notes.txt", "text")], files - assert metadata["cua_verifier"]["grade"]["state"] == [{"name": "mailbox", "exit_code": 0, "timed_out": False}] + assert metadata["cua_verifier"]["grade"]["state"] == [ + {"name": "mailbox", "exit_code": 0, "timed_out": False, "cut": False, "truncated": False}] graded = metadata["verifications"]["cua"] assert graded["score"] == 1.0 and graded["gate_passed"], graded["results"] gated = metadata["verifications"]["cua-gate"] diff --git a/tst/unit/task_step/verifiers/test_cua_files.py b/tst/unit/task_step/verifiers/test_cua_files.py index 853e62b62..13aad0d55 100644 --- a/tst/unit/task_step/verifiers/test_cua_files.py +++ b/tst/unit/task_step/verifiers/test_cua_files.py @@ -1,17 +1,20 @@ """cua_files: each format a CUA task's agent leaves as the judge reads it: documents, slides, spreadsheets and PDFs dumped as text, text as it is and cut at the limit, anything else by its size and digest, and a format whose library -isn't installed, or a file its library can't read, named as not rendered.""" +isn't installed, a file its library can't read, or an Office file that expands past the limit, named as not rendered; +a long PDF read up to its page limit.""" from __future__ import annotations import builtins import io +import zipfile import docx import pptx import pypdf import pytest +from agent_env.task_step.task_steps.verifiers.judge_utils import cua_files from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render @@ -116,3 +119,32 @@ def test_grouped_slide_text_is_shown(): deck.save(out) assert "Revenue up 12%" in render("deck.pptx", out.getvalue(), 10_000).text + + +def test_an_office_file_that_expands_past_the_limit_is_named_not_opened(monkeypatch): + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as package: + package.writestr("word/document.xml", b"\0" * (4 * 1024 * 1024)) + bomb, memo = out.getvalue(), _docx() + monkeypatch.setattr(cua_files, "MAX_EXPANDED_BYTES", 1024 * 1024) + assert render("memo.docx", memo, 1000).kind == "docx" + opened = [] + monkeypatch.setattr(docx, "Document", lambda *a: opened.append(a)) + + shown = render("memo.docx", bomb, 1000) + + assert shown.kind == "unrendered" and "expands past 1048576 bytes" in shown.text and len(bomb) < 64 * 1024 + assert opened == [] + + +def test_a_long_pdf_is_read_up_to_its_page_limit(monkeypatch): + writer = pypdf.PdfWriter() + for _ in range(3): + writer.add_blank_page(width=200, height=200) + out = io.BytesIO() + writer.write(out) + monkeypatch.setattr(cua_files, "MAX_PDF_PAGES", 2) + + shown = render("long.pdf", out.getvalue(), 10_000) + + assert "## Page 2" in shown.text and "## Page 3" not in shown.text and "stopped at 2 of 3 pages" in shown.text diff --git a/tst/unit/task_step/verifiers/test_cua_verifier.py b/tst/unit/task_step/verifiers/test_cua_verifier.py index 71ed46df0..aab944f41 100644 --- a/tst/unit/task_step/verifiers/test_cua_verifier.py +++ b/tst/unit/task_step/verifiers/test_cua_verifier.py @@ -2,29 +2,37 @@ negative penalties, must-pass gates), and a run against a computer at its driver's boundary (``tst.unit.computer_home``) with the judge faked: the files the agent left read through the driver and rendered into the judge's prompt, a missing one named, the golden files and the state commands' output beside them, the caller's guidance or the default, and the -files graded alone when no prompt step names them.""" +files graded alone when no prompt step names them; state output that was cut marked incomplete, a Windows computer +refused, and a file too slow to render named as not rendered.""" from __future__ import annotations import io import json import re +import time +from types import SimpleNamespace import openpyxl import pytest +from agentenv_protocol.computer import COMPUTER from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.env.envs.computer.env import ComputerEnv from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.verifiers import cua_verifier from agent_env.task_step.task_steps.verifiers.cua_verifier import ( DEFAULT_GOLDEN_NOTE, DEFAULT_GUIDANCE, + MAX_FILE_CHARS, RECORDS_KEY, CuaVerifierTaskStep, criteria_from_rubric, cua_rubric_score, ) -from tst.unit.computer_home import Computer +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.computer_home import CARD, Computer RUBRIC = {"criteria": [ {"id": "sheet-exists", "title": "Provides report.xlsx.", "weight": 8, @@ -162,7 +170,7 @@ async def test_the_files_left_in_the_computer_are_rendered_into_the_judges_promp (f"{computer.home}/Desktop/report.xlsx", "xlsx", False), ("~/notes.txt", "text", False), ("~/missing.pdf", None, True)] assert record["golden_files"] == [{"name": "report.csv", "bytes": 10, "rendered_as": "text"}] - assert record["state"] == [{"name": "mailbox", "exit_code": 0, "timed_out": False}] + assert record["state"] == [{"name": "mailbox", "exit_code": 0, "timed_out": False, "cut": False, "truncated": False}] @pytest.mark.asyncio @@ -267,3 +275,66 @@ def test_it_round_trips_through_the_registry_keeping_a_files_only_grade(): def test_bad_configs_are_refused(fields, problem): with pytest.raises(ValueError, match=problem): _step(**fields) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stdout, truncated, why", [ + ("x" * (MAX_FILE_CHARS + 5), False, f"cut after {MAX_FILE_CHARS} characters"), + ("short", True, "cut short by the computer's driver"), +]) +async def test_state_output_that_was_cut_is_marked_incomplete(computer, stdout, truncated, why): + step = _step(result_files=["~/notes.txt"], state_commands=[{"name": "mailbox", "command": "cat /tmp/m.json"}]) + prompts: list[str] = [] + _judge(step, prompts) + + async def state(deployed, command): + return {"exit_code": 0, "timed_out": False, "stdout": stdout, "truncated": truncated} + + step._state = state + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + await step.execute(ctx) + + assert f"INCOMPLETE: {why}. What isn't shown can't support a criterion." in prompts[0] + [record] = ctx.metadata[RECORDS_KEY]["grade"]["state"] + assert record["cut"] == (len(stdout) > MAX_FILE_CHARS) and record["truncated"] == truncated + + +@pytest.mark.asyncio +async def test_a_windows_computer_is_refused_before_anything_is_read(computer): + card = {**CARD, "capabilities": {**CARD["capabilities"], "extensions": [ + *CARD["capabilities"]["extensions"], {"uri": COMPUTER, "params": {"platform": {"os": "windows"}}}]}} + step = _step(result_files=["~/notes.txt"]) + _judge(step, []) + + with pytest.raises(RuntimeError, match="is a windows computer, which it can't read"): + await step.execute(TaskStepContext(deployed_envs=[computer.deployed(card=card)])) + assert computer.calls == [] + + +def test_preflight_refuses_a_stored_windows_computer(monkeypatch): + desk = ComputerEnv.__new__(ComputerEnv) + desk.id, desk.environment_name = "desk", "desk" + desk.computer = SimpleNamespace(os=SimpleNamespace(type="windows")) + monkeypatch.setattr(cua_verifier, "stored_env", lambda env_id: desk) + monkeypatch.setattr(RubricsVerifierTaskStep, "preflight", lambda self: []) + + assert _step(result_files=["~/a.txt"]).preflight() == [ + "step 'grade': env 'desk''s computer is windows, which cua_verifier can't read: it reads files and runs state " + "commands through sh and POSIX tools"] + desk.computer = SimpleNamespace(os=SimpleNamespace(type="linux")) + assert _step(result_files=["~/a.txt"]).preflight() == [] + + +@pytest.mark.asyncio +async def test_a_file_that_takes_too_long_to_render_is_named_not_rendered(computer, monkeypatch): + (computer.home / "notes.txt").write_text("done\n") + monkeypatch.setattr(cua_verifier, "RENDER_TIMEOUT_S", 0.05) + monkeypatch.setattr(cua_verifier, "render", lambda *a: time.sleep(0.5)) + step = _step(result_files=["~/notes.txt"]) + prompts: list[str] = [] + _judge(step, prompts) + + await step.execute(TaskStepContext(deployed_envs=[computer.deployed()])) + + assert "not rendered: rendering it took over 0.05 s" in prompts[0] diff --git a/uv.lock b/uv.lock index 3bd395e10..de8651941 100644 --- a/uv.lock +++ b/uv.lock @@ -265,7 +265,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'verl'", specifier = ">=19" }, { name = "pydantic", specifier = ">=2.12" }, { name = "pymongo", specifier = ">=4.7" }, - { name = "pypdf", marker = "extra == 'cua'", specifier = ">=4" }, + { name = "pypdf", marker = "extra == 'cua'", specifier = ">=6" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "pytest-dependency", marker = "extra == 'dev'", specifier = ">=0.6.0" }, From 0246f2bdd66d8b17a01b99fe71099d0edce75229 Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 17:36:13 -0800 Subject: [PATCH 4/5] fix(verifiers): cua_verifier renders each file in a memory- and time-limited process, and names a damaged one A file is rendered in a child process with its address space capped at 2 GiB and killed after 60 s, so a crafted document can't exhaust or block the worker. The Office packages' expansion check runs inside render's error handling, so a document with damaged compressed data is named as unreadable, not raised. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../task_steps/verifiers/cua_verifier.py | 18 +--- .../verifiers/judge_utils/cua_files.py | 93 +++++++++++++++---- .../task_step/verifiers/test_cua_files.py | 35 ++++++- .../task_step/verifiers/test_cua_verifier.py | 17 +--- 4 files changed, 111 insertions(+), 52 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py index db0aba8a7..5756b0f01 100644 --- a/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/cua_verifier.py @@ -20,7 +20,6 @@ from __future__ import annotations -import asyncio import base64 import binascii import logging @@ -33,7 +32,7 @@ from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.preflight import stored_env from agent_env.task_step.task_steps.computer.computer_exec import computer_problem, find_computer -from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import Rendered, render +from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render_isolated from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -57,8 +56,6 @@ READ_TIMEOUT_S = 120.0 #: The most of one file's rendering, and of one command's output, the judge reads. MAX_FILE_CHARS = 40_000 -#: How long rendering one file may take before it's named as not rendered. -RENDER_TIMEOUT_S = 60.0 #: Computers the step can't read: it reads files and runs state commands through ``sh`` and POSIX tools. UNSUPPORTED_OS = frozenset({"windows"}) #: The most of a file one read returns: its base64 well within the 1 MiB a driver returns of a command's output. @@ -125,15 +122,6 @@ def cua_rubric_score(rows: list[dict]) -> float: return aggregate_score(binary, ScoreAggregator.WEIGHTED_AVERAGE) -async def _render(name: str, data: bytes) -> Rendered: - """``render`` off the event loop, named as not rendered when it takes over ``RENDER_TIMEOUT_S``.""" - try: - return await asyncio.wait_for(asyncio.to_thread(render, name, data, MAX_FILE_CHARS), RENDER_TIMEOUT_S) - except TimeoutError: - return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it took over " - f"{RENDER_TIMEOUT_S:g} s)") - - def _deployed_os(deployed: Any) -> Optional[str]: """The OS a deployed computer's card names (computer/v1's ``platform``), None when it names none.""" return (protocol_v1.extension_params(deployed.environment_card or {}, COMPUTER).get("platform") or {}).get("os") @@ -284,7 +272,7 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: produced.append(f"### File {number}: {path}\n(a file of {data} bytes, too large to show: over " f"{MAX_READ_BYTES} bytes)") continue - shown = await _render(path, data) + shown = await render_isolated(path, data, MAX_FILE_CHARS) record["files"].append({"path": path, "bytes": len(data), "rendered_as": shown.kind, "cut": shown.cut}) produced.append(f"### File {number}: {path} ({shown.kind})\n\n{shown.text}\n") sections.append("## Files under evaluation\nWhat the agent left at each path in its computer, rendered as " @@ -295,7 +283,7 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: if golden := self._golden_files(): shown_golden = [] for name, data in golden.items(): - shown = await _render(name, data) + shown = await render_isolated(name, data, MAX_FILE_CHARS) record["golden_files"].append({"name": name, "bytes": len(data), "rendered_as": shown.kind}) shown_golden.append(f"### Golden: {name} ({shown.kind})\n\n{shown.text}\n" "") diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py index 27ae01082..e8f479fd1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py @@ -1,13 +1,18 @@ """Files a CUA task's agent produced, and their golden counterparts, as text a rubric judge reads in its prompt (``cua_verifier``): text formats as they are; spreadsheets, documents, slides and PDFs dumped by the libraries the ``cua`` extra installs (openpyxl, python-docx, python-pptx, pypdf), or named as not rendered without them; anything else -by its size and digest. Each rendering is cut at ``max_chars``.""" +by its size and digest. Each rendering is cut at ``max_chars``. ``render_isolated`` renders in a child process held to +``RENDER_MEMORY_BYTES`` and killed after ``RENDER_TIMEOUT_S``, since a crafted file can make a library's work large.""" from __future__ import annotations +import asyncio import hashlib import io +import json import posixpath +import resource +import sys import zipfile from dataclasses import dataclass from typing import Callable @@ -31,6 +36,12 @@ EXPAND_CHUNK_BYTES = 1024 * 1024 #: The most pages of a PDF a rendering reads. MAX_PDF_PAGES = 200 +#: The most memory (address space) a rendering's child process may take. +RENDER_MEMORY_BYTES = 2 * 1024 * 1024 * 1024 +#: How long a rendering's child process may run before it's killed. +RENDER_TIMEOUT_S = 60.0 +#: The child process ``render_isolated`` runs: this module's ``main``. +RENDER_COMMAND = (sys.executable, "-m", __name__) @dataclass(frozen=True) @@ -50,11 +61,11 @@ def render(name: str, data: bytes, max_chars: int) -> Rendered: renderer = _RENDERERS.get(extension) if renderer is not None: kind, library, read = renderer - if kind != "pdf" and (expanded := _expands_past_limit(data)) is not None: - return Rendered("unrendered", f"({extension} file of {len(data)} bytes, not rendered: it {expanded}, over " - f"the {MAX_EXPANDED_BYTES} bytes a rendering opens)") try: text = read(data) + except _TooLarge: + return Rendered("unrendered", f"({extension} file of {len(data)} bytes, not rendered: it expands past " + f"{MAX_EXPANDED_BYTES} bytes, more than a rendering opens)") except ImportError: return Rendered("unrendered", f"({extension} file of {len(data)} bytes, not rendered: {library} isn't " "installed; install agentenv-framework[cua])") @@ -69,22 +80,58 @@ def render(name: str, data: bytes, max_chars: int) -> Rendered: "isn't shown)") -def _expands_past_limit(data: bytes) -> str | None: - """None when the zip package ``data`` decompresses within ``MAX_EXPANDED_BYTES``, counted by decompressing it a - chunk at a time rather than by its headers, which can lie; else how far it was seen to expand. A file that isn't a - readable zip is left to its library, which names it as unreadable.""" +class _TooLarge(Exception): + """An Office package that decompresses past ``MAX_EXPANDED_BYTES``.""" + + +def _check_expansion(data: bytes) -> None: + """Raises ``_TooLarge`` when the zip package ``data`` decompresses past ``MAX_EXPANDED_BYTES``, counted by + decompressing it a chunk at a time rather than by its headers, which can lie. A package that can't be read raises + as its library would, for ``render`` to name it unreadable.""" + total = 0 + with zipfile.ZipFile(io.BytesIO(data)) as package: + for member in package.infolist(): + with package.open(member) as stream: + while chunk := stream.read(EXPAND_CHUNK_BYTES): + total += len(chunk) + if total > MAX_EXPANDED_BYTES: + raise _TooLarge + + +async def render_isolated(name: str, data: bytes, max_chars: int) -> Rendered: + """``render`` in a child process (``RENDER_COMMAND``) held to ``RENDER_MEMORY_BYTES`` and killed after + ``RENDER_TIMEOUT_S``; a file whose rendering ran out of either is named as not rendered.""" + child = await asyncio.create_subprocess_exec( + *RENDER_COMMAND, name, str(max_chars), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, preexec_fn=_hold_memory) + try: + out, _ = await asyncio.wait_for(child.communicate(data), RENDER_TIMEOUT_S) + except TimeoutError: + child.kill() + await child.wait() + return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it took over " + f"{RENDER_TIMEOUT_S:g} s)") + if child.returncode != 0: + return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it ran out of memory or " + f"failed, exit {child.returncode})") + shown = json.loads(out) + return Rendered(shown["kind"], shown["text"], shown["cut"]) + + +def _hold_memory() -> None: + """In the child, before it runs: its address space capped at ``RENDER_MEMORY_BYTES`` where the OS enforces it.""" try: - with zipfile.ZipFile(io.BytesIO(data)) as package: - total = 0 - for member in package.infolist(): - with package.open(member) as stream: - while chunk := stream.read(EXPAND_CHUNK_BYTES): - total += len(chunk) - if total > MAX_EXPANDED_BYTES: - return f"expands past {MAX_EXPANDED_BYTES} bytes" - except (zipfile.BadZipFile, zipfile.LargeZipFile, NotImplementedError, OSError, EOFError): - return None - return None + resource.setrlimit(resource.RLIMIT_AS, (RENDER_MEMORY_BYTES, RENDER_MEMORY_BYTES)) + except (ValueError, OSError): # macOS doesn't enforce RLIMIT_AS; the time limit still holds + pass + + +def main() -> None: + """The child ``render_isolated`` runs: the file's bytes on stdin, its name and ``max_chars`` as arguments, its + rendering as JSON on stdout.""" + name, max_chars = sys.argv[1], int(sys.argv[2]) + shown = render(name, sys.stdin.buffer.read(), max_chars) + json.dump({"kind": shown.kind, "text": shown.text, "cut": shown.cut}, sys.stdout) def _cut(kind: str, text: str, max_chars: int) -> Rendered: @@ -111,6 +158,7 @@ def _xlsx(data: bytes) -> str: import openpyxl from openpyxl.utils import get_column_letter + _check_expansion(data) formulas = openpyxl.load_workbook(io.BytesIO(data), data_only=False, read_only=True) values = openpyxl.load_workbook(io.BytesIO(data), data_only=True, read_only=True) lines, listed, visited = [], 0, 0 @@ -145,6 +193,7 @@ def _docx(data: bytes) -> str: """Each paragraph with its style, then each table's rows, cells split by `` | ``.""" import docx + _check_expansion(data) document = docx.Document(io.BytesIO(data)) lines = [f"[{paragraph.style.name}] {paragraph.text}" for paragraph in document.paragraphs if paragraph.text.strip()] for number, table in enumerate(document.tables, start=1): @@ -157,6 +206,8 @@ def _pptx(data: bytes) -> str: """Each slide's shapes' text and tables, a group's shapes within it included, and its notes.""" import pptx + _check_expansion(data) + def shape_lines(shapes) -> list[str]: lines = [] for shape in shapes: @@ -193,3 +244,7 @@ def _pdf(data: bytes) -> str: _RENDERERS: dict[str, tuple[str, str, Callable[[bytes], str]]] = { ".xlsx": ("xlsx", "openpyxl", _xlsx), ".xlsm": ("xlsx", "openpyxl", _xlsx), ".docx": ("docx", "python-docx", _docx), ".pptx": ("pptx", "python-pptx", _pptx), ".pdf": ("pdf", "pypdf", _pdf)} + + +if __name__ == "__main__": + main() diff --git a/tst/unit/task_step/verifiers/test_cua_files.py b/tst/unit/task_step/verifiers/test_cua_files.py index 13aad0d55..b30acea53 100644 --- a/tst/unit/task_step/verifiers/test_cua_files.py +++ b/tst/unit/task_step/verifiers/test_cua_files.py @@ -1,7 +1,8 @@ """cua_files: each format a CUA task's agent leaves as the judge reads it: documents, slides, spreadsheets and PDFs dumped as text, text as it is and cut at the limit, anything else by its size and digest, and a format whose library isn't installed, a file its library can't read, or an Office file that expands past the limit, named as not rendered; -a long PDF read up to its page limit.""" +a long PDF read up to its page limit; a damaged one named as unreadable; and a rendering in its own process, killed when +it runs too long and named as not rendered when it fails.""" from __future__ import annotations @@ -15,7 +16,7 @@ import pytest from agent_env.task_step.task_steps.verifiers.judge_utils import cua_files -from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render +from agent_env.task_step.task_steps.verifiers.judge_utils.cua_files import render, render_isolated def _docx() -> bytes: @@ -148,3 +149,33 @@ def test_a_long_pdf_is_read_up_to_its_page_limit(monkeypatch): shown = render("long.pdf", out.getvalue(), 10_000) assert "## Page 2" in shown.text and "## Page 3" not in shown.text and "stopped at 2 of 3 pages" in shown.text + + +def test_a_document_with_damaged_compressed_data_is_named_as_unreadable(): + memo = bytearray(_docx()) + start = memo.find(b"word/document.xml") + len("word/document.xml") + memo[start + 20:start + 60] = b"\xff" * 40 # inside the member's deflate stream + + shown = render("memo.docx", bytes(memo), 1000) + + assert shown.kind == "unrendered" and "couldn't read" in shown.text + + +@pytest.mark.asyncio +async def test_a_rendering_runs_in_its_own_process(): + shown = await render_isolated("memo.docx", _docx(), 1000) + assert shown.kind == "docx" and "The figures are corrected." in shown.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("command, problem", [ + (("sh", "-c", "sleep 5"), "rendering it took over 0.2 s"), + (("sh", "-c", "exit 9"), "ran out of memory or failed, exit 9"), +]) +async def test_a_rendering_that_runs_too_long_or_fails_is_named_not_rendered(monkeypatch, command, problem): + monkeypatch.setattr(cua_files, "RENDER_COMMAND", command) + monkeypatch.setattr(cua_files, "RENDER_TIMEOUT_S", 0.2) + + shown = await render_isolated("memo.docx", _docx(), 1000) + + assert shown.kind == "unrendered" and problem in shown.text diff --git a/tst/unit/task_step/verifiers/test_cua_verifier.py b/tst/unit/task_step/verifiers/test_cua_verifier.py index aab944f41..d629e2bd7 100644 --- a/tst/unit/task_step/verifiers/test_cua_verifier.py +++ b/tst/unit/task_step/verifiers/test_cua_verifier.py @@ -3,14 +3,13 @@ with the judge faked: the files the agent left read through the driver and rendered into the judge's prompt, a missing one named, the golden files and the state commands' output beside them, the caller's guidance or the default, and the files graded alone when no prompt step names them; state output that was cut marked incomplete, a Windows computer -refused, and a file too slow to render named as not rendered.""" +refused.""" from __future__ import annotations import io import json import re -import time from types import SimpleNamespace import openpyxl @@ -324,17 +323,3 @@ def test_preflight_refuses_a_stored_windows_computer(monkeypatch): "commands through sh and POSIX tools"] desk.computer = SimpleNamespace(os=SimpleNamespace(type="linux")) assert _step(result_files=["~/a.txt"]).preflight() == [] - - -@pytest.mark.asyncio -async def test_a_file_that_takes_too_long_to_render_is_named_not_rendered(computer, monkeypatch): - (computer.home / "notes.txt").write_text("done\n") - monkeypatch.setattr(cua_verifier, "RENDER_TIMEOUT_S", 0.05) - monkeypatch.setattr(cua_verifier, "render", lambda *a: time.sleep(0.5)) - step = _step(result_files=["~/notes.txt"]) - prompts: list[str] = [] - _judge(step, prompts) - - await step.execute(TaskStepContext(deployed_envs=[computer.deployed()])) - - assert "not rendered: rendering it took over 0.05 s" in prompts[0] From 7484f69bc2de19d7114958175d09d21d6a94791b Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 17:40:07 -0800 Subject: [PATCH 5/5] fix(verifiers): a cancelled cua_verifier rendering kills its process Co-Authored-By: Claude Opus 5.5 (1M context) --- .../verifiers/judge_utils/cua_files.py | 9 +++++--- .../task_step/verifiers/test_cua_files.py | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py index e8f479fd1..706d069f0 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/cua_files.py @@ -100,17 +100,20 @@ def _check_expansion(data: bytes) -> None: async def render_isolated(name: str, data: bytes, max_chars: int) -> Rendered: """``render`` in a child process (``RENDER_COMMAND``) held to ``RENDER_MEMORY_BYTES`` and killed after - ``RENDER_TIMEOUT_S``; a file whose rendering ran out of either is named as not rendered.""" + ``RENDER_TIMEOUT_S``; a file whose rendering ran out of either is named as not rendered. The child is killed + whenever the wait ends early, a cancelled grade included, so none outlives its rendering.""" child = await asyncio.create_subprocess_exec( *RENDER_COMMAND, name, str(max_chars), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, preexec_fn=_hold_memory) try: out, _ = await asyncio.wait_for(child.communicate(data), RENDER_TIMEOUT_S) except TimeoutError: - child.kill() - await child.wait() return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it took over " f"{RENDER_TIMEOUT_S:g} s)") + finally: + if child.returncode is None: + child.kill() + await asyncio.shield(child.wait()) if child.returncode != 0: return Rendered("unrendered", f"(file of {len(data)} bytes, not rendered: rendering it ran out of memory or " f"failed, exit {child.returncode})") diff --git a/tst/unit/task_step/verifiers/test_cua_files.py b/tst/unit/task_step/verifiers/test_cua_files.py index b30acea53..3fb4c06ce 100644 --- a/tst/unit/task_step/verifiers/test_cua_files.py +++ b/tst/unit/task_step/verifiers/test_cua_files.py @@ -2,12 +2,14 @@ dumped as text, text as it is and cut at the limit, anything else by its size and digest, and a format whose library isn't installed, a file its library can't read, or an Office file that expands past the limit, named as not rendered; a long PDF read up to its page limit; a damaged one named as unreadable; and a rendering in its own process, killed when -it runs too long and named as not rendered when it fails.""" +it runs too long or its grade is cancelled, and named as not rendered when it fails.""" from __future__ import annotations +import asyncio import builtins import io +import os import zipfile import docx @@ -179,3 +181,21 @@ async def test_a_rendering_that_runs_too_long_or_fails_is_named_not_rendered(mon shown = await render_isolated("memo.docx", _docx(), 1000) assert shown.kind == "unrendered" and problem in shown.text + + +@pytest.mark.asyncio +async def test_a_cancelled_rendering_kills_its_process(monkeypatch, tmp_path): + pid_file = tmp_path / "pid" + monkeypatch.setattr(cua_files, "RENDER_COMMAND", ("sh", "-c", f"echo $$ > {pid_file}; exec sleep 30")) + + rendering = asyncio.ensure_future(render_isolated("memo.docx", b"x", 1000)) + for _ in range(100): + if pid_file.exists() and pid_file.read_text().strip(): + break + await asyncio.sleep(0.02) + rendering.cancel() + with pytest.raises(asyncio.CancelledError): + await rendering + + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0)