From 3d3ffd899d10d867b3f8f7b1902bd93b832b1fbd Mon Sep 17 00:00:00 2001 From: William Loo Date: Wed, 18 Nov 2026 16:10:35 -0800 Subject: [PATCH 1/4] feat(task-steps): computer_exec, a command run inside a computer through its driver Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- .../env/envs/computer/utils/inside.py | 11 +- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 2 + .../task_steps/computer/computer_exec.py | 188 +++++++++++++++++ .../computer_exec/envs/exec-desk/env.toml | 3 + .../computer/computer_exec/tasks/setup.json | 13 ++ .../computer/computer_exec_test.py | 29 +++ tst/unit/computer_home.py | 5 +- tst/unit/task_step/computer_exec_test.py | 195 ++++++++++++++++++ 10 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/computer/computer_exec.py create mode 100644 tst/integration/computer/computer_exec/envs/exec-desk/env.toml create mode 100644 tst/integration/computer/computer_exec/tasks/setup.json create mode 100644 tst/integration/computer/computer_exec_test.py create mode 100644 tst/unit/task_step/computer_exec_test.py diff --git a/AGENTS.md b/AGENTS.md index 784e1ea05..543ee8072 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/env/envs/computer/utils/inside.py b/src/agent_env/env/envs/computer/utils/inside.py index c03ccecaa..b77ffd885 100644 --- a/src/agent_env/env/envs/computer/utils/inside.py +++ b/src/agent_env/env/envs/computer/utils/inside.py @@ -92,11 +92,12 @@ async def read_file(deployed: DeployedSandboxEnv, path: str) -> bytes: async def run(deployed: DeployedSandboxEnv, argv: list[str], *, timeout_s: float, - env: Optional[dict[str, str]] = None) -> dict: - """computer-exec/v1 ``run``'s answer for ``argv``, under a request id of its own. A call whose connection breaks - fails rather than being sent again: a driver that restarted would run the command a second time, beside the first, - which outlives the driver that started it.""" - params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {})} + env: Optional[dict[str, str]] = None, cwd: Optional[str] = None) -> dict: + """computer-exec/v1 ``run``'s answer for ``argv``, in ``cwd`` relative to the home, the home by default, under a + request id of its own. A call whose connection breaks fails rather than being sent again: a driver that restarted + would run the command a second time, beside the first, which outlives the driver that started it.""" + params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {}), + **({"cwd": cwd} if cwd else {})} try: return await deployed.invoke(COMPUTER_EXEC, "run", params, timeout=timeout_s + CALL_MARGIN_S) except httpx.TransportError as e: diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index f6cd5045a..dcf251f19 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -156,7 +156,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage", "browser_close_tab", "browser_delete_cookies", "browser_lease_context", "browser_open_tab", "browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli", - "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", + "combine_universe_verdicts", "computer_exec", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", "end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "user_sim_judge", "run_container_unit_tests_verifier", "run_docker_container", "set_env_clock_rate", "snapshot_env", "snapshot_computer_env", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index c4b5c5b9d..43bd7f79e 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -104,6 +104,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep + from agent_env.task_step.task_steps.computer.computer_exec import ComputerExecTaskStep from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep @@ -174,6 +175,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(SnapshotAgentStateTaskStep): SnapshotAgentStateTaskStep, _get_type(SnapshotEnvTaskStep): SnapshotEnvTaskStep, _get_type(SnapshotComputerEnvTaskStep): SnapshotComputerEnvTaskStep, + _get_type(ComputerExecTaskStep): ComputerExecTaskStep, _get_type(RunCodeTaskStep): RunCodeTaskStep, _get_type(PeerAgentsTaskStep): PeerAgentsTaskStep, _get_type(VerifyA2ASnapshotStep): VerifyA2ASnapshotStep, diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py new file mode 100644 index 000000000..e253b417f --- /dev/null +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -0,0 +1,188 @@ +"""computer_exec: a command run inside a deployed computer as its session user, through its driver's computer-exec/v1 +``run``, a control route the agent can't call, as a task sets a computer up: a program run to its end, its exit status +and output recorded, or with ``background`` an app started and left running.""" + +from __future__ import annotations + +import logging +import shlex +from typing import ClassVar, Optional, Union + +from agentenv_protocol.computer import COMPUTER_EXEC + +from agent_env.entity.refs import EntityRef +from agent_env.env.env import DeployedEnv, DeployedSandboxEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils import inside +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.preflight import stored_env +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +logger = logging.getLogger(__name__) + +#: Where the step records what each run did, by step id. +RECORDS_KEY = "computer_exec" +#: How long a command may run when the step doesn't say. +DEFAULT_TIMEOUT_S = 120.0 +#: The most of each stream the record keeps, and the most of stderr a failure says: the end of it. +RECORDED_CHARS = 4000 +FAILURE_CHARS = 300 +#: How a ``background`` command starts: its program found and executable, else it fails as the shell does with 127, +#: then detached from the run's pipes, so the run returns as the shell does. +_BACKGROUND = ('case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' + ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' + '"$@" /dev/null 2>&1 &') + +Command = Union[list[str], str] + + +class ComputerExecTaskStep(TaskStep): + """Run ``command`` inside the computer of the env ``env_id`` as its session user, in ``cwd`` (relative to the home, + the home by default), with ``env`` added to its environment, for at most ``timeout_s``. ``command`` is a program and + its arguments, or a string run by ``sh -c``. For a multi env, ``computer`` names the computer, by its environment + name, and may be left out when the env has one. + + The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and + stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless + ``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app + launched for the agent is: the step waits for the start alone, and fails as a run does when its program isn't + there to start.""" + + type: ClassVar[str] = "computer_exec" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, command: Command, + computer: Optional[str] = None, cwd: Optional[str] = None, env: Optional[dict[str, str]] = None, + timeout_s: float = DEFAULT_TIMEOUT_S, background: bool = False, check: bool = True, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.command = command + self.computer = computer + self.cwd = cwd + self.env = env + self.timeout_s = timeout_s + self.background = background + self.check = check + + def config_problems(self) -> list[str]: + problems = [] + if not isinstance(self.env_id, str) or not self.env_id: + problems.append(f"step {self.id!r}: env_id must be a non-empty string") + if isinstance(self.command, str): + if not self.command.strip(): + problems.append(f"step {self.id!r}: command must not be blank") + elif not (isinstance(self.command, list) and self.command and all(isinstance(a, str) for a in self.command)): + problems.append(f"step {self.id!r}: command must be a non-empty list of strings, or a string run by sh -c") + if self.computer is not None and (not isinstance(self.computer, str) or not self.computer): + problems.append(f"step {self.id!r}: computer, when given, must be a non-empty string") + if self.cwd is not None and (not isinstance(self.cwd, str) or self.cwd.startswith("/")): + problems.append(f"step {self.id!r}: cwd is a folder relative to the session user's home, not {self.cwd!r}") + if self.env is not None and not (isinstance(self.env, dict) and all( + isinstance(k, str) and k and isinstance(v, str) for k, v in self.env.items())): + problems.append(f"step {self.id!r}: env maps variable names to string values") + if isinstance(self.timeout_s, bool) or not isinstance(self.timeout_s, (int, float)) or self.timeout_s <= 0: + problems.append(f"step {self.id!r}: timeout_s must be a number of seconds above 0") + for name in ("background", "check"): + if not isinstance(getattr(self, name), bool): + problems.append(f"step {self.id!r}: {name} must be true or false") + return problems + + def preflight(self) -> list[str]: + """The env this step names when it has no computer, or none of that name, or several when it names none.""" + problems = super().preflight() + if not isinstance(self.env_id, str) or not self.env_id or (env := stored_env(self.env_id)) is None: + return problems + if problem := _computer_problem(env, self.computer): + problems.append(f"step {self.id!r}: {problem}") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "command": self.command, "computer": self.computer, + "cwd": self.cwd, "env": self.env, "timeout_s": self.timeout_s, "background": self.background, + "check": self.check} + + @classmethod + def from_dict(cls, data: dict) -> ComputerExecTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], command=data["command"], + computer=data.get("computer"), cwd=data.get("cwd"), env=data.get("env"), + timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False), + check=data.get("check", True)) + + def argv(self) -> list[str]: + """What the driver runs: the command, through ``sh -c`` when it's a string, started detached with + ``background``.""" + argv = ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command) + return ["sh", "-c", _BACKGROUND, "computer_exec", *argv] if self.background else argv + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = find_computer(context, self.env_id, self.computer) + if missing := [m for m in inside.missing(deployed) if m.startswith(COMPUTER_EXEC)]: + raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer serves no {missing[0]}") + if self.timeout_s > (limit := inside.max_timeout_s(deployed)): + raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer " + f"of env {self.env_id!r} lets a command run") + argv = self.argv() + answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd) + record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "background": self.background, + "exit_code": answer.get("exit_code"), "timed_out": bool(answer.get("timed_out")), + "truncated": bool(answer.get("truncated")), + "stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS), + "stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)} + context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record + shown = shlex.join(argv) + if record["timed_out"]: + outcome = f"timed out after {self.timeout_s} s" + elif record["exit_code"] != 0: + outcome = f"exited {record['exit_code']}" + else: + logger.info(f"{self.id}: {'started' if self.background else 'ran'} {shown} in env {self.env_id!r}") + return context + message = f"{self.type} {self.id!r}: {shown} {outcome} in env {self.env_id!r}" + if stderr := _tail(record["stderr"].strip(), FAILURE_CHARS): + message += f": {stderr}" + if self.check: + raise RuntimeError(message) + logger.warning(f"{message} (check is off)") + return context + + +def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv: + """The deploy record of the computer in ``env_id`` that serves computer-exec/v1 ``run``: the env's own, or one of a + multi env's computers' (``DeployedGatewayEnv.child_envs``), the one ``computer`` names else its only one.""" + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") + children = {name: DeployedEnv.from_dict(data) for name, data in (getattr(deployed, "child_envs", None) or {}).items()} + if computer is not None: + found = children.get(computer) + if found is None: + raise RuntimeError(f"env {env_id!r} has no computer named {computer!r} (it has {sorted(children)})") + return found + if deployed.supports(COMPUTER_EXEC, "run"): + return deployed + computers = [record for record in children.values() if record.supports(COMPUTER_EXEC, "run")] + if len(computers) != 1: + raise RuntimeError(f"env {env_id!r} has {len(computers) or 'no'} computers serving {COMPUTER_EXEC} run" + + (f": name one as computer, of {sorted(children)}" if computers else "")) + return computers[0] + + +def _computer_problem(env: object, computer: Optional[str]) -> Optional[str]: + if isinstance(env, ComputerEnv): + return None if computer is None else f"env {env.id!r} is a computer env, so it takes no computer name" + if not isinstance(env, MultiEnv): + return f"env {getattr(env, 'id', '?')!r} is no computer env, nor a multi env with one" + browser = env.website_browser().computer + names = [computer_env.environment_name for computer_env in (*env.computer_envs, *([browser] if browser else []))] + if computer is not None and computer not in names: + return f"env {env.id!r} has no computer named {computer!r} (it has {names})" + if computer is None and len(names) != 1: + return (f"env {env.id!r} has {len(names) or 'no'} computers" + + (f": name one as computer, of {names}" if names else "")) + return None + + +def _tail(text: str, chars: int) -> str: + return text if len(text) <= chars else "…" + text[-chars:] diff --git a/tst/integration/computer/computer_exec/envs/exec-desk/env.toml b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml new file mode 100644 index 000000000..e1dca97f8 --- /dev/null +++ b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/computer_exec/tasks/setup.json b/tst/integration/computer/computer_exec/tasks/setup.json new file mode 100644 index 000000000..da776c039 --- /dev/null +++ b/tst/integration/computer/computer_exec/tasks/setup.json @@ -0,0 +1,13 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "exec-desk"}, + {"id": "write", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["env"], + "command": ["sh", "-c", "mkdir -p Desktop && echo computer-exec > Desktop/note.txt && cat Desktop/note.txt"]}, + {"id": "launch", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["write"], + "command": ["thunar", "Desktop"], "background": true}, + {"id": "window", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["launch"], "timeout_s": 60, + "command": "for i in $(seq 100); do xwininfo -root -tree | grep -i -m1 'thunar' && exit 0; sleep 0.5; done; exit 1"}, + {"id": "tolerated", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["window"], + "command": "echo nope >&2; exit 7", "check": false}, + {"id": "missing", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["tolerated"], + "command": ["no-such-app", "Desktop"], "background": true, "check": false} +] diff --git a/tst/integration/computer/computer_exec_test.py b/tst/integration/computer/computer_exec_test.py new file mode 100644 index 000000000..d2a1ce7c4 --- /dev/null +++ b/tst/integration/computer/computer_exec_test.py @@ -0,0 +1,29 @@ +"""computer_exec on a computer env, end to end: the ubuntu-24.04 base with the reference driver deployed, a command run +in the session user's home to its end, its output recorded; Thunar started with ``background`` on the folder that wrote, +as a task opens a file in an app for the agent, and its window then found on the display, so the app outlived the run +that started it; and a failing command, and a background one whose program isn't there, recorded, not failing the +task, with ``check`` off.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from tst.integration.computer.conftest import run_context + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BUNDLE = Path(__file__).parent / "computer_exec" + + +def test_commands_run_in_a_computer_and_an_app_started_in_the_background_stays_up(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "setup", "--no-emulate"]) + + assert ran.exit_code == 0, ran.output + records = run_context(ran.output)["metadata"]["computer_exec"] + assert records["write"]["exit_code"] == 0 and records["write"]["stdout"] == "computer-exec\n", records["write"] + assert records["launch"]["exit_code"] == 0 and records["launch"]["background"], records["launch"] + assert records["window"]["exit_code"] == 0 and "thunar" in records["window"]["stdout"].lower(), records["window"] + assert records["tolerated"]["exit_code"] == 7 and records["tolerated"]["stderr"] == "nope\n", records["tolerated"] + assert records["missing"]["exit_code"] == 127 and "not found" in records["missing"]["stderr"], records["missing"] diff --git a/tst/unit/computer_home.py b/tst/unit/computer_home.py index 5f7d8f086..07ed95c94 100644 --- a/tst/unit/computer_home.py +++ b/tst/unit/computer_home.py @@ -74,8 +74,9 @@ async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, v def _run(self, params: dict) -> dict: try: - done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})}, - capture_output=True, text=True, timeout=params["timeout_s"]) + done = subprocess.run(params["argv"], cwd=self.home / (params.get("cwd") or ""), + env={**os.environ, **(params.get("env") or {})}, capture_output=True, text=True, + timeout=params["timeout_s"]) except subprocess.TimeoutExpired: return {"exit_code": None, "stdout": "", "stderr": "", "truncated": False, "timed_out": True} return {"exit_code": done.returncode, "stdout": done.stdout, "stderr": done.stderr, "truncated": False, diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py new file mode 100644 index 000000000..4e6fd89f1 --- /dev/null +++ b/tst/unit/task_step/computer_exec_test.py @@ -0,0 +1,195 @@ +"""computer_exec against a computer at its driver's boundary (``tst.unit.computer_home``): a command run in its home as +given or through ``sh -c``, in a folder and with variables of its own, its run recorded; a failure or a timeout failing +the step unless ``check`` is off; a ``background`` command started and left running; a multi env's computer found by +name; and the configs it refuses.""" + +from __future__ import annotations + +import dataclasses +import time +from types import SimpleNamespace + +import pytest + +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.multi_env import MultiEnv, WebsiteBrowser +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.computer.computer_exec import RECORDS_KEY, ComputerExecTaskStep +from tst.unit.computer_home import TOKEN, Computer + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + return Computer(tmp_path).install(monkeypatch) + + +def _context(*deployed) -> TaskStepContext: + return TaskStepContext(deployed_agents=[], deployed_envs=list(deployed), metadata={}) + + +def _step(command, **fields) -> ComputerExecTaskStep: + return ComputerExecTaskStep(id="exec", version=None, env_id=fields.pop("env_id", "desk"), command=command, **fields) + + +@pytest.mark.asyncio +async def test_a_command_runs_in_the_home_as_given_with_the_computers_token_and_is_recorded(computer): + ctx = await _step(["sh", "-c", "pwd; echo out; echo err >&2"]).execute(_context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["exit_code"] == 0 and not record["timed_out"] + assert record["stdout"] == f"{computer.home}\nout\n" and record["stderr"] == "err\n" + assert computer.headers == [{"Authorization": f"Bearer {TOKEN}"}] + _, params, _ = computer.calls[0] + assert params["argv"] == ["sh", "-c", "pwd; echo out; echo err >&2"] and params["timeout_s"] == 120.0 + + +@pytest.mark.asyncio +async def test_a_string_runs_through_sh_in_its_folder_with_its_variables(computer): + (computer.home / "work").mkdir() + + ctx = await _step('echo "$GREETING" > made.txt', cwd="work", env={"GREETING": "hi"}, + timeout_s=5).execute(_context(computer.deployed())) + + assert (computer.home / "work" / "made.txt").read_text() == "hi\n" + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"] == ["sh", "-c", 'echo "$GREETING" > made.txt'] + _, params, _ = computer.calls[0] + assert params["cwd"] == "work" and params["env"] == {"GREETING": "hi"} and params["timeout_s"] == 5 + + +@pytest.mark.asyncio +async def test_a_command_that_fails_fails_the_step_with_the_end_of_its_stderr_and_is_still_recorded(computer): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 3 in env 'desk': no such thing"): + await _step("echo 'no such thing' >&2; exit 3").execute(ctx) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 3 + + +@pytest.mark.asyncio +async def test_with_check_off_a_failure_is_recorded_and_the_step_goes_on(computer): + ctx = await _step("exit 4", check=False).execute(_context(computer.deployed())) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 4 + + +@pytest.mark.asyncio +async def test_a_command_that_times_out_fails_the_step(computer): + with pytest.raises(RuntimeError, match=r"timed out after 0.2 s"): + await _step("sleep 5", timeout_s=0.2).execute(_context(computer.deployed())) + + +@pytest.mark.asyncio +async def test_a_background_command_is_started_and_left_running(computer): + started = time.monotonic() + + ctx = await _step("sleep 1; echo done > late.txt", background=True).execute(_context(computer.deployed())) + + assert time.monotonic() - started < 1 + assert not (computer.home / "late.txt").exists() + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", "sleep 1; echo done > late.txt"] + for _ in range(50): + if (computer.home / "late.txt").exists(): + break + time.sleep(0.1) + assert (computer.home / "late.txt").read_text() == "done\n" + + +@pytest.mark.parametrize("command", [["no-such-app-xyz", "a.txt"], ["./missing.sh"]]) +@pytest.mark.asyncio +async def test_a_background_command_whose_program_isnt_there_fails_the_step(computer, command): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 127 in env 'desk': .*not found or not executable"): + await _step(command, background=True).execute(ctx) + + ctx = await _step(command, background=True, check=False).execute(ctx) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 + + +@pytest.mark.asyncio +async def test_a_timeout_above_what_the_computer_allows_is_refused_before_anything_runs(computer): + with pytest.raises(ValueError, match="above the 3600 s"): + await _step("true", timeout_s=4000).execute(_context(computer.deployed())) + assert computer.calls == [] + + +def _multi(computer: Computer, *names: str) -> DeployedGatewayEnv: + return DeployedGatewayEnv(env_id="multi", env_version=1, gateway_url="http://gw", sandbox_id="sb-gw", + environment_card={"name": "multi"}, + child_envs={name: dataclasses.asdict(computer.deployed(env_id=name)) for name in names}) + + +@pytest.mark.asyncio +async def test_a_multi_envs_only_computer_is_found_by_itself_and_one_of_several_by_name(computer): + ctx = await _step("echo one", env_id="multi").execute(_context(_multi(computer, "desk"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "one\n" + + ctx = await _step("echo two", env_id="multi", computer="b").execute(_context(_multi(computer, "a", "b"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "two\n" + + with pytest.raises(RuntimeError, match=r"has 2 computers .*: name one as computer"): + await _step("true", env_id="multi").execute(_context(_multi(computer, "a", "b"))) + with pytest.raises(RuntimeError, match=r"has no computer named 'c'"): + await _step("true", env_id="multi", computer="c").execute(_context(_multi(computer, "a", "b"))) + + +@pytest.mark.asyncio +async def test_an_env_with_no_computer_exec_is_refused(computer): + with pytest.raises(RuntimeError, match="has no computers serving"): + await _step("true").execute(_context(computer.deployed(card={"name": "desk"}))) + + +def _stored_multi(*computers: str, browser: str | None = None) -> MultiEnv: + def computer_env(name): + env = ComputerEnv.__new__(ComputerEnv) + env.environment_name = name + return env + + office = MultiEnv.__new__(MultiEnv) + office.id, office._computers = "office", SimpleNamespace(envs=[computer_env(name) for name in computers]) + office.website_browser = lambda: WebsiteBrowser(computer=computer_env(browser) if browser else None) + return office + + +@pytest.mark.parametrize("computers, browser, named, problem", [ + (("desk",), None, None, None), + ((), "chromium", None, None), + (("desk",), "chromium", "chromium", None), + (("desk",), "chromium", None, "has 2 computers: name one as computer, of ['desk', 'chromium']"), + (("desk",), None, "chromium", "has no computer named 'chromium' (it has ['desk'])"), + ((), None, None, "has no computers"), +], ids=["its-only-computer", "its-browser-computer-alone", "its-browser-computer-by-name", "two-unnamed", + "an-unknown-name", "none"]) +def test_preflight_finds_a_multi_envs_computers_and_its_browser_computer(monkeypatch, computers, browser, named, + problem): + monkeypatch.setattr("agent_env.task_step.task_steps.computer.computer_exec.stored_env", + lambda id: _stored_multi(*computers, browser=browser)) + monkeypatch.setattr(TaskStep, "preflight", lambda self: []) + + problems = _step("true", env_id="office", computer=named).preflight() + + assert problems == ([] if problem is None else [f"step 'exec': env 'office' {problem}"]) + + +@pytest.mark.parametrize("fields, problem", [ + ({"command": []}, "command must be a non-empty list"), + ({"command": " "}, "command must not be blank"), + ({"command": ["ls", 3]}, "command must be a non-empty list"), + ({"command": "true", "cwd": "/tmp"}, "relative to the session user's home"), + ({"command": "true", "env": {"A": 1}}, "env maps variable names to string values"), + ({"command": "true", "timeout_s": 0}, "timeout_s must be a number of seconds above 0"), + ({"command": "true", "background": "yes"}, "background must be true or false"), +]) +def test_bad_configs_are_named(fields, problem): + assert any(problem in p for p in _step(**fields).config_problems()) + + +def test_it_round_trips_through_the_registry(): + step = _step(["open", "x.xlsx"], computer="desk", cwd="Desktop", env={"A": "1"}, timeout_s=30, background=True, + check=False) + again = get_task_step_registry()["computer_exec"].from_dict(step.to_dict()) + assert again.to_dict() == step.to_dict() From 9836edb0ace4dbe5caacb4acba4bbf166a66633a Mon Sep 17 00:00:00 2001 From: William Loo Date: Thu, 19 Nov 2026 10:35:28 -0800 Subject: [PATCH 2/4] fix(task-steps): computer_exec watches a background start and fails one that exits non-zero right after A background command is now watched for START_WINDOW_S (2 s) after it starts: if it exits non-zero in that window, the step fails with its exit status and the end of its stderr. That catches a string command naming a program that isn't there (its program is sh, which passed the old check) and an app given a bad flag or with nowhere to open. An exit of 0 counts as started, since xdg-open and single-instance apps hand off and exit. Failures name the task's command, not the launcher around it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../task_steps/computer/computer_exec.py | 55 +++++++++++++++---- .../computer/computer_exec/tasks/setup.json | 6 +- .../computer/computer_exec_test.py | 6 ++ tst/unit/task_step/computer_exec_test.py | 42 ++++++++++++-- 4 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py index e253b417f..0736387ef 100644 --- a/src/agent_env/task_step/task_steps/computer/computer_exec.py +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -28,11 +28,34 @@ #: The most of each stream the record keeps, and the most of stderr a failure says: the end of it. RECORDED_CHARS = 4000 FAILURE_CHARS = 300 -#: How a ``background`` command starts: its program found and executable, else it fails as the shell does with 127, -#: then detached from the run's pipes, so the run returns as the shell does. -_BACKGROUND = ('case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' - ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' - '"$@" /dev/null 2>&1 &') +#: How long a ``background`` app is watched after it starts: one that exits non-zero within it fails the step. +START_WINDOW_S = 2.0 +#: How much longer than its start window a ``background`` start may take, for the shell and the stderr it reads. +START_MARGIN_S = 3.0 +#: The most of a failed start's stderr the launcher keeps: the end of it. +START_STDERR_BYTES = 4096 + + +def background_script() -> str: + """How a ``background`` command starts, as ``sh -c`` runs it with the command as its arguments: its program found + and executable, else it fails as the shell does with 127; then started detached from the run's pipes, its stdout + discarded and the end of its stderr kept, and watched for ``START_WINDOW_S``. One that exits non-zero in that window + fails with its exit status and its stderr, which also catches a string command (whose program is ``sh``) naming a + program that isn't there; one that exits 0, as ``xdg-open`` and single-instance apps do once they hand off, or is + still running, has started.""" + ticks = max(1, round(START_WINDOW_S * 10)) + return ( + 'case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' + ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' + 'd=$(mktemp -d) || exit 1; ' + f'( {{ "$@" 2>&1 >/dev/null; echo $? > "$d/rc"; }} | tail -c {START_STDERR_BYTES} > "$d/err" ) ' + '/dev/null 2>&1 & ' + f'n=0; while [ ! -s "$d/rc" ] && [ $n -lt {ticks} ]; do sleep 0.1; n=$((n+1)); done; ' + '[ -s "$d/rc" ] || exit 0; ' + 'rc=$(cat "$d/rc"); [ "$rc" = 0 ] && { rm -rf "$d"; exit 0; }; ' + 'n=0; while [ ! -s "$d/err" ] && [ $n -lt 10 ]; do sleep 0.1; n=$((n+1)); done; ' + 'echo "computer_exec: $* exited $rc right after starting" >&2; cat "$d/err" >&2; rm -rf "$d"; exit "$rc"') + Command = Union[list[str], str] @@ -46,8 +69,9 @@ class ComputerExecTaskStep(TaskStep): The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless ``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app - launched for the agent is: the step waits for the start alone, and fails as a run does when its program isn't - there to start.""" + launched for the agent is: the step watches it for ``START_WINDOW_S`` and fails as a run does when its program + isn't there to start, or when it exits non-zero in that window, with the end of its stderr; an exit of 0 counts as + started, since ``xdg-open`` and single-instance apps hand off to another process and exit.""" type: ClassVar[str] = "computer_exec" entity_refs = (EntityRef.env("env_id"),) @@ -87,6 +111,10 @@ def config_problems(self) -> list[str]: for name in ("background", "check"): if not isinstance(getattr(self, name), bool): problems.append(f"step {self.id!r}: {name} must be true or false") + if (self.background is True and isinstance(self.timeout_s, (int, float)) + and self.timeout_s < (least := START_WINDOW_S + START_MARGIN_S)): + problems.append(f"step {self.id!r}: a background start is watched for {START_WINDOW_S:g} s, so its " + f"timeout_s must be at least {least:g}") return problems def preflight(self) -> list[str]: @@ -110,11 +138,14 @@ def from_dict(cls, data: dict) -> ComputerExecTaskStep: timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False), check=data.get("check", True)) + def command_argv(self) -> list[str]: + """The command as a program and its arguments: through ``sh -c`` when it's a string.""" + return ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command) + def argv(self) -> list[str]: - """What the driver runs: the command, through ``sh -c`` when it's a string, started detached with - ``background``.""" - argv = ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command) - return ["sh", "-c", _BACKGROUND, "computer_exec", *argv] if self.background else argv + """What the driver runs: the command, started by ``background_script`` with ``background``.""" + argv = self.command_argv() + return ["sh", "-c", background_script(), "computer_exec", *argv] if self.background else argv async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = find_computer(context, self.env_id, self.computer) @@ -131,7 +162,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS), "stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)} context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record - shown = shlex.join(argv) + shown = shlex.join(self.command_argv()) # the task's command; the record keeps the launcher around it if record["timed_out"]: outcome = f"timed out after {self.timeout_s} s" elif record["exit_code"] != 0: diff --git a/tst/integration/computer/computer_exec/tasks/setup.json b/tst/integration/computer/computer_exec/tasks/setup.json index da776c039..e6ca033ea 100644 --- a/tst/integration/computer/computer_exec/tasks/setup.json +++ b/tst/integration/computer/computer_exec/tasks/setup.json @@ -9,5 +9,9 @@ {"id": "tolerated", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["window"], "command": "echo nope >&2; exit 7", "check": false}, {"id": "missing", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["tolerated"], - "command": ["no-such-app", "Desktop"], "background": true, "check": false} + "command": ["no-such-app", "Desktop"], "background": true, "check": false}, + {"id": "missing-string", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing"], + "command": "thunarr Desktop", "background": true, "check": false}, + {"id": "bad-flag", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing-string"], + "command": ["ls", "--no-such-flag"], "background": true, "check": false} ] diff --git a/tst/integration/computer/computer_exec_test.py b/tst/integration/computer/computer_exec_test.py index d2a1ce7c4..3b437c939 100644 --- a/tst/integration/computer/computer_exec_test.py +++ b/tst/integration/computer/computer_exec_test.py @@ -27,3 +27,9 @@ def test_commands_run_in_a_computer_and_an_app_started_in_the_background_stays_u assert records["window"]["exit_code"] == 0 and "thunar" in records["window"]["stdout"].lower(), records["window"] assert records["tolerated"]["exit_code"] == 7 and records["tolerated"]["stderr"] == "nope\n", records["tolerated"] assert records["missing"]["exit_code"] == 127 and "not found" in records["missing"]["stderr"], records["missing"] + # A string command's program is sh, so it's caught when sh exits right after starting; so is a program given a bad + # flag. (A second thunar would exit 0: it hands off to the one running, which counts as started.) + assert records["missing-string"]["exit_code"] == 127, records["missing-string"] + assert "right after starting" in records["missing-string"]["stderr"], records["missing-string"] + assert records["bad-flag"]["exit_code"] not in (0, None), records["bad-flag"] + assert "right after starting" in records["bad-flag"]["stderr"], records["bad-flag"] diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py index 4e6fd89f1..d5d3b3180 100644 --- a/tst/unit/task_step/computer_exec_test.py +++ b/tst/unit/task_step/computer_exec_test.py @@ -1,6 +1,7 @@ """computer_exec against a computer at its driver's boundary (``tst.unit.computer_home``): a command run in its home as given or through ``sh -c``, in a folder and with variables of its own, its run recorded; a failure or a timeout failing -the step unless ``check`` is off; a ``background`` command started and left running; a multi env's computer found by +the step unless ``check`` is off; a ``background`` command started and left running, and failing when its program +isn't there or it exits non-zero right after starting, a string command's included; a multi env's computer found by name; and the configs it refuses.""" from __future__ import annotations @@ -17,6 +18,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.computer import computer_exec from agent_env.task_step.task_steps.computer.computer_exec import RECORDS_KEY, ComputerExecTaskStep from tst.unit.computer_home import TOKEN, Computer @@ -82,22 +84,49 @@ async def test_a_command_that_times_out_fails_the_step(computer): await _step("sleep 5", timeout_s=0.2).execute(_context(computer.deployed())) +@pytest.fixture +def short_start(monkeypatch): + monkeypatch.setattr(computer_exec, "START_WINDOW_S", 0.3) + + @pytest.mark.asyncio -async def test_a_background_command_is_started_and_left_running(computer): +async def test_a_background_command_is_started_and_left_running(computer, short_start): started = time.monotonic() - ctx = await _step("sleep 1; echo done > late.txt", background=True).execute(_context(computer.deployed())) + ctx = await _step("sleep 3; echo done > late.txt", background=True).execute(_context(computer.deployed())) - assert time.monotonic() - started < 1 + assert time.monotonic() - started < 2.5 assert not (computer.home / "late.txt").exists() - assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", "sleep 1; echo done > late.txt"] - for _ in range(50): + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", "sleep 3; echo done > late.txt"] + for _ in range(60): if (computer.home / "late.txt").exists(): break time.sleep(0.1) assert (computer.home / "late.txt").read_text() == "done\n" +@pytest.mark.asyncio +@pytest.mark.parametrize("command, exit_code, stderr", [ + ("thunarr-xyz Desktop", 127, "not found"), + (["sh", "-c", "echo 'unknown option --bad' >&2; exit 2"], 2, "unknown option --bad"), +]) +async def test_a_background_app_that_exits_non_zero_right_after_starting_fails_with_its_stderr( + computer, short_start, command, exit_code, stderr): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=rf"(?s)^computer_exec 'exec': sh -c .{{0,80}} exited {exit_code} in env " + rf"'desk': .*right after starting.*{stderr}"): + await _step(command, background=True).execute(ctx) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == exit_code + + +@pytest.mark.asyncio +async def test_a_background_app_that_hands_off_and_exits_0_has_started(computer, short_start): + ctx = await _step("echo handed off >&2; exit 0", background=True).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 0 + + @pytest.mark.parametrize("command", [["no-such-app-xyz", "a.txt"], ["./missing.sh"]]) @pytest.mark.asyncio async def test_a_background_command_whose_program_isnt_there_fails_the_step(computer, command): @@ -183,6 +212,7 @@ def test_preflight_finds_a_multi_envs_computers_and_its_browser_computer(monkeyp ({"command": "true", "env": {"A": 1}}, "env maps variable names to string values"), ({"command": "true", "timeout_s": 0}, "timeout_s must be a number of seconds above 0"), ({"command": "true", "background": "yes"}, "background must be true or false"), + ({"command": "true", "background": True, "timeout_s": 4}, "timeout_s must be at least 5"), ]) def test_bad_configs_are_named(fields, problem): assert any(problem in p for p in _step(**fields).config_problems()) From c716164e6b191252b9140f67934b57ba54b634e9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 10:45:07 -0800 Subject: [PATCH 3/4] feat(task-steps): computer_exec runs a command as root with scope system scope: "system" runs the command as root through the computer's sandbox (RootChannel, docker exec -u 0), as site routing and the clock follower's config already do, in the session user's folder and HOME, under timeout; it isn't offered with background, nor on a computer booted from a disk, which core can't reach as root. OSWorld's setups that sudo (apt-get installs, a mkdir, an su) need it. RootChannel gains exec, which answers the exit code and output instead of raising. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../env/envs/computer/utils/sites.py | 18 ++++- .../task_steps/computer/computer_exec.py | 62 +++++++++++++-- .../computer/computer_exec/tasks/setup.json | 8 +- .../computer/computer_exec_test.py | 8 +- tst/unit/task_step/computer_exec_test.py | 78 ++++++++++++++++++- 6 files changed, 159 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d4c9b7aa..df89ddfa8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, never with `background` and not on a computer booted from a disk; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index b24fc67f8..f18b25eec 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -418,13 +418,25 @@ async def run(self, argv: list[str]) -> str: if isinstance(self.sandbox, VmSandbox): return await self.sandbox.exec_script(f"docker exec -u 0 {shlex.quote(self.sandbox.container_name)} " f"{shlex.join(argv)}") + await self._check_root() + return await self._exec(argv) + + async def exec(self, argv: list[str]) -> tuple[int, str, str]: + """``argv`` run as root: its exit code, stdout and stderr, whatever the code.""" + if isinstance(self.sandbox, VmSandbox): + return await self.sandbox.exec_with_output( + "sudo", "bash", "-c", f"docker exec -u 0 {shlex.quote(self.sandbox.container_name)} {shlex.join(argv)}") + await self._check_root() + return await self.sandbox.exec_with_output(*argv) + + async def _check_root(self) -> None: if not self.checked: uid = (await self._exec(["id", "-u"])).strip() if uid != "0": - raise RuntimeError(f"site routing installs as root, and the {getattr(self.sandbox, 'type', 'given')} " - f"sandbox runs the computer's commands as uid {uid}; deploy it on a VM sandbox") + raise RuntimeError(f"root in the computer runs through its sandbox, and the " + f"{getattr(self.sandbox, 'type', 'given')} sandbox runs the computer's commands as " + f"uid {uid}; deploy it on a VM sandbox") self.checked = True - return await self._exec(argv) async def _exec(self, argv: list[str]) -> str: code, stdout, stderr = await self.sandbox.exec_with_output(*argv) diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py index 0736387ef..7d87c70b0 100644 --- a/src/agent_env/task_step/task_steps/computer/computer_exec.py +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -1,10 +1,13 @@ """computer_exec: a command run inside a deployed computer as its session user, through its driver's computer-exec/v1 -``run``, a control route the agent can't call, as a task sets a computer up: a program run to its end, its exit status -and output recorded, or with ``background`` an app started and left running.""" +``run``, a control route the agent can't call, or with ``scope: "system"`` as root, through the computer's sandbox, as a +task sets a computer up: a program run to its end, its exit status and output recorded, or with ``background`` an app +started and left running.""" from __future__ import annotations +import asyncio import logging +import posixpath import shlex from typing import ClassVar, Optional, Union @@ -14,7 +17,9 @@ from agent_env.env.env import DeployedEnv, DeployedSandboxEnv from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils import inside +from agent_env.env.envs.computer.utils.sites import RootChannel from agent_env.env.envs.multi_env import MultiEnv +from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider from agent_env.task_step.context import TaskStepContext from agent_env.task_step.preflight import stored_env from agent_env.task_step.task_step import TaskStep, TaskStepDependency @@ -34,6 +39,16 @@ START_MARGIN_S = 3.0 #: The most of a failed start's stderr the launcher keeps: the end of it. START_STDERR_BYTES = 4096 +#: Who a command runs as: the session user, through the driver, or for ``system``, root, through the sandbox. +SCOPES = ("session", "system") +#: How long the session user's home takes to find, which a ``system`` command runs in. +HOME_TIMEOUT_S = 30.0 +#: How much longer than ``timeout_s`` a ``system`` command's exec may take before the step stops waiting for it. +SYSTEM_MARGIN_S = 30.0 +#: How long ``timeout`` gives a ``system`` command after its stop signal before it kills it. +SYSTEM_KILL_AFTER_S = 5 +#: What ``timeout`` exits with when it stopped the command. +TIMED_OUT_EXIT = 124 def background_script() -> str: @@ -66,6 +81,9 @@ class ComputerExecTaskStep(TaskStep): its arguments, or a string run by ``sh -c``. For a multi env, ``computer`` names the computer, by its environment name, and may be left out when the env has one. + With ``scope: "system"`` it runs as root instead, through the computer's sandbox rather than its driver, in the + same folder and with the session user's ``HOME``; a computer booted from a disk has no such route. + The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless ``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app @@ -79,7 +97,8 @@ class ComputerExecTaskStep(TaskStep): def __init__(self, id: str, version: Optional[int], env_id: str, command: Command, computer: Optional[str] = None, cwd: Optional[str] = None, env: Optional[dict[str, str]] = None, timeout_s: float = DEFAULT_TIMEOUT_S, background: bool = False, check: bool = True, - depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): + scope: str = "session", depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) self.env_id = env_id self.command = command @@ -89,6 +108,7 @@ def __init__(self, id: str, version: Optional[int], env_id: str, command: Comman self.timeout_s = timeout_s self.background = background self.check = check + self.scope = scope def config_problems(self) -> list[str]: problems = [] @@ -115,6 +135,12 @@ def config_problems(self) -> list[str]: and self.timeout_s < (least := START_WINDOW_S + START_MARGIN_S)): problems.append(f"step {self.id!r}: a background start is watched for {START_WINDOW_S:g} s, so its " f"timeout_s must be at least {least:g}") + if self.scope not in SCOPES: + problems.append(f"step {self.id!r}: scope is 'session', the session user's, or 'system', root's, " + f"not {self.scope!r}") + elif self.scope == "system" and self.background is True: + problems.append(f"step {self.id!r}: background runs as the session user; a 'system' command runs to its " + "end") return problems def preflight(self) -> list[str]: @@ -129,14 +155,14 @@ def preflight(self) -> list[str]: def to_dict(self) -> dict: return {**super().to_dict(), "env_id": self.env_id, "command": self.command, "computer": self.computer, "cwd": self.cwd, "env": self.env, "timeout_s": self.timeout_s, "background": self.background, - "check": self.check} + "check": self.check, "scope": self.scope} @classmethod def from_dict(cls, data: dict) -> ComputerExecTaskStep: return cls(**cls._base_from_dict(data), env_id=data["env_id"], command=data["command"], computer=data.get("computer"), cwd=data.get("cwd"), env=data.get("env"), timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False), - check=data.get("check", True)) + check=data.get("check", True), scope=data.get("scope", "session")) def command_argv(self) -> list[str]: """The command as a program and its arguments: through ``sh -c`` when it's a string.""" @@ -155,14 +181,18 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer " f"of env {self.env_id!r} lets a command run") argv = self.argv() - answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd) - record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "background": self.background, + if self.scope == "system": + answer = await self._as_root(deployed, argv) + else: + answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd) + record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "scope": self.scope, + "background": self.background, "exit_code": answer.get("exit_code"), "timed_out": bool(answer.get("timed_out")), "truncated": bool(answer.get("truncated")), "stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS), "stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)} context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record - shown = shlex.join(self.command_argv()) # the task's command; the record keeps the launcher around it + shown = shlex.join(self.command_argv()) + (" as root" if self.scope == "system" else "") if record["timed_out"]: outcome = f"timed out after {self.timeout_s} s" elif record["exit_code"] != 0: @@ -178,6 +208,22 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning(f"{message} (check is off)") return context + async def _as_root(self, deployed: DeployedSandboxEnv, argv: list[str]) -> dict: + """``argv`` run as root through the computer's sandbox (``RootChannel``), in the folder and with the ``HOME`` a + session run has, so ``cwd`` and ``~`` name the same files, under ``timeout``; answered as ``run`` answers.""" + if (deployed.metadata.get("computer") or {}).get("form") == "disk": + raise RuntimeError(f"{self.type} {self.id!r}: a 'system' command runs through the computer's container, " + f"and env {self.env_id!r}'s computer boots from a disk, which core can't reach as root") + home = (await inside.run(deployed, ["pwd"], timeout_s=HOME_TIMEOUT_S))["stdout"].strip() + variables = [f"{name}={value}" for name, value in (self.env or {}).items()] + command = ["timeout", "-k", str(SYSTEM_KILL_AFTER_S), f"{self.timeout_s:g}", "env", "-C", + posixpath.join(home, self.cwd) if self.cwd else home, f"HOME={home}", *variables, *argv] + sandbox = await build_sandbox_provider(deployed.sandbox_type).get_sandbox(deployed.sandbox_id) + code, stdout, stderr = await asyncio.wait_for(RootChannel(sandbox).exec(command), + self.timeout_s + SYSTEM_MARGIN_S) + return {"exit_code": None if code == TIMED_OUT_EXIT else code, "timed_out": code == TIMED_OUT_EXIT, + "stdout": stdout, "stderr": stderr} + def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv: """The deploy record of the computer in ``env_id`` that serves computer-exec/v1 ``run``: the env's own, or one of a diff --git a/tst/integration/computer/computer_exec/tasks/setup.json b/tst/integration/computer/computer_exec/tasks/setup.json index e6ca033ea..bfa6551e8 100644 --- a/tst/integration/computer/computer_exec/tasks/setup.json +++ b/tst/integration/computer/computer_exec/tasks/setup.json @@ -13,5 +13,11 @@ {"id": "missing-string", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing"], "command": "thunarr Desktop", "background": true, "check": false}, {"id": "bad-flag", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing-string"], - "command": ["ls", "--no-such-flag"], "background": true, "check": false} + "command": ["ls", "--no-such-flag"], "background": true, "check": false}, + {"id": "root", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["bad-flag"], "scope": "system", + "command": "id -u; pwd; echo ~; touch /etc/computer-exec-root"}, + {"id": "owner", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["root"], + "command": ["stat", "-c", "%U", "/etc/computer-exec-root"]}, + {"id": "denied", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["owner"], + "command": "touch /etc/computer-exec-denied", "check": false} ] diff --git a/tst/integration/computer/computer_exec_test.py b/tst/integration/computer/computer_exec_test.py index 3b437c939..561f4ba1b 100644 --- a/tst/integration/computer/computer_exec_test.py +++ b/tst/integration/computer/computer_exec_test.py @@ -1,8 +1,8 @@ """computer_exec on a computer env, end to end: the ubuntu-24.04 base with the reference driver deployed, a command run in the session user's home to its end, its output recorded; Thunar started with ``background`` on the folder that wrote, as a task opens a file in an app for the agent, and its window then found on the display, so the app outlived the run -that started it; and a failing command, and a background one whose program isn't there, recorded, not failing the -task, with ``check`` off.""" +that started it; a failing command, and a background one whose program isn't there, recorded, not failing the task, +with ``check`` off; and a ``system`` command run as root in the session user's home, which the session user can't do.""" from pathlib import Path @@ -33,3 +33,7 @@ def test_commands_run_in_a_computer_and_an_app_started_in_the_background_stays_u assert "right after starting" in records["missing-string"]["stderr"], records["missing-string"] assert records["bad-flag"]["exit_code"] not in (0, None), records["bad-flag"] assert "right after starting" in records["bad-flag"]["stderr"], records["bad-flag"] + uid, home, tilde = records["root"]["stdout"].split() + assert records["root"]["exit_code"] == 0 and uid == "0" and home == tilde != "/root", records["root"] + assert records["owner"]["stdout"] == "root\n", records["owner"] + assert records["denied"]["exit_code"] != 0 and "Permission denied" in records["denied"]["stderr"], records["denied"] diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py index d5d3b3180..9a5033167 100644 --- a/tst/unit/task_step/computer_exec_test.py +++ b/tst/unit/task_step/computer_exec_test.py @@ -1,12 +1,15 @@ """computer_exec against a computer at its driver's boundary (``tst.unit.computer_home``): a command run in its home as given or through ``sh -c``, in a folder and with variables of its own, its run recorded; a failure or a timeout failing the step unless ``check`` is off; a ``background`` command started and left running, and failing when its program -isn't there or it exits non-zero right after starting, a string command's included; a multi env's computer found by -name; and the configs it refuses.""" +isn't there or it exits non-zero right after starting, a string command's included; a ``system`` command run as root +through the computer's sandbox, in the session user's folder; a multi env's computer found by name; and the configs it +refuses.""" from __future__ import annotations +import asyncio import dataclasses +import subprocess import time from types import SimpleNamespace @@ -36,6 +39,35 @@ def _step(command, **fields) -> ComputerExecTaskStep: return ComputerExecTaskStep(id="exec", version=None, env_id=fields.pop("env_id", "desk"), command=command, **fields) +class _Sandbox: + """A computer's sandbox whose exec is ``uid``'s, running what it's given on this machine.""" + + type = "fake" + + def __init__(self): + self.uid = "0" + self.calls: list[list[str]] = [] + + async def exec_with_output(self, *argv): + self.calls.append(list(argv)) + if list(argv) == ["id", "-u"]: + return 0, f"{self.uid}\n", "" + done = await asyncio.to_thread(subprocess.run, list(argv), capture_output=True, text=True) + return done.returncode, done.stdout, done.stderr + + +@pytest.fixture +def sandbox(monkeypatch): + held = _Sandbox() + + async def get_sandbox(sandbox_id): + return held + + monkeypatch.setattr(computer_exec, "build_sandbox_provider", + lambda sandbox_type: SimpleNamespace(get_sandbox=get_sandbox)) + return held + + @pytest.mark.asyncio async def test_a_command_runs_in_the_home_as_given_with_the_computers_token_and_is_recorded(computer): ctx = await _step(["sh", "-c", "pwd; echo out; echo err >&2"]).execute(_context(computer.deployed())) @@ -139,6 +171,44 @@ async def test_a_background_command_whose_program_isnt_there_fails_the_step(comp assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 +@pytest.mark.asyncio +async def test_a_system_command_runs_as_root_through_the_sandbox_in_the_session_users_folder_and_home( + computer, sandbox): + (computer.home / "work").mkdir() + + ctx = await _step('pwd; echo "$HOME $GREETING"', scope="system", cwd="work", env={"GREETING": "hi"}).execute( + _context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["exit_code"] == 0 and record["scope"] == "system" + assert record["stdout"] == f"{computer.home}/work\n{computer.home} hi\n" + assert sandbox.calls[-1][:4] == ["timeout", "-k", "5", "120"] + assert [params["argv"] for _, params, _ in computer.calls] == [["pwd"]] + + +@pytest.mark.asyncio +async def test_a_system_command_that_times_out_or_cant_start_fails_unless_check_is_off_and_is_recorded( + computer, sandbox): + with pytest.raises(RuntimeError, match=r"sh -c 'sleep 5' as root timed out after 0.5 s"): + await _step("sleep 5", scope="system", timeout_s=0.5).execute(_context(computer.deployed())) + + ctx = await _step(["no-such-app-xyz"], scope="system", check=False).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 + ctx = await _step("true", scope="system", cwd="missing", check=False).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 125 + + +@pytest.mark.asyncio +async def test_a_system_command_is_refused_on_a_disk_computer_and_on_a_sandbox_whose_exec_isnt_roots( + computer, sandbox): + with pytest.raises(RuntimeError, match="boots from a disk"): + await _step("true", scope="system").execute(_context(computer.deployed(form="disk"))) + + sandbox.uid = "1000" + with pytest.raises(RuntimeError, match="as uid 1000; deploy it on a VM sandbox"): + await _step("true", scope="system").execute(_context(computer.deployed())) + + @pytest.mark.asyncio async def test_a_timeout_above_what_the_computer_allows_is_refused_before_anything_runs(computer): with pytest.raises(ValueError, match="above the 3600 s"): @@ -213,6 +283,8 @@ def test_preflight_finds_a_multi_envs_computers_and_its_browser_computer(monkeyp ({"command": "true", "timeout_s": 0}, "timeout_s must be a number of seconds above 0"), ({"command": "true", "background": "yes"}, "background must be true or false"), ({"command": "true", "background": True, "timeout_s": 4}, "timeout_s must be at least 5"), + ({"command": "true", "scope": "root"}, "scope is 'session', the session user's, or 'system', root's"), + ({"command": "true", "scope": "system", "background": True}, "a 'system' command runs to its end"), ]) def test_bad_configs_are_named(fields, problem): assert any(problem in p for p in _step(**fields).config_problems()) @@ -223,3 +295,5 @@ def test_it_round_trips_through_the_registry(): check=False) again = get_task_step_registry()["computer_exec"].from_dict(step.to_dict()) assert again.to_dict() == step.to_dict() + root = _step("apt-get install -y jq", scope="system") + assert get_task_step_registry()["computer_exec"].from_dict(root.to_dict()).scope == "system" From 20445f23703c307b8e70b80b0428107bfaa219b4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 11:05:13 -0800 Subject: [PATCH 4/4] fix(task-steps): computer_exec caps root output, times out by its deadline, refuses sh on Windows, leaves no temp files - A system command's streams are capped in the computer (tail -c, SYSTEM_OUTPUT_BYTES) and base64'd, so a command that prints without end can't fill the task runner's memory and any bytes decode; the record says when it was cut. - It timed out when timeout stopped or killed it past the deadline (124, or 137 for one that ignored the stop signal), not when it exits with that status of its own, earlier. - On a Windows computer (its card's computer/v1 platform), a string or background command, which needs sh, is refused before anything runs. - A background start still running when its window ends removes the launcher's folder; the app keeps the stderr file it writes open. The launcher's polling interval and stderr wait are named, and its loops derive from them. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../task_steps/computer/computer_exec.py | 72 +++++++++++++++---- tst/unit/task_step/computer_exec_test.py | 61 +++++++++++++++- 3 files changed, 119 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index df89ddfa8..04ed36bc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, never with `background` and not on a computer booted from a disk; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py index 7d87c70b0..6e828eb70 100644 --- a/src/agent_env/task_step/task_steps/computer/computer_exec.py +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -6,12 +6,16 @@ from __future__ import annotations import asyncio +import base64 +import binascii import logging import posixpath import shlex +import time from typing import ClassVar, Optional, Union -from agentenv_protocol.computer import COMPUTER_EXEC +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER, COMPUTER_EXEC from agent_env.entity.refs import EntityRef from agent_env.env.env import DeployedEnv, DeployedSandboxEnv @@ -39,6 +43,9 @@ START_MARGIN_S = 3.0 #: The most of a failed start's stderr the launcher keeps: the end of it. START_STDERR_BYTES = 4096 +#: How often the launcher checks a ``background`` start, and how long it waits for a failed start's stderr. +START_POLL_S = 0.1 +START_STDERR_WAIT_S = 1.0 #: Who a command runs as: the session user, through the driver, or for ``system``, root, through the sandbox. SCOPES = ("session", "system") #: How long the session user's home takes to find, which a ``system`` command runs in. @@ -47,8 +54,22 @@ SYSTEM_MARGIN_S = 30.0 #: How long ``timeout`` gives a ``system`` command after its stop signal before it kills it. SYSTEM_KILL_AFTER_S = 5 -#: What ``timeout`` exits with when it stopped the command. +#: What ``timeout`` exits with when it stopped the command, and when it had to kill it (128 + SIGKILL). TIMED_OUT_EXIT = 124 +KILLED_EXIT = 137 +#: The most of each stream a ``system`` command keeps in the computer, the end of it, so a command that prints without +#: end can't fill the task runner's memory; the record keeps less still. +SYSTEM_OUTPUT_BYTES = 65536 +#: How a ``system`` command runs, given that cap and then the command: the end of each stream kept and base64'd, so +#: whatever bytes it printed decode, and the command's exit status passed on as its own. +_CAPPED = """cap=$1; shift +d=$(mktemp -d) || exit 125 +{ { "$@"; echo $? >"$d/rc"; } 2>&1 >&3 3>&- | tail -c "$cap" >"$d/err"; } 3>&1 | tail -c "$cap" | base64 -w0 +base64 -w0 "$d/err" >&2 +rc=$(cat "$d/rc" 2>/dev/null) +rm -rf "$d" +exit "${rc:-125}" +""" def background_script() -> str: @@ -57,18 +78,20 @@ def background_script() -> str: discarded and the end of its stderr kept, and watched for ``START_WINDOW_S``. One that exits non-zero in that window fails with its exit status and its stderr, which also catches a string command (whose program is ``sh``) naming a program that isn't there; one that exits 0, as ``xdg-open`` and single-instance apps do once they hand off, or is - still running, has started.""" - ticks = max(1, round(START_WINDOW_S * 10)) + still running, has started. Its folder goes either way: an app still running keeps the stderr file it writes open, + so removing the folder takes nothing from it.""" + window = max(1, round(START_WINDOW_S / START_POLL_S)) + stderr_wait = max(1, round(START_STDERR_WAIT_S / START_POLL_S)) return ( 'case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' 'd=$(mktemp -d) || exit 1; ' f'( {{ "$@" 2>&1 >/dev/null; echo $? > "$d/rc"; }} | tail -c {START_STDERR_BYTES} > "$d/err" ) ' '/dev/null 2>&1 & ' - f'n=0; while [ ! -s "$d/rc" ] && [ $n -lt {ticks} ]; do sleep 0.1; n=$((n+1)); done; ' - '[ -s "$d/rc" ] || exit 0; ' + f'n=0; while [ ! -s "$d/rc" ] && [ $n -lt {window} ]; do sleep {START_POLL_S:g}; n=$((n+1)); done; ' + '[ -s "$d/rc" ] || { rm -rf "$d"; exit 0; }; ' 'rc=$(cat "$d/rc"); [ "$rc" = 0 ] && { rm -rf "$d"; exit 0; }; ' - 'n=0; while [ ! -s "$d/err" ] && [ $n -lt 10 ]; do sleep 0.1; n=$((n+1)); done; ' + f'n=0; while [ ! -s "$d/err" ] && [ $n -lt {stderr_wait} ]; do sleep {START_POLL_S:g}; n=$((n+1)); done; ' 'echo "computer_exec: $* exited $rc right after starting" >&2; cat "$d/err" >&2; rm -rf "$d"; exit "$rc"') @@ -180,6 +203,10 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if self.timeout_s > (limit := inside.max_timeout_s(deployed)): raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer " f"of env {self.env_id!r} lets a command run") + if (self.background or isinstance(self.command, str)) and _os(deployed) == "windows": + raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer runs Windows, which has no sh " + "to run a string or start an app in the background: give the command as a program and " + "its arguments, without background") argv = self.argv() if self.scope == "system": answer = await self._as_root(deployed, argv) @@ -210,19 +237,26 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: async def _as_root(self, deployed: DeployedSandboxEnv, argv: list[str]) -> dict: """``argv`` run as root through the computer's sandbox (``RootChannel``), in the folder and with the ``HOME`` a - session run has, so ``cwd`` and ``~`` name the same files, under ``timeout``; answered as ``run`` answers.""" + session run has, so ``cwd`` and ``~`` name the same files, under ``timeout``, its output capped in the computer + (``_CAPPED``); answered as ``run`` answers. It timed out when ``timeout`` stopped or killed it past the + deadline, which a command exiting with that status of its own, earlier, didn't.""" if (deployed.metadata.get("computer") or {}).get("form") == "disk": raise RuntimeError(f"{self.type} {self.id!r}: a 'system' command runs through the computer's container, " f"and env {self.env_id!r}'s computer boots from a disk, which core can't reach as root") home = (await inside.run(deployed, ["pwd"], timeout_s=HOME_TIMEOUT_S))["stdout"].strip() variables = [f"{name}={value}" for name, value in (self.env or {}).items()] - command = ["timeout", "-k", str(SYSTEM_KILL_AFTER_S), f"{self.timeout_s:g}", "env", "-C", + command = ["sh", "-c", _CAPPED, "computer_exec", str(SYSTEM_OUTPUT_BYTES), + "timeout", "-k", str(SYSTEM_KILL_AFTER_S), f"{self.timeout_s:g}", "env", "-C", posixpath.join(home, self.cwd) if self.cwd else home, f"HOME={home}", *variables, *argv] sandbox = await build_sandbox_provider(deployed.sandbox_type).get_sandbox(deployed.sandbox_id) + started = time.monotonic() code, stdout, stderr = await asyncio.wait_for(RootChannel(sandbox).exec(command), self.timeout_s + SYSTEM_MARGIN_S) - return {"exit_code": None if code == TIMED_OUT_EXIT else code, "timed_out": code == TIMED_OUT_EXIT, - "stdout": stdout, "stderr": stderr} + timed_out = code in (TIMED_OUT_EXIT, KILLED_EXIT) and time.monotonic() - started >= self.timeout_s + out, err = _unpacked(stdout), _unpacked(stderr) + return {"exit_code": None if timed_out else code, "timed_out": timed_out, + "truncated": max(len(out), len(err)) >= SYSTEM_OUTPUT_BYTES, + "stdout": out.decode("utf-8", errors="replace"), "stderr": err.decode("utf-8", errors="replace")} def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv: @@ -231,7 +265,8 @@ def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str] deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) if deployed is None: raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") - children = {name: DeployedEnv.from_dict(data) for name, data in (getattr(deployed, "child_envs", None) or {}).items()} + children = {name: DeployedEnv.from_dict(data) + for name, data in (getattr(deployed, "child_envs", None) or {}).items()} if computer is not None: found = children.get(computer) if found is None: @@ -261,5 +296,18 @@ def _computer_problem(env: object, computer: Optional[str]) -> Optional[str]: return None +def _os(deployed: DeployedSandboxEnv) -> Optional[str]: + """The computer's OS, as its card's computer/v1 platform says.""" + return (protocol_v1.extension_params(deployed.environment_card or {}, COMPUTER).get("platform") or {}).get("os") + + +def _unpacked(text: str) -> bytes: + """A stream ``_CAPPED`` base64'd, as bytes; what the sandbox said in its place, as it said it.""" + try: + return base64.b64decode(text.strip(), validate=True) + except (binascii.Error, ValueError): + return text.encode() + + def _tail(text: str, chars: int) -> str: return text if len(text) <= chars else "…" + text[-chars:] diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py index 9a5033167..872022071 100644 --- a/tst/unit/task_step/computer_exec_test.py +++ b/tst/unit/task_step/computer_exec_test.py @@ -14,6 +14,7 @@ from types import SimpleNamespace import pytest +from agentenv_protocol.computer import COMPUTER from agent_env.env.env import DeployedGatewayEnv from agent_env.env.envs.computer.env import ComputerEnv @@ -23,7 +24,7 @@ from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.computer import computer_exec from agent_env.task_step.task_steps.computer.computer_exec import RECORDS_KEY, ComputerExecTaskStep -from tst.unit.computer_home import TOKEN, Computer +from tst.unit.computer_home import CARD, TOKEN, Computer @pytest.fixture @@ -129,7 +130,8 @@ async def test_a_background_command_is_started_and_left_running(computer, short_ assert time.monotonic() - started < 2.5 assert not (computer.home / "late.txt").exists() - assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", "sleep 3; echo done > late.txt"] + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", + "sleep 3; echo done > late.txt"] for _ in range(60): if (computer.home / "late.txt").exists(): break @@ -182,7 +184,7 @@ async def test_a_system_command_runs_as_root_through_the_sandbox_in_the_session_ record = ctx.metadata[RECORDS_KEY]["exec"] assert record["exit_code"] == 0 and record["scope"] == "system" assert record["stdout"] == f"{computer.home}/work\n{computer.home} hi\n" - assert sandbox.calls[-1][:4] == ["timeout", "-k", "5", "120"] + assert sandbox.calls[-1][4:9] == ["65536", "timeout", "-k", "5", "120"] assert [params["argv"] for _, params, _ in computer.calls] == [["pwd"]] @@ -198,6 +200,34 @@ async def test_a_system_command_that_times_out_or_cant_start_fails_unless_check_ assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 125 +@pytest.mark.asyncio +async def test_a_system_command_timed_out_when_the_deadline_stopped_or_killed_it_not_when_it_exits_so_itself( + computer, sandbox, monkeypatch): + monkeypatch.setattr(computer_exec, "SYSTEM_KILL_AFTER_S", 1) + + ctx = await _step("exit 124", scope="system", check=False).execute(_context(computer.deployed())) + record = ctx.metadata[RECORDS_KEY]["exec"] + assert (record["exit_code"], record["timed_out"]) == (124, False) + + # It ignores the stop signal, so timeout kills it, and exits 137 rather than 124. + ctx = await _step("trap '' TERM; sleep 30", scope="system", timeout_s=0.5, check=False).execute( + _context(computer.deployed())) + record = ctx.metadata[RECORDS_KEY]["exec"] + assert (record["exit_code"], record["timed_out"]) == (None, True) + + +@pytest.mark.asyncio +async def test_a_system_commands_output_is_capped_in_the_computer_and_any_bytes_decode(computer, sandbox, monkeypatch): + monkeypatch.setattr(computer_exec, "SYSTEM_OUTPUT_BYTES", 1000) + + ctx = await _step("yes 0123456789 | head -c 55000; printf 'bad \\377\\376 bytes' >&2", scope="system").execute( + _context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["truncated"] and len(record["stdout"]) == 1000 and record["stdout"].endswith("0123456789\n") + assert record["stderr"] == "bad \ufffd\ufffd bytes" + + @pytest.mark.asyncio async def test_a_system_command_is_refused_on_a_disk_computer_and_on_a_sandbox_whose_exec_isnt_roots( computer, sandbox): @@ -209,6 +239,31 @@ async def test_a_system_command_is_refused_on_a_disk_computer_and_on_a_sandbox_w await _step("true", scope="system").execute(_context(computer.deployed())) +@pytest.mark.asyncio +async def test_on_a_windows_computer_a_string_or_background_command_is_refused_and_a_program_runs(computer): + windows = {**CARD, "capabilities": {**CARD["capabilities"], "extensions": [ + *CARD["capabilities"]["extensions"], {"uri": COMPUTER, "params": {"platform": {"os": "windows"}}}]}} + + for step in (_step("echo hi"), _step(["notepad.exe"], background=True)): + with pytest.raises(RuntimeError, match="runs Windows, which has no sh"): + await step.execute(_context(computer.deployed(card=windows))) + assert computer.calls == [] + + ctx = await _step(["echo", "hi"]).execute(_context(computer.deployed(card=windows))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "hi\n" + + +@pytest.mark.asyncio +async def test_a_background_start_still_running_when_its_window_ends_leaves_no_files(computer, tmp_path, monkeypatch): + monkeypatch.setattr(computer_exec, "START_WINDOW_S", 0.3) + scratch = tmp_path / "scratch" + scratch.mkdir() + + await _step(["sleep", "3"], background=True, env={"TMPDIR": str(scratch)}).execute(_context(computer.deployed())) + + assert list(scratch.iterdir()) == [] + + @pytest.mark.asyncio async def test_a_timeout_above_what_the_computer_allows_is_refused_before_anything_runs(computer): with pytest.raises(ValueError, match="above the 3600 s"):