diff --git a/AGENTS.md b/AGENTS.md index d6bbf6649..04ed36bc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/env/envs/computer/utils/inside.py b/src/agent_env/env/envs/computer/utils/inside.py index c03ccecaa..b77ffd885 100644 --- a/src/agent_env/env/envs/computer/utils/inside.py +++ b/src/agent_env/env/envs/computer/utils/inside.py @@ -92,11 +92,12 @@ async def read_file(deployed: DeployedSandboxEnv, path: str) -> bytes: async def run(deployed: DeployedSandboxEnv, argv: list[str], *, timeout_s: float, - env: Optional[dict[str, str]] = None) -> dict: - """computer-exec/v1 ``run``'s answer for ``argv``, under a request id of its own. A call whose connection breaks - fails rather than being sent again: a driver that restarted would run the command a second time, beside the first, - which outlives the driver that started it.""" - params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {})} + env: Optional[dict[str, str]] = None, cwd: Optional[str] = None) -> dict: + """computer-exec/v1 ``run``'s answer for ``argv``, in ``cwd`` relative to the home, the home by default, under a + request id of its own. A call whose connection breaks fails rather than being sent again: a driver that restarted + would run the command a second time, beside the first, which outlives the driver that started it.""" + params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {}), + **({"cwd": cwd} if cwd else {})} try: return await deployed.invoke(COMPUTER_EXEC, "run", params, timeout=timeout_s + CALL_MARGIN_S) except httpx.TransportError as e: diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index b24fc67f8..f18b25eec 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -418,13 +418,25 @@ async def run(self, argv: list[str]) -> str: if isinstance(self.sandbox, VmSandbox): return await self.sandbox.exec_script(f"docker exec -u 0 {shlex.quote(self.sandbox.container_name)} " f"{shlex.join(argv)}") + await self._check_root() + return await self._exec(argv) + + async def exec(self, argv: list[str]) -> tuple[int, str, str]: + """``argv`` run as root: its exit code, stdout and stderr, whatever the code.""" + if isinstance(self.sandbox, VmSandbox): + return await self.sandbox.exec_with_output( + "sudo", "bash", "-c", f"docker exec -u 0 {shlex.quote(self.sandbox.container_name)} {shlex.join(argv)}") + await self._check_root() + return await self.sandbox.exec_with_output(*argv) + + async def _check_root(self) -> None: if not self.checked: uid = (await self._exec(["id", "-u"])).strip() if uid != "0": - raise RuntimeError(f"site routing installs as root, and the {getattr(self.sandbox, 'type', 'given')} " - f"sandbox runs the computer's commands as uid {uid}; deploy it on a VM sandbox") + raise RuntimeError(f"root in the computer runs through its sandbox, and the " + f"{getattr(self.sandbox, 'type', 'given')} sandbox runs the computer's commands as " + f"uid {uid}; deploy it on a VM sandbox") self.checked = True - return await self._exec(argv) async def _exec(self, argv: list[str]) -> str: code, stdout, stderr = await self.sandbox.exec_with_output(*argv) diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 02653164c..d13328f9d 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -156,7 +156,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage", "browser_close_tab", "browser_delete_cookies", "browser_lease_context", "browser_open_tab", "browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli", - "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", + "combine_universe_verdicts", "computer_exec", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", "end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "user_sim_judge", "run_container_unit_tests_verifier", "run_docker_container", "set_env_clock_rate", "snapshot_env", "snapshot_computer_env", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 7d42cbb2e..7ef801da7 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -105,6 +105,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep + from agent_env.task_step.task_steps.computer.computer_exec import ComputerExecTaskStep from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep @@ -175,6 +176,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(SnapshotAgentStateTaskStep): SnapshotAgentStateTaskStep, _get_type(SnapshotEnvTaskStep): SnapshotEnvTaskStep, _get_type(SnapshotComputerEnvTaskStep): SnapshotComputerEnvTaskStep, + _get_type(ComputerExecTaskStep): ComputerExecTaskStep, _get_type(RunCodeTaskStep): RunCodeTaskStep, _get_type(PeerAgentsTaskStep): PeerAgentsTaskStep, _get_type(VerifyA2ASnapshotStep): VerifyA2ASnapshotStep, diff --git a/src/agent_env/task_step/task_steps/computer/computer_exec.py b/src/agent_env/task_step/task_steps/computer/computer_exec.py new file mode 100644 index 000000000..6e828eb70 --- /dev/null +++ b/src/agent_env/task_step/task_steps/computer/computer_exec.py @@ -0,0 +1,313 @@ +"""computer_exec: a command run inside a deployed computer as its session user, through its driver's computer-exec/v1 +``run``, a control route the agent can't call, or with ``scope: "system"`` as root, through the computer's sandbox, as a +task sets a computer up: a program run to its end, its exit status and output recorded, or with ``background`` an app +started and left running.""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import logging +import posixpath +import shlex +import time +from typing import ClassVar, Optional, Union + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER, COMPUTER_EXEC + +from agent_env.entity.refs import EntityRef +from agent_env.env.env import DeployedEnv, DeployedSandboxEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils import inside +from agent_env.env.envs.computer.utils.sites import RootChannel +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.preflight import stored_env +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +logger = logging.getLogger(__name__) + +#: Where the step records what each run did, by step id. +RECORDS_KEY = "computer_exec" +#: How long a command may run when the step doesn't say. +DEFAULT_TIMEOUT_S = 120.0 +#: The most of each stream the record keeps, and the most of stderr a failure says: the end of it. +RECORDED_CHARS = 4000 +FAILURE_CHARS = 300 +#: How long a ``background`` app is watched after it starts: one that exits non-zero within it fails the step. +START_WINDOW_S = 2.0 +#: How much longer than its start window a ``background`` start may take, for the shell and the stderr it reads. +START_MARGIN_S = 3.0 +#: The most of a failed start's stderr the launcher keeps: the end of it. +START_STDERR_BYTES = 4096 +#: How often the launcher checks a ``background`` start, and how long it waits for a failed start's stderr. +START_POLL_S = 0.1 +START_STDERR_WAIT_S = 1.0 +#: Who a command runs as: the session user, through the driver, or for ``system``, root, through the sandbox. +SCOPES = ("session", "system") +#: How long the session user's home takes to find, which a ``system`` command runs in. +HOME_TIMEOUT_S = 30.0 +#: How much longer than ``timeout_s`` a ``system`` command's exec may take before the step stops waiting for it. +SYSTEM_MARGIN_S = 30.0 +#: How long ``timeout`` gives a ``system`` command after its stop signal before it kills it. +SYSTEM_KILL_AFTER_S = 5 +#: What ``timeout`` exits with when it stopped the command, and when it had to kill it (128 + SIGKILL). +TIMED_OUT_EXIT = 124 +KILLED_EXIT = 137 +#: The most of each stream a ``system`` command keeps in the computer, the end of it, so a command that prints without +#: end can't fill the task runner's memory; the record keeps less still. +SYSTEM_OUTPUT_BYTES = 65536 +#: How a ``system`` command runs, given that cap and then the command: the end of each stream kept and base64'd, so +#: whatever bytes it printed decode, and the command's exit status passed on as its own. +_CAPPED = """cap=$1; shift +d=$(mktemp -d) || exit 125 +{ { "$@"; echo $? >"$d/rc"; } 2>&1 >&3 3>&- | tail -c "$cap" >"$d/err"; } 3>&1 | tail -c "$cap" | base64 -w0 +base64 -w0 "$d/err" >&2 +rc=$(cat "$d/rc" 2>/dev/null) +rm -rf "$d" +exit "${rc:-125}" +""" + + +def background_script() -> str: + """How a ``background`` command starts, as ``sh -c`` runs it with the command as its arguments: its program found + and executable, else it fails as the shell does with 127; then started detached from the run's pipes, its stdout + discarded and the end of its stderr kept, and watched for ``START_WINDOW_S``. One that exits non-zero in that window + fails with its exit status and its stderr, which also catches a string command (whose program is ``sh``) naming a + program that isn't there; one that exits 0, as ``xdg-open`` and single-instance apps do once they hand off, or is + still running, has started. Its folder goes either way: an app still running keeps the stderr file it writes open, + so removing the folder takes nothing from it.""" + window = max(1, round(START_WINDOW_S / START_POLL_S)) + stderr_wait = max(1, round(START_STDERR_WAIT_S / START_POLL_S)) + return ( + 'case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac' + ' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; ' + 'd=$(mktemp -d) || exit 1; ' + f'( {{ "$@" 2>&1 >/dev/null; echo $? > "$d/rc"; }} | tail -c {START_STDERR_BYTES} > "$d/err" ) ' + '/dev/null 2>&1 & ' + f'n=0; while [ ! -s "$d/rc" ] && [ $n -lt {window} ]; do sleep {START_POLL_S:g}; n=$((n+1)); done; ' + '[ -s "$d/rc" ] || { rm -rf "$d"; exit 0; }; ' + 'rc=$(cat "$d/rc"); [ "$rc" = 0 ] && { rm -rf "$d"; exit 0; }; ' + f'n=0; while [ ! -s "$d/err" ] && [ $n -lt {stderr_wait} ]; do sleep {START_POLL_S:g}; n=$((n+1)); done; ' + 'echo "computer_exec: $* exited $rc right after starting" >&2; cat "$d/err" >&2; rm -rf "$d"; exit "$rc"') + + +Command = Union[list[str], str] + + +class ComputerExecTaskStep(TaskStep): + """Run ``command`` inside the computer of the env ``env_id`` as its session user, in ``cwd`` (relative to the home, + the home by default), with ``env`` added to its environment, for at most ``timeout_s``. ``command`` is a program and + its arguments, or a string run by ``sh -c``. For a multi env, ``computer`` names the computer, by its environment + name, and may be left out when the env has one. + + With ``scope: "system"`` it runs as root instead, through the computer's sandbox rather than its driver, in the + same folder and with the session user's ``HOME``; a computer booted from a disk has no such route. + + The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and + stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless + ``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app + launched for the agent is: the step watches it for ``START_WINDOW_S`` and fails as a run does when its program + isn't there to start, or when it exits non-zero in that window, with the end of its stderr; an exit of 0 counts as + started, since ``xdg-open`` and single-instance apps hand off to another process and exit.""" + + type: ClassVar[str] = "computer_exec" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, command: Command, + computer: Optional[str] = None, cwd: Optional[str] = None, env: Optional[dict[str, str]] = None, + timeout_s: float = DEFAULT_TIMEOUT_S, background: bool = False, check: bool = True, + scope: str = "session", depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.command = command + self.computer = computer + self.cwd = cwd + self.env = env + self.timeout_s = timeout_s + self.background = background + self.check = check + self.scope = scope + + def config_problems(self) -> list[str]: + problems = [] + if not isinstance(self.env_id, str) or not self.env_id: + problems.append(f"step {self.id!r}: env_id must be a non-empty string") + if isinstance(self.command, str): + if not self.command.strip(): + problems.append(f"step {self.id!r}: command must not be blank") + elif not (isinstance(self.command, list) and self.command and all(isinstance(a, str) for a in self.command)): + problems.append(f"step {self.id!r}: command must be a non-empty list of strings, or a string run by sh -c") + if self.computer is not None and (not isinstance(self.computer, str) or not self.computer): + problems.append(f"step {self.id!r}: computer, when given, must be a non-empty string") + if self.cwd is not None and (not isinstance(self.cwd, str) or self.cwd.startswith("/")): + problems.append(f"step {self.id!r}: cwd is a folder relative to the session user's home, not {self.cwd!r}") + if self.env is not None and not (isinstance(self.env, dict) and all( + isinstance(k, str) and k and isinstance(v, str) for k, v in self.env.items())): + problems.append(f"step {self.id!r}: env maps variable names to string values") + if isinstance(self.timeout_s, bool) or not isinstance(self.timeout_s, (int, float)) or self.timeout_s <= 0: + problems.append(f"step {self.id!r}: timeout_s must be a number of seconds above 0") + for name in ("background", "check"): + if not isinstance(getattr(self, name), bool): + problems.append(f"step {self.id!r}: {name} must be true or false") + if (self.background is True and isinstance(self.timeout_s, (int, float)) + and self.timeout_s < (least := START_WINDOW_S + START_MARGIN_S)): + problems.append(f"step {self.id!r}: a background start is watched for {START_WINDOW_S:g} s, so its " + f"timeout_s must be at least {least:g}") + if self.scope not in SCOPES: + problems.append(f"step {self.id!r}: scope is 'session', the session user's, or 'system', root's, " + f"not {self.scope!r}") + elif self.scope == "system" and self.background is True: + problems.append(f"step {self.id!r}: background runs as the session user; a 'system' command runs to its " + "end") + return problems + + def preflight(self) -> list[str]: + """The env this step names when it has no computer, or none of that name, or several when it names none.""" + problems = super().preflight() + if not isinstance(self.env_id, str) or not self.env_id or (env := stored_env(self.env_id)) is None: + return problems + if problem := _computer_problem(env, self.computer): + problems.append(f"step {self.id!r}: {problem}") + return problems + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "command": self.command, "computer": self.computer, + "cwd": self.cwd, "env": self.env, "timeout_s": self.timeout_s, "background": self.background, + "check": self.check, "scope": self.scope} + + @classmethod + def from_dict(cls, data: dict) -> ComputerExecTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], command=data["command"], + computer=data.get("computer"), cwd=data.get("cwd"), env=data.get("env"), + timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False), + check=data.get("check", True), scope=data.get("scope", "session")) + + def command_argv(self) -> list[str]: + """The command as a program and its arguments: through ``sh -c`` when it's a string.""" + return ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command) + + def argv(self) -> list[str]: + """What the driver runs: the command, started by ``background_script`` with ``background``.""" + argv = self.command_argv() + return ["sh", "-c", background_script(), "computer_exec", *argv] if self.background else argv + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = find_computer(context, self.env_id, self.computer) + if missing := [m for m in inside.missing(deployed) if m.startswith(COMPUTER_EXEC)]: + raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer serves no {missing[0]}") + if self.timeout_s > (limit := inside.max_timeout_s(deployed)): + raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer " + f"of env {self.env_id!r} lets a command run") + if (self.background or isinstance(self.command, str)) and _os(deployed) == "windows": + raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer runs Windows, which has no sh " + "to run a string or start an app in the background: give the command as a program and " + "its arguments, without background") + argv = self.argv() + if self.scope == "system": + answer = await self._as_root(deployed, argv) + else: + answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd) + record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "scope": self.scope, + "background": self.background, + "exit_code": answer.get("exit_code"), "timed_out": bool(answer.get("timed_out")), + "truncated": bool(answer.get("truncated")), + "stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS), + "stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)} + context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record + shown = shlex.join(self.command_argv()) + (" as root" if self.scope == "system" else "") + if record["timed_out"]: + outcome = f"timed out after {self.timeout_s} s" + elif record["exit_code"] != 0: + outcome = f"exited {record['exit_code']}" + else: + logger.info(f"{self.id}: {'started' if self.background else 'ran'} {shown} in env {self.env_id!r}") + return context + message = f"{self.type} {self.id!r}: {shown} {outcome} in env {self.env_id!r}" + if stderr := _tail(record["stderr"].strip(), FAILURE_CHARS): + message += f": {stderr}" + if self.check: + raise RuntimeError(message) + logger.warning(f"{message} (check is off)") + return context + + async def _as_root(self, deployed: DeployedSandboxEnv, argv: list[str]) -> dict: + """``argv`` run as root through the computer's sandbox (``RootChannel``), in the folder and with the ``HOME`` a + session run has, so ``cwd`` and ``~`` name the same files, under ``timeout``, its output capped in the computer + (``_CAPPED``); answered as ``run`` answers. It timed out when ``timeout`` stopped or killed it past the + deadline, which a command exiting with that status of its own, earlier, didn't.""" + if (deployed.metadata.get("computer") or {}).get("form") == "disk": + raise RuntimeError(f"{self.type} {self.id!r}: a 'system' command runs through the computer's container, " + f"and env {self.env_id!r}'s computer boots from a disk, which core can't reach as root") + home = (await inside.run(deployed, ["pwd"], timeout_s=HOME_TIMEOUT_S))["stdout"].strip() + variables = [f"{name}={value}" for name, value in (self.env or {}).items()] + command = ["sh", "-c", _CAPPED, "computer_exec", str(SYSTEM_OUTPUT_BYTES), + "timeout", "-k", str(SYSTEM_KILL_AFTER_S), f"{self.timeout_s:g}", "env", "-C", + posixpath.join(home, self.cwd) if self.cwd else home, f"HOME={home}", *variables, *argv] + sandbox = await build_sandbox_provider(deployed.sandbox_type).get_sandbox(deployed.sandbox_id) + started = time.monotonic() + code, stdout, stderr = await asyncio.wait_for(RootChannel(sandbox).exec(command), + self.timeout_s + SYSTEM_MARGIN_S) + timed_out = code in (TIMED_OUT_EXIT, KILLED_EXIT) and time.monotonic() - started >= self.timeout_s + out, err = _unpacked(stdout), _unpacked(stderr) + return {"exit_code": None if timed_out else code, "timed_out": timed_out, + "truncated": max(len(out), len(err)) >= SYSTEM_OUTPUT_BYTES, + "stdout": out.decode("utf-8", errors="replace"), "stderr": err.decode("utf-8", errors="replace")} + + +def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv: + """The deploy record of the computer in ``env_id`` that serves computer-exec/v1 ``run``: the env's own, or one of a + multi env's computers' (``DeployedGatewayEnv.child_envs``), the one ``computer`` names else its only one.""" + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs") + children = {name: DeployedEnv.from_dict(data) + for name, data in (getattr(deployed, "child_envs", None) or {}).items()} + if computer is not None: + found = children.get(computer) + if found is None: + raise RuntimeError(f"env {env_id!r} has no computer named {computer!r} (it has {sorted(children)})") + return found + if deployed.supports(COMPUTER_EXEC, "run"): + return deployed + computers = [record for record in children.values() if record.supports(COMPUTER_EXEC, "run")] + if len(computers) != 1: + raise RuntimeError(f"env {env_id!r} has {len(computers) or 'no'} computers serving {COMPUTER_EXEC} run" + + (f": name one as computer, of {sorted(children)}" if computers else "")) + return computers[0] + + +def _computer_problem(env: object, computer: Optional[str]) -> Optional[str]: + if isinstance(env, ComputerEnv): + return None if computer is None else f"env {env.id!r} is a computer env, so it takes no computer name" + if not isinstance(env, MultiEnv): + return f"env {getattr(env, 'id', '?')!r} is no computer env, nor a multi env with one" + browser = env.website_browser().computer + names = [computer_env.environment_name for computer_env in (*env.computer_envs, *([browser] if browser else []))] + if computer is not None and computer not in names: + return f"env {env.id!r} has no computer named {computer!r} (it has {names})" + if computer is None and len(names) != 1: + return (f"env {env.id!r} has {len(names) or 'no'} computers" + + (f": name one as computer, of {names}" if names else "")) + return None + + +def _os(deployed: DeployedSandboxEnv) -> Optional[str]: + """The computer's OS, as its card's computer/v1 platform says.""" + return (protocol_v1.extension_params(deployed.environment_card or {}, COMPUTER).get("platform") or {}).get("os") + + +def _unpacked(text: str) -> bytes: + """A stream ``_CAPPED`` base64'd, as bytes; what the sandbox said in its place, as it said it.""" + try: + return base64.b64decode(text.strip(), validate=True) + except (binascii.Error, ValueError): + return text.encode() + + +def _tail(text: str, chars: int) -> str: + return text if len(text) <= chars else "…" + text[-chars:] diff --git a/tst/integration/computer/computer_exec/envs/exec-desk/env.toml b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml new file mode 100644 index 000000000..e1dca97f8 --- /dev/null +++ b/tst/integration/computer/computer_exec/envs/exec-desk/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/computer_exec/tasks/setup.json b/tst/integration/computer/computer_exec/tasks/setup.json new file mode 100644 index 000000000..bfa6551e8 --- /dev/null +++ b/tst/integration/computer/computer_exec/tasks/setup.json @@ -0,0 +1,23 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "exec-desk"}, + {"id": "write", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["env"], + "command": ["sh", "-c", "mkdir -p Desktop && echo computer-exec > Desktop/note.txt && cat Desktop/note.txt"]}, + {"id": "launch", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["write"], + "command": ["thunar", "Desktop"], "background": true}, + {"id": "window", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["launch"], "timeout_s": 60, + "command": "for i in $(seq 100); do xwininfo -root -tree | grep -i -m1 'thunar' && exit 0; sleep 0.5; done; exit 1"}, + {"id": "tolerated", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["window"], + "command": "echo nope >&2; exit 7", "check": false}, + {"id": "missing", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["tolerated"], + "command": ["no-such-app", "Desktop"], "background": true, "check": false}, + {"id": "missing-string", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing"], + "command": "thunarr Desktop", "background": true, "check": false}, + {"id": "bad-flag", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["missing-string"], + "command": ["ls", "--no-such-flag"], "background": true, "check": false}, + {"id": "root", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["bad-flag"], "scope": "system", + "command": "id -u; pwd; echo ~; touch /etc/computer-exec-root"}, + {"id": "owner", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["root"], + "command": ["stat", "-c", "%U", "/etc/computer-exec-root"]}, + {"id": "denied", "type": "computer_exec", "env_id": "exec-desk", "depends_on": ["owner"], + "command": "touch /etc/computer-exec-denied", "check": false} +] diff --git a/tst/integration/computer/computer_exec_test.py b/tst/integration/computer/computer_exec_test.py new file mode 100644 index 000000000..561f4ba1b --- /dev/null +++ b/tst/integration/computer/computer_exec_test.py @@ -0,0 +1,39 @@ +"""computer_exec on a computer env, end to end: the ubuntu-24.04 base with the reference driver deployed, a command run +in the session user's home to its end, its output recorded; Thunar started with ``background`` on the folder that wrote, +as a task opens a file in an app for the agent, and its window then found on the display, so the app outlived the run +that started it; a failing command, and a background one whose program isn't there, recorded, not failing the task, +with ``check`` off; and a ``system`` command run as root in the session user's home, which the session user can't do.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from tst.integration.computer.conftest import run_context + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BUNDLE = Path(__file__).parent / "computer_exec" + + +def test_commands_run_in_a_computer_and_an_app_started_in_the_background_stays_up(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "setup", "--no-emulate"]) + + assert ran.exit_code == 0, ran.output + records = run_context(ran.output)["metadata"]["computer_exec"] + assert records["write"]["exit_code"] == 0 and records["write"]["stdout"] == "computer-exec\n", records["write"] + assert records["launch"]["exit_code"] == 0 and records["launch"]["background"], records["launch"] + assert records["window"]["exit_code"] == 0 and "thunar" in records["window"]["stdout"].lower(), records["window"] + assert records["tolerated"]["exit_code"] == 7 and records["tolerated"]["stderr"] == "nope\n", records["tolerated"] + assert records["missing"]["exit_code"] == 127 and "not found" in records["missing"]["stderr"], records["missing"] + # A string command's program is sh, so it's caught when sh exits right after starting; so is a program given a bad + # flag. (A second thunar would exit 0: it hands off to the one running, which counts as started.) + assert records["missing-string"]["exit_code"] == 127, records["missing-string"] + assert "right after starting" in records["missing-string"]["stderr"], records["missing-string"] + assert records["bad-flag"]["exit_code"] not in (0, None), records["bad-flag"] + assert "right after starting" in records["bad-flag"]["stderr"], records["bad-flag"] + uid, home, tilde = records["root"]["stdout"].split() + assert records["root"]["exit_code"] == 0 and uid == "0" and home == tilde != "/root", records["root"] + assert records["owner"]["stdout"] == "root\n", records["owner"] + assert records["denied"]["exit_code"] != 0 and "Permission denied" in records["denied"]["stderr"], records["denied"] diff --git a/tst/unit/computer_home.py b/tst/unit/computer_home.py index 5f7d8f086..07ed95c94 100644 --- a/tst/unit/computer_home.py +++ b/tst/unit/computer_home.py @@ -74,8 +74,9 @@ async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, v def _run(self, params: dict) -> dict: try: - done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})}, - capture_output=True, text=True, timeout=params["timeout_s"]) + done = subprocess.run(params["argv"], cwd=self.home / (params.get("cwd") or ""), + env={**os.environ, **(params.get("env") or {})}, capture_output=True, text=True, + timeout=params["timeout_s"]) except subprocess.TimeoutExpired: return {"exit_code": None, "stdout": "", "stderr": "", "truncated": False, "timed_out": True} return {"exit_code": done.returncode, "stdout": done.stdout, "stderr": done.stderr, "truncated": False, diff --git a/tst/unit/task_step/computer_exec_test.py b/tst/unit/task_step/computer_exec_test.py new file mode 100644 index 000000000..872022071 --- /dev/null +++ b/tst/unit/task_step/computer_exec_test.py @@ -0,0 +1,354 @@ +"""computer_exec against a computer at its driver's boundary (``tst.unit.computer_home``): a command run in its home as +given or through ``sh -c``, in a folder and with variables of its own, its run recorded; a failure or a timeout failing +the step unless ``check`` is off; a ``background`` command started and left running, and failing when its program +isn't there or it exits non-zero right after starting, a string command's included; a ``system`` command run as root +through the computer's sandbox, in the session user's folder; a multi env's computer found by name; and the configs it +refuses.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import subprocess +import time +from types import SimpleNamespace + +import pytest +from agentenv_protocol.computer import COMPUTER + +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.multi_env import MultiEnv, WebsiteBrowser +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.computer import computer_exec +from agent_env.task_step.task_steps.computer.computer_exec import RECORDS_KEY, ComputerExecTaskStep +from tst.unit.computer_home import CARD, TOKEN, Computer + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + return Computer(tmp_path).install(monkeypatch) + + +def _context(*deployed) -> TaskStepContext: + return TaskStepContext(deployed_agents=[], deployed_envs=list(deployed), metadata={}) + + +def _step(command, **fields) -> ComputerExecTaskStep: + return ComputerExecTaskStep(id="exec", version=None, env_id=fields.pop("env_id", "desk"), command=command, **fields) + + +class _Sandbox: + """A computer's sandbox whose exec is ``uid``'s, running what it's given on this machine.""" + + type = "fake" + + def __init__(self): + self.uid = "0" + self.calls: list[list[str]] = [] + + async def exec_with_output(self, *argv): + self.calls.append(list(argv)) + if list(argv) == ["id", "-u"]: + return 0, f"{self.uid}\n", "" + done = await asyncio.to_thread(subprocess.run, list(argv), capture_output=True, text=True) + return done.returncode, done.stdout, done.stderr + + +@pytest.fixture +def sandbox(monkeypatch): + held = _Sandbox() + + async def get_sandbox(sandbox_id): + return held + + monkeypatch.setattr(computer_exec, "build_sandbox_provider", + lambda sandbox_type: SimpleNamespace(get_sandbox=get_sandbox)) + return held + + +@pytest.mark.asyncio +async def test_a_command_runs_in_the_home_as_given_with_the_computers_token_and_is_recorded(computer): + ctx = await _step(["sh", "-c", "pwd; echo out; echo err >&2"]).execute(_context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["exit_code"] == 0 and not record["timed_out"] + assert record["stdout"] == f"{computer.home}\nout\n" and record["stderr"] == "err\n" + assert computer.headers == [{"Authorization": f"Bearer {TOKEN}"}] + _, params, _ = computer.calls[0] + assert params["argv"] == ["sh", "-c", "pwd; echo out; echo err >&2"] and params["timeout_s"] == 120.0 + + +@pytest.mark.asyncio +async def test_a_string_runs_through_sh_in_its_folder_with_its_variables(computer): + (computer.home / "work").mkdir() + + ctx = await _step('echo "$GREETING" > made.txt', cwd="work", env={"GREETING": "hi"}, + timeout_s=5).execute(_context(computer.deployed())) + + assert (computer.home / "work" / "made.txt").read_text() == "hi\n" + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"] == ["sh", "-c", 'echo "$GREETING" > made.txt'] + _, params, _ = computer.calls[0] + assert params["cwd"] == "work" and params["env"] == {"GREETING": "hi"} and params["timeout_s"] == 5 + + +@pytest.mark.asyncio +async def test_a_command_that_fails_fails_the_step_with_the_end_of_its_stderr_and_is_still_recorded(computer): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 3 in env 'desk': no such thing"): + await _step("echo 'no such thing' >&2; exit 3").execute(ctx) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 3 + + +@pytest.mark.asyncio +async def test_with_check_off_a_failure_is_recorded_and_the_step_goes_on(computer): + ctx = await _step("exit 4", check=False).execute(_context(computer.deployed())) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 4 + + +@pytest.mark.asyncio +async def test_a_command_that_times_out_fails_the_step(computer): + with pytest.raises(RuntimeError, match=r"timed out after 0.2 s"): + await _step("sleep 5", timeout_s=0.2).execute(_context(computer.deployed())) + + +@pytest.fixture +def short_start(monkeypatch): + monkeypatch.setattr(computer_exec, "START_WINDOW_S", 0.3) + + +@pytest.mark.asyncio +async def test_a_background_command_is_started_and_left_running(computer, short_start): + started = time.monotonic() + + ctx = await _step("sleep 3; echo done > late.txt", background=True).execute(_context(computer.deployed())) + + assert time.monotonic() - started < 2.5 + assert not (computer.home / "late.txt").exists() + assert ctx.metadata[RECORDS_KEY]["exec"]["argv"][3:] == ["computer_exec", "sh", "-c", + "sleep 3; echo done > late.txt"] + for _ in range(60): + if (computer.home / "late.txt").exists(): + break + time.sleep(0.1) + assert (computer.home / "late.txt").read_text() == "done\n" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("command, exit_code, stderr", [ + ("thunarr-xyz Desktop", 127, "not found"), + (["sh", "-c", "echo 'unknown option --bad' >&2; exit 2"], 2, "unknown option --bad"), +]) +async def test_a_background_app_that_exits_non_zero_right_after_starting_fails_with_its_stderr( + computer, short_start, command, exit_code, stderr): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=rf"(?s)^computer_exec 'exec': sh -c .{{0,80}} exited {exit_code} in env " + rf"'desk': .*right after starting.*{stderr}"): + await _step(command, background=True).execute(ctx) + + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == exit_code + + +@pytest.mark.asyncio +async def test_a_background_app_that_hands_off_and_exits_0_has_started(computer, short_start): + ctx = await _step("echo handed off >&2; exit 0", background=True).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 0 + + +@pytest.mark.parametrize("command", [["no-such-app-xyz", "a.txt"], ["./missing.sh"]]) +@pytest.mark.asyncio +async def test_a_background_command_whose_program_isnt_there_fails_the_step(computer, command): + ctx = _context(computer.deployed()) + + with pytest.raises(RuntimeError, match=r"exited 127 in env 'desk': .*not found or not executable"): + await _step(command, background=True).execute(ctx) + + ctx = await _step(command, background=True, check=False).execute(ctx) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 + + +@pytest.mark.asyncio +async def test_a_system_command_runs_as_root_through_the_sandbox_in_the_session_users_folder_and_home( + computer, sandbox): + (computer.home / "work").mkdir() + + ctx = await _step('pwd; echo "$HOME $GREETING"', scope="system", cwd="work", env={"GREETING": "hi"}).execute( + _context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["exit_code"] == 0 and record["scope"] == "system" + assert record["stdout"] == f"{computer.home}/work\n{computer.home} hi\n" + assert sandbox.calls[-1][4:9] == ["65536", "timeout", "-k", "5", "120"] + assert [params["argv"] for _, params, _ in computer.calls] == [["pwd"]] + + +@pytest.mark.asyncio +async def test_a_system_command_that_times_out_or_cant_start_fails_unless_check_is_off_and_is_recorded( + computer, sandbox): + with pytest.raises(RuntimeError, match=r"sh -c 'sleep 5' as root timed out after 0.5 s"): + await _step("sleep 5", scope="system", timeout_s=0.5).execute(_context(computer.deployed())) + + ctx = await _step(["no-such-app-xyz"], scope="system", check=False).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 127 + ctx = await _step("true", scope="system", cwd="missing", check=False).execute(_context(computer.deployed())) + assert ctx.metadata[RECORDS_KEY]["exec"]["exit_code"] == 125 + + +@pytest.mark.asyncio +async def test_a_system_command_timed_out_when_the_deadline_stopped_or_killed_it_not_when_it_exits_so_itself( + computer, sandbox, monkeypatch): + monkeypatch.setattr(computer_exec, "SYSTEM_KILL_AFTER_S", 1) + + ctx = await _step("exit 124", scope="system", check=False).execute(_context(computer.deployed())) + record = ctx.metadata[RECORDS_KEY]["exec"] + assert (record["exit_code"], record["timed_out"]) == (124, False) + + # It ignores the stop signal, so timeout kills it, and exits 137 rather than 124. + ctx = await _step("trap '' TERM; sleep 30", scope="system", timeout_s=0.5, check=False).execute( + _context(computer.deployed())) + record = ctx.metadata[RECORDS_KEY]["exec"] + assert (record["exit_code"], record["timed_out"]) == (None, True) + + +@pytest.mark.asyncio +async def test_a_system_commands_output_is_capped_in_the_computer_and_any_bytes_decode(computer, sandbox, monkeypatch): + monkeypatch.setattr(computer_exec, "SYSTEM_OUTPUT_BYTES", 1000) + + ctx = await _step("yes 0123456789 | head -c 55000; printf 'bad \\377\\376 bytes' >&2", scope="system").execute( + _context(computer.deployed())) + + record = ctx.metadata[RECORDS_KEY]["exec"] + assert record["truncated"] and len(record["stdout"]) == 1000 and record["stdout"].endswith("0123456789\n") + assert record["stderr"] == "bad \ufffd\ufffd bytes" + + +@pytest.mark.asyncio +async def test_a_system_command_is_refused_on_a_disk_computer_and_on_a_sandbox_whose_exec_isnt_roots( + computer, sandbox): + with pytest.raises(RuntimeError, match="boots from a disk"): + await _step("true", scope="system").execute(_context(computer.deployed(form="disk"))) + + sandbox.uid = "1000" + with pytest.raises(RuntimeError, match="as uid 1000; deploy it on a VM sandbox"): + await _step("true", scope="system").execute(_context(computer.deployed())) + + +@pytest.mark.asyncio +async def test_on_a_windows_computer_a_string_or_background_command_is_refused_and_a_program_runs(computer): + windows = {**CARD, "capabilities": {**CARD["capabilities"], "extensions": [ + *CARD["capabilities"]["extensions"], {"uri": COMPUTER, "params": {"platform": {"os": "windows"}}}]}} + + for step in (_step("echo hi"), _step(["notepad.exe"], background=True)): + with pytest.raises(RuntimeError, match="runs Windows, which has no sh"): + await step.execute(_context(computer.deployed(card=windows))) + assert computer.calls == [] + + ctx = await _step(["echo", "hi"]).execute(_context(computer.deployed(card=windows))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "hi\n" + + +@pytest.mark.asyncio +async def test_a_background_start_still_running_when_its_window_ends_leaves_no_files(computer, tmp_path, monkeypatch): + monkeypatch.setattr(computer_exec, "START_WINDOW_S", 0.3) + scratch = tmp_path / "scratch" + scratch.mkdir() + + await _step(["sleep", "3"], background=True, env={"TMPDIR": str(scratch)}).execute(_context(computer.deployed())) + + assert list(scratch.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_a_timeout_above_what_the_computer_allows_is_refused_before_anything_runs(computer): + with pytest.raises(ValueError, match="above the 3600 s"): + await _step("true", timeout_s=4000).execute(_context(computer.deployed())) + assert computer.calls == [] + + +def _multi(computer: Computer, *names: str) -> DeployedGatewayEnv: + return DeployedGatewayEnv(env_id="multi", env_version=1, gateway_url="http://gw", sandbox_id="sb-gw", + environment_card={"name": "multi"}, + child_envs={name: dataclasses.asdict(computer.deployed(env_id=name)) for name in names}) + + +@pytest.mark.asyncio +async def test_a_multi_envs_only_computer_is_found_by_itself_and_one_of_several_by_name(computer): + ctx = await _step("echo one", env_id="multi").execute(_context(_multi(computer, "desk"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "one\n" + + ctx = await _step("echo two", env_id="multi", computer="b").execute(_context(_multi(computer, "a", "b"))) + assert ctx.metadata[RECORDS_KEY]["exec"]["stdout"] == "two\n" + + with pytest.raises(RuntimeError, match=r"has 2 computers .*: name one as computer"): + await _step("true", env_id="multi").execute(_context(_multi(computer, "a", "b"))) + with pytest.raises(RuntimeError, match=r"has no computer named 'c'"): + await _step("true", env_id="multi", computer="c").execute(_context(_multi(computer, "a", "b"))) + + +@pytest.mark.asyncio +async def test_an_env_with_no_computer_exec_is_refused(computer): + with pytest.raises(RuntimeError, match="has no computers serving"): + await _step("true").execute(_context(computer.deployed(card={"name": "desk"}))) + + +def _stored_multi(*computers: str, browser: str | None = None) -> MultiEnv: + def computer_env(name): + env = ComputerEnv.__new__(ComputerEnv) + env.environment_name = name + return env + + office = MultiEnv.__new__(MultiEnv) + office.id, office._computers = "office", SimpleNamespace(envs=[computer_env(name) for name in computers]) + office.website_browser = lambda: WebsiteBrowser(computer=computer_env(browser) if browser else None) + return office + + +@pytest.mark.parametrize("computers, browser, named, problem", [ + (("desk",), None, None, None), + ((), "chromium", None, None), + (("desk",), "chromium", "chromium", None), + (("desk",), "chromium", None, "has 2 computers: name one as computer, of ['desk', 'chromium']"), + (("desk",), None, "chromium", "has no computer named 'chromium' (it has ['desk'])"), + ((), None, None, "has no computers"), +], ids=["its-only-computer", "its-browser-computer-alone", "its-browser-computer-by-name", "two-unnamed", + "an-unknown-name", "none"]) +def test_preflight_finds_a_multi_envs_computers_and_its_browser_computer(monkeypatch, computers, browser, named, + problem): + monkeypatch.setattr("agent_env.task_step.task_steps.computer.computer_exec.stored_env", + lambda id: _stored_multi(*computers, browser=browser)) + monkeypatch.setattr(TaskStep, "preflight", lambda self: []) + + problems = _step("true", env_id="office", computer=named).preflight() + + assert problems == ([] if problem is None else [f"step 'exec': env 'office' {problem}"]) + + +@pytest.mark.parametrize("fields, problem", [ + ({"command": []}, "command must be a non-empty list"), + ({"command": " "}, "command must not be blank"), + ({"command": ["ls", 3]}, "command must be a non-empty list"), + ({"command": "true", "cwd": "/tmp"}, "relative to the session user's home"), + ({"command": "true", "env": {"A": 1}}, "env maps variable names to string values"), + ({"command": "true", "timeout_s": 0}, "timeout_s must be a number of seconds above 0"), + ({"command": "true", "background": "yes"}, "background must be true or false"), + ({"command": "true", "background": True, "timeout_s": 4}, "timeout_s must be at least 5"), + ({"command": "true", "scope": "root"}, "scope is 'session', the session user's, or 'system', root's"), + ({"command": "true", "scope": "system", "background": True}, "a 'system' command runs to its end"), +]) +def test_bad_configs_are_named(fields, problem): + assert any(problem in p for p in _step(**fields).config_problems()) + + +def test_it_round_trips_through_the_registry(): + step = _step(["open", "x.xlsx"], computer="desk", cwd="Desktop", env={"A": "1"}, timeout_s=30, background=True, + check=False) + again = get_task_step_registry()["computer_exec"].from_dict(step.to_dict()) + assert again.to_dict() == step.to_dict() + root = _step("apt-get install -y jq", scope="system") + assert get_task_step_registry()["computer_exec"].from_dict(root.to_dict()).scope == "system"