diff --git a/AGENTS.md b/AGENTS.md index e03fa8703..88c284f91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent, a program that isn't there exiting 127; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; on a multi env, `load_artifact` with `env_id` puts an environment artifact named after one of its computers, or after an app of one, into that computer, the computer's own as files at their paths under the session user's home; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (its driver draws each input over the frames in its caller's colour, and a pointer per role where the frames lack one, unless `annotation_settings` says `is_enabled: false`; a driver whose card lists no `annotate` records it plain, with a warning, and the run's recording entry says `annotated`) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent, a program that isn't there exiting 127; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; on a multi env, `load_artifact` with `env_id` puts an environment artifact named after one of its computers, or after an app of one, into that computer, the computer's own as files at their paths under the session user's home; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c8f1e7c27..aef79ea55 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -755,7 +755,7 @@ class MyDriver(ComputerDriverEnvironment): of its own, the latest 8 frames wait for the encoder, and it is closed when the recording stops. A driver without `@record` records by the base class polling `@capture`, with a pointer drawn at `@pointer_position`: the slow path, at most 15 fps. -- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns +- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?, annotate?}` returns `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment and returns the manifest; `end_all {reason?}` ends every recording not yet ended, each under the grant it started with, and returns `{manifests}`, as the framework does before it resets or tears a @@ -768,6 +768,21 @@ class MyDriver(ComputerDriverEnvironment): `/.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no grant they stay under `~/.agentenv-recordings/`. An ended recording's manifest uploads beside its segments, at `/manifest.json`, so a reader that didn't end it finds it there. +- **Annotations.** `start` with `annotate`, an object of options (`{}` for none; the card lists those a + driver takes under its params' `annotate.options`), draws each input over the frames captured while it + shows, in the colour of the role that sent it (the agent's first): a ring where a click landed, round a + hollow dot that leaves the pointer's tip in sight (a square for the right button, a ring for each click + of a double or triple), a trail for a drag, an + arrow for a scroll, a cross for a move, and a chip for a key chord, typed text as it was typed, and + each of the driver's own or forwarded tools whose annotations don't say it only reads, by name and + main argument. Where the frames can't show the pointer themselves, as frames polled from captures that + leave it out do (cua-driver's), a pointer is drawn for each role and pointer that has acted, where its + last input left it, along a drag's path as it moves, after a fading line from where it jumped: the one + that moved last at full strength, the others fainter, each named once there are several. Every `computer_*` input is marked + as `act` delivers it, best effort, and shows on + every frame while it lasts and on the first one after it however late, as at a low fps. A segment a + native recorder finished is kept as it came. The manifest's `annotate` is the object the recording + drew with, and is left out for one that took a native recorder's segments. - **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path, `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size, SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py new file mode 100644 index 000000000..d0e0df402 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -0,0 +1,519 @@ +"""What an annotated recording (computer-recording/v1 ``start`` with ``annotate``) draws over its frames: where each +input landed and what it typed or pressed, in the colour of the role that sent it, on the frames captured while its +mark shows. ``ComputerRuntime.act`` marks every ``computer_*`` input as it is delivered; a call to one of the +driver's own or forwarded tools, which acts without it, shows as a chip naming the tool and its main argument. Where +the frames can't show the pointer themselves, as frames from screenshots that leave it out, the agent's pointer is +drawn where its inputs left it.""" +from __future__ import annotations + +import dataclasses +import functools +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Optional, Union + +import av +from PIL import Image as PILImage +from PIL import ImageDraw, ImageFont +from pydantic import BaseModel, ConfigDict, ValidationError + +from .errors import InvalidArgument + +#: The role of a call that names none: the agent. +DEFAULT_ROLE = "default" +#: The pointer an input acts through when it names none: the computer's own, the only one the SDK drives yet. +DEFAULT_POINTER = "default" +#: Each role's colour: the agent's first, then each other role's in the order it first acts on the recording. +COLOURS = ((255, 138, 0), (0, 184, 148), (175, 82, 222), (255, 59, 105), (52, 120, 246), (250, 204, 21)) +#: How long a mark shows from its input, in seconds; a drag's, after its moves. A mark's kind is its operation's name, +#: or chip for one drawn as a caption. +SHOW_S = {"click": 0.8, "pointer_down": 0.8, "pointer_up": 0.8, "pointer_move": 0.5, "scroll": 0.9, "drag": 0.8, + "chip": 1.6} +#: The last part of a mark's time, in seconds, over which it fades out. +FADE_S = 0.3 +#: How long a mark no frame has drawn is kept, in seconds: past it, the recording's frames have stopped coming. +KEEP_S = 30.0 +#: How long after the pointer jumps a line shows where it came from, in seconds. +JUMP_S = 0.4 +#: The inputs whose marks are drawn at a point of the display, which leave the pointer at their last point, a drag's +#: after moving it along its path. +POINTED = ("click", "drag", "scroll", "pointer_move", "pointer_down", "pointer_up") +#: The most a chip of typed text shows, which grows with its length from the chip's own time. +MOST_TEXT_S = 4.0 +#: The most of a chip's text it shows; the rest is cut with an ellipsis. +CHIP_CHARS = 60 +#: The most chips shown at once, the newest lowest. +MOST_CHIPS = 4 +#: How far above the bottom edge, as a fraction of the frame's height, the lowest chip ends. +CHIPS_ABOVE = 0.1 +#: A frame's longer edge that marks are sized for; on a larger or smaller frame they scale with it. +UNIT_EDGE = 1280 +#: Arguments a tool's chip shows, the first one a call has. +MAIN_ARGUMENTS = ("text", "keys", "key", "url", "query", "label", "name", "element", "title", "code", "script") +#: Key names as chips show them. +KEY_NAMES = {"Control": "Ctrl", "Escape": "Esc", "ArrowLeft": "Left", "ArrowRight": "Right", "ArrowUp": "Up", + "ArrowDown": "Down", " ": "Space"} + + +class AnnotateOptions(BaseModel): + """What computer-recording/v1 ``start``'s ``annotate`` takes: no options yet, so ``{}`` draws every input. An + option is a field here, which the card lists, ``start`` checks, and a task step's annotation settings take too.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + +def options_problem(error: ValidationError, model: type[BaseModel] = AnnotateOptions) -> str: + """What is wrong with annotation options ``model`` refused, in a sentence.""" + takes = ", ".join(model.model_fields) or "no options yet" + problems = [] + for e in error.errors(): + where = ".".join(str(part) for part in e["loc"]) + if e["type"] == "extra_forbidden": + problems.append(f"it takes {takes}, not {where!r}") + elif not where: + problems.append(f"it is an object of options, got {type(e['input']).__name__}") + else: + problems.append(f"{where}: {e['msg'].lower()}") + return "; ".join(problems) + + +def check_annotate(annotate: object) -> Optional[dict]: + """``start``'s ``annotate``: None, or an object of the options this driver takes, as given.""" + if annotate is None: + return None + try: + return AnnotateOptions.model_validate(annotate).model_dump(exclude_unset=True) + except ValidationError as e: + raise InvalidArgument(f"annotate: {options_problem(e)}; give {{}} to draw every input") from None + + +@dataclass(frozen=True) +class Mark: + """One input on a recording: ``kind`` is its operation (click, drag, scroll, pointer_move, pointer_down, + pointer_up) for one drawn at a point, or chip for one drawn as a caption; ``points`` are fractions of the display, + so they land on any frame size; it shows from ``at`` until ``until``, a drag moving for its first ``moving_s``. + ``pointer`` is the pointer it acted through.""" + + kind: str + at: datetime + until: datetime + role: str + points: tuple[tuple[float, float], ...] = () + button: str = "left" + count: int = 1 + direction: str = "" + label: str = "" + moving_s: float = 0.0 + pointer: str = DEFAULT_POINTER + + +@dataclass(frozen=True) +class PointerMove: + """Where a role's pointer went at ``at``: from ``was`` along ``path``, a single point for a jump, a drag's path + while it moves for ``moving_s``. ``faded`` and ``label`` are how a frame draws it among others: fainter when it + didn't move last, named when there are several.""" + + at: datetime + role: str + pointer: str + was: tuple[float, float] + path: tuple[tuple[float, float], ...] + moving_s: float = 0.0 + faded: bool = False + label: str = "" + + +#: What a frame draws: a mark or a pointer, in its colour, at its age in seconds. +Shown = tuple[Union[Mark, PointerMove], tuple[int, int, int], float] + + +class Annotations: + """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role. Each + mark shows on every frame captured while it lasts, and on the first one after its input however late that comes, + as at a low fps or from a slow capture. With ``pointer``, as for frames that can't show the pointer themselves, + each frame also draws a pointer for each role and pointer that has acted, where its last input before the frame + left it: the one that moved last at full strength, the others fainter, each named once there are several.""" + + def __init__(self, pointer: bool = False) -> None: + self.marks: list[Mark] = [] + self.drawn: set[int] = set() + self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} + self.pointer = pointer + self.moves: dict[tuple[str, str], list[PointerMove]] = {} # each role's and pointer's, oldest first + + def add(self, marks: Sequence[Mark]) -> None: + for mark in marks: + self.colours.setdefault(mark.role, COLOURS[len(self.colours) % len(COLOURS)]) + stale = mark.at - timedelta(seconds=KEEP_S) + self._keep([kept for kept in self.marks if kept.until > stale]) + self.marks.append(mark) + if self.pointer and mark.kind in POINTED: + moves = self.moves.setdefault((mark.role, mark.pointer), []) + path = mark.points if mark.kind == "drag" else mark.points[-1:] + was = moves[-1].path[-1] if moves else path[0] + moves.append(PointerMove(mark.at, mark.role, mark.pointer, was, path, mark.moving_s)) + + def _pointers(self, at: datetime) -> list[Shown]: + """The pointers a frame captured ``at`` shows, each as its last move before the frame at its age, the one that + moved last drawn over the others; none for a role or pointer that hadn't acted by then.""" + current = [] + for key, moves in self.moves.items(): + last = next((i for i in range(len(moves) - 1, -1, -1) if moves[i].at <= at), None) + if last is not None: + self.moves[key] = moves[last:] # the frames come in order, so an earlier move won't show again + current.append(moves[last]) + current.sort(key=lambda move: move.at) + several = len(current) > 1 + shown = [dataclasses.replace(move, faded=move is not current[-1], + label=_pointer_label(move) if several else "") for move in current] + return [(move, self.colours[move.role], (at - move.at).total_seconds()) for move in shown] + + def showing(self, at: datetime) -> list[Shown]: + """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a + mark over by then that no frame has drawn yet is drawn once, at the age it is last fully shown. A stream's + frames come in the order they were captured, so a mark that is over and drawn is over for every later one.""" + due = [] + for mark in self.marks: + if mark.at > at: + continue + age = (at - mark.at).total_seconds() + if at >= mark.until: + if id(mark) in self.drawn: + continue + age = max(0.0, (mark.until - mark.at).total_seconds() - FADE_S) + due.append((mark, self.colours[mark.role], age)) + chips = [mark for mark, _colour, _age in due if mark.kind == "chip"] + waiting = {id(mark) for mark in chips[:-MOST_CHIPS]} # older chips than a frame shows wait for a later one + shown = [entry for entry in due if id(entry[0]) not in waiting] + self.drawn |= {id(mark) for mark, _colour, _age in shown} + self._keep([mark for mark in self.marks if mark.until > at or id(mark) not in self.drawn]) + return shown + (self._pointers(at) if self.pointer else []) + + def _keep(self, marks: list[Mark]) -> None: + self.marks = marks + self.drawn &= {id(mark) for mark in marks} + + +def input_marks(operation: str, args: tuple, *, role: Optional[str], at: datetime, size: tuple[int, int], + point: Optional[tuple] = None, repeat: int = 1, hold_ms: Optional[int] = None) -> list[Mark]: + """The marks of one ``ComputerRuntime.act`` input: ``args`` are the operation's, in screen units on a display of + ``size``; ``point`` is where an input with no point of its own acts (its ``move_to``, or the pointer).""" + role = role or DEFAULT_ROLE + width, height = size + + def where(x: float, y: float) -> tuple[float, float]: + return (x / width, y / height) + + def shown(kind: str, extra_s: float = 0.0) -> datetime: + return at + timedelta(seconds=SHOW_S[kind] + extra_s) + + def chip(label: str, seconds: Optional[float] = None) -> Mark: + until = at + timedelta(seconds=seconds) if seconds is not None else shown("chip") + return Mark("chip", at, until, role, label=label if role == DEFAULT_ROLE else f"{role}: {label}") + + held = args[-1] if operation in ("click", "drag", "scroll") and args and isinstance(args[-1], list) else [] + marks: list[Mark] = [] + if operation == "click": + x, y, button, count = args[:4] + marks.append(Mark("click", at, shown("click"), role, (where(x, y),), button=button, count=count)) + elif operation == "drag": + path, button, duration = args[:3] + marks.append(Mark("drag", at, shown("drag", duration / 1000), role, tuple(where(*p) for p in path), + button=button, moving_s=duration / 1000)) + elif operation == "scroll": + x, y, direction, amount = args[:4] + marks.append(Mark("scroll", at, shown("scroll"), role, (where(x, y),), direction=direction, count=amount)) + elif operation == "pointer_move": + marks.append(Mark(operation, at, shown(operation), role, (where(*args[:2]),))) + elif operation in ("pointer_down", "pointer_up"): + if point is not None: + marks.append(Mark(operation, at, shown(operation), role, (where(*point[:2]),), button=args[0])) + else: + marks.append(chip(f"{'press' if operation == 'pointer_down' else 'release'} {args[0]}")) + elif operation == "type_text": + text = args[0] + marks.append(chip(f'type "{_shown_text(text)}"', min(MOST_TEXT_S, SHOW_S["chip"] + len(text) / 25))) + elif operation == "press": + times = f" x{repeat}" if repeat > 1 else "" + marks.append(chip(f"key {_chord(args[0])}{times}")) + elif operation == "key_down": + for_ms = f" {hold_ms} ms" if hold_ms is not None else "" + marks.append(chip(f"hold {_chord(args[0])}{for_ms}")) + elif operation == "key_up": + marks.append(chip(f"release {_chord(args[0])}")) + elif operation == "navigate": + marks.append(chip(f"open {_cut(args[0])}")) + elif operation == "history": + marks.append(chip(args[0])) + elif operation == "tabs": + action, tab, url = args + if action != "list": + marks.append(chip(" ".join(str(part) for part in ("tab", action, tab or _cut(url or "")) if part))) + elif operation == "dialog": + marks.append(chip("dialog OK" if args[0] else "dialog Cancel")) + elif operation == "upload": + marks.append(chip(f"upload {_cut(', '.join(args[0]))}")) + else: + marks.append(chip(operation)) + if held: + marks.append(chip(f"{_chord(held)} + {operation}")) + return marks + + +def tool_mark(tool: str, arguments: Mapping[str, Any], *, role: Optional[str], at: datetime) -> Mark: + """The chip of a call to one of the driver's own or forwarded tools: its name and its main argument.""" + role = role or DEFAULT_ROLE + main = next((arguments[name] for name in MAIN_ARGUMENTS if arguments.get(name) not in (None, "", [])), None) + if isinstance(main, list) and all(isinstance(part, str) for part in main): + shown = _chord(main) + elif isinstance(main, str): + shown = f'"{_shown_text(main)}"' if main.strip() else "" + else: + shown = "" if main is None else _cut(str(main)) + label = f"{tool} {shown}".strip() + return Mark("chip", at, at + timedelta(seconds=SHOW_S["chip"]), role, + label=label if role == DEFAULT_ROLE else f"{role}: {label}") + + +def draw(image: Any, shown: Sequence[Shown]) -> PILImage.Image: + """``image``, a Pillow image or a PyAV frame, with what a frame shows drawn, each in its colour at its age.""" + base = (image.to_image() if isinstance(image, av.VideoFrame) else image).convert("RGBA") + overlay = PILImage.new("RGBA", base.size, (0, 0, 0, 0)) + pen = ImageDraw.Draw(overlay) + unit = max(base.size) / UNIT_EDGE + + def pixels(point: tuple[float, float]) -> tuple[float, float]: + return (point[0] * base.width, point[1] * base.height) + + chips = [] + for item, colour, age in shown: + if isinstance(item, PointerMove): + _draw_pointer(pen, item, pixels(item.was), [pixels(p) for p in item.path], colour, unit, age) + continue + if item.kind == "chip": + chips.append((item, colour)) + continue + span = max((item.until - item.at).total_seconds(), 1e-3) + fade = 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) + _DRAWERS[item.kind](pen, item, [pixels(p) for p in item.points], colour, unit, age, fade) + if chips: + _draw_chips(pen, chips, base.size, unit) + return PILImage.alpha_composite(base, overlay).convert("RGB") + + +@dataclass(frozen=True) +class Style: + """How marks look: sizes in pixels of a frame whose longer edge is UNIT_EDGE, scaled with the frame; each coloured + stroke over a wider white halo, so a mark reads on any background.""" + + stroke: float = 3 # a ring's and a cross's coloured stroke + halo: float = 5 # the white band under a ring's stroke + line: float = 4 # a trail's and an arrow's coloured stroke + line_halo: float = 7 + outline: float = 1.5 # a dot's white outline + ring_grow_s: float = 0.35 # how long a click's ring takes to grow to its full size, in seconds + ring_from: float = 12 # a click's ring's radius as it starts + ring_grows: float = 18 # and how much it grows + ring_apart: float = 8 # between the rings of a double or triple click + most_rings: int = 3 + dot: float = 5 # the hollow dot at a click's point, which leaves the pointer's tip there in sight + dot_stroke: float = 2 + pointer: float = 1.6 # the agent's pointer, as a multiple of the usual arrow cursor's size + pointer_outline: float = 2 + pointer_faded_alpha: int = 120 # a pointer that didn't move last + jump_alpha: int = 170 # the line from where the pointer jumped, at its start + press_down: float = 10 # a press's filled ring's radius + press_up: float = 16 # a release's ring's radius + press_fill: float = 0.7 # a press's fill opacity, as a share of its stroke's + cross: float = 9 # each arm of a move's cross + cross_alpha: int = 200 # a move's cross is fainter than the marks of inputs that act + arrow: float = 46 # a scroll's arrow + arrowhead: float = 14 # an arrowhead's sides, beside the stroke's own width + arrowhead_angle: float = 0.5 # radians either side of its line + start_dot: float = 6 # the dot where a drag starts + chip_text: float = 20 + chip_pad: float = 10 # beside a chip's text; above and below it, CHIP_PAD_HIGH of that + chip_gap: float = 8 # between stacked chips + chip_radius: float = 8 + chip_alpha: int = 235 + chip_low: float = 48 # the least the lowest chip ends above the bottom edge + label_text: float = 15 # a scroll's count and a button's name beside its mark + label_pad: float = 4 + label_from: float = 16 # a button's name, right and down from its click + label_alpha: float = 0.9 + least_text: int = 9 # the smallest text, in pixels, on any frame + + +STYLE = Style() +#: A chip's padding above and below its text, as a share of the padding beside it. +CHIP_PAD_HIGH = 0.7 +WHITE = (255, 255, 255) + + +def _width(width: float, unit: float) -> int: + return max(1, round(width * unit)) + + +def _draw_click(pen, mark, points, colour, unit, age, fade) -> None: + x, y = points[0] + grow = min(1.0, age / STYLE.ring_grow_s) + alpha = round(255 * fade) + for ring in range(max(1, min(mark.count, STYLE.most_rings))): + r = unit * (STYLE.ring_from + STYLE.ring_grows * grow + STYLE.ring_apart * ring) + box = (x - r, y - r, x + r, y + r) + shape = pen.rectangle if mark.button == "right" else pen.ellipse + shape(box, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + shape(box, outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) + dot = unit * STYLE.dot + pen.ellipse((x - dot, y - dot, x + dot, y + dot), outline=(*colour, alpha), width=_width(STYLE.dot_stroke, unit)) + if mark.button not in ("left", "right"): + _label(pen, (x + unit * STYLE.label_from, y + unit * STYLE.label_from), mark.button, colour, unit, alpha) + + +def _draw_press(pen, mark, points, colour, unit, age, fade) -> None: + x, y = points[0] + alpha = round(255 * fade) + down = mark.kind == "pointer_down" + r = unit * (STYLE.press_down if down else STYLE.press_up) + fill = (*colour, round(alpha * STYLE.press_fill)) if down else None + pen.ellipse((x - r, y - r, x + r, y + r), fill=fill, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + pen.ellipse((x - r, y - r, x + r, y + r), outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) + + +def _draw_move(pen, mark, points, colour, unit, age, fade) -> None: + x, y = points[0] + alpha = round(STYLE.cross_alpha * fade) + arm = unit * STYLE.cross + for dx, dy in ((arm, 0), (0, arm)): + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) + + +def _draw_scroll(pen, mark, points, colour, unit, age, fade) -> None: + x, y = points[0] + alpha = round(255 * fade) + dx, dy = {"up": (0, -1), "down": (0, 1), "left": (-1, 0), "right": (1, 0)}.get(mark.direction, (0, 1)) + end = (x + dx * unit * STYLE.arrow, y + dy * unit * STYLE.arrow) + for width, fill in ((STYLE.line_halo, (*WHITE, alpha)), (STYLE.line, (*colour, alpha))): + pen.line((x, y, *end), fill=fill, width=_width(width, unit)) + _arrowhead(pen, (x, y), end, unit, fill, width) + _label(pen, (end[0] + unit * STYLE.label_pad * 2, end[1] + unit * STYLE.label_pad), f"x{mark.count}", colour, + unit, alpha) + + +def _draw_drag(pen, mark, points, colour, unit, age, fade) -> None: + alpha = round(255 * fade) + shown = points if age >= mark.moving_s else _along(points, age / mark.moving_s) + for width, fill in ((STYLE.line_halo, (*WHITE, alpha)), (STYLE.line, (*colour, alpha))): + if len(shown) > 1: + pen.line([coord for point in shown for coord in point], fill=fill, width=_width(width, unit), + joint="curve") + _arrowhead(pen, shown[-2], shown[-1], unit, fill, width) + sx, sy = points[0] + r = unit * STYLE.start_dot + pen.ellipse((sx - r, sy - r, sx + r, sy + r), fill=(*colour, alpha), outline=(*WHITE, alpha), + width=_width(STYLE.outline, unit)) + + +#: The usual arrow cursor, its tip at (0, 0), in pixels of a UNIT_EDGE frame. +ARROW = ((0, 0), (0, 17), (4, 13), (7, 20), (10, 19), (7, 12), (12, 12)) + + +def _draw_pointer(pen, move: PointerMove, was, path, colour, unit, age) -> None: + """A pointer: along a drag's path while it moves, else where its input left it, after a fading line from where + it jumped; fainter when it didn't move last, and named when it is one of several.""" + tip = _along(path, age / move.moving_s)[-1] if len(path) > 1 and age < move.moving_s else path[-1] + if age < JUMP_S and math.dist(was, path[0]) > unit * STYLE.cross: + alpha = round(STYLE.jump_alpha * (1 - age / JUMP_S)) + pen.line((*was, *path[0]), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) + scale = unit * STYLE.pointer + arrow = [(tip[0] + dx * scale, tip[1] + dy * scale) for dx, dy in ARROW] + alpha = STYLE.pointer_faded_alpha if move.faded else 255 + pen.polygon(arrow, fill=(*colour, alpha), outline=(*WHITE, alpha), width=_width(STYLE.pointer_outline, unit)) + if move.label: + corner = max(dx for dx, _dy in ARROW), max(dy for _dx, dy in ARROW) + _label(pen, (tip[0] + corner[0] * scale, tip[1] + corner[1] * scale), move.label, colour, unit, alpha) + + +#: How each kind of mark drawn at a point is drawn, by its operation. +_DRAWERS = {"click": _draw_click, "pointer_down": _draw_press, "pointer_up": _draw_press, + "pointer_move": _draw_move, "scroll": _draw_scroll, "drag": _draw_drag} + + +def _draw_chips(pen, chips, size, unit) -> None: + """Chips stacked centred above the bottom edge, clear of a panel or dock there, the newest lowest.""" + font = _font(max(STYLE.least_text, round(STYLE.chip_text * unit))) + pad, high, gap = unit * STYLE.chip_pad, unit * STYLE.chip_pad * CHIP_PAD_HIGH, unit * STYLE.chip_gap + y = size[1] - max(unit * STYLE.chip_low, size[1] * CHIPS_ABOVE) + for mark, colour in reversed(chips): + left, top, right, bottom = pen.textbbox((0, 0), mark.label, font=font) + w, h = right - left + 2 * pad, bottom - top + 2 * high + x0, y0 = (size[0] - w) / 2, y - h + pen.rounded_rectangle((x0, y0, x0 + w, y), radius=unit * STYLE.chip_radius, fill=(*colour, STYLE.chip_alpha), + outline=(*WHITE, STYLE.chip_alpha), width=_width(STYLE.outline, unit)) + pen.text((x0 + pad - left, y0 + high - top), mark.label, font=font, fill=_ink(colour)) + y = y0 - gap + + +def _label(pen, at: tuple[float, float], text: str, colour, unit, alpha) -> None: + font = _font(max(STYLE.least_text, round(STYLE.label_text * unit))) + left, top, right, bottom = pen.textbbox(at, text, font=font) + pad = unit * STYLE.label_pad + pen.rounded_rectangle((left - pad, top - pad, right + pad, bottom + pad), radius=pad, + fill=(*colour, round(alpha * STYLE.label_alpha))) + pen.text(at, text, font=font, fill=(*_ink(colour)[:3], alpha)) + + +def _arrowhead(pen, start, end, unit, fill, width) -> None: + angle = math.atan2(end[1] - start[1], end[0] - start[0]) + size = unit * (STYLE.arrowhead + width) + sides = [(end[0] - size * math.cos(angle + turn), end[1] - size * math.sin(angle + turn)) + for turn in (-STYLE.arrowhead_angle, STYLE.arrowhead_angle)] + pen.polygon([end, *sides], fill=fill) + + +def _along(points: list[tuple[float, float]], fraction: float) -> list[tuple[float, float]]: + """The first ``fraction`` of the path through ``points``, by length.""" + legs = [math.dist(a, b) for a, b in zip(points, points[1:])] + left = sum(legs) * max(0.0, min(1.0, fraction)) + shown = [points[0]] + for (a, b), leg in zip(zip(points, points[1:]), legs): + if left >= leg: + shown.append(b) + left -= leg + continue + t = left / leg if leg else 0.0 + shown.append((a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t)) + break + return shown + + +def _pointer_label(move: PointerMove) -> str: + """A pointer's name beside it: its role's, the agent's as agent, and its own when it isn't the default.""" + role = "agent" if move.role == DEFAULT_ROLE else move.role + return role if move.pointer == DEFAULT_POINTER else f"{role} {move.pointer}" + + +def _ink(colour: tuple[int, int, int]) -> tuple[int, int, int, int]: + """Black or white, whichever reads on ``colour``.""" + r, g, b = colour + return (20, 20, 20, 255) if 0.299 * r + 0.587 * g + 0.114 * b > 160 else (255, 255, 255, 255) + + +def _chord(keys: Sequence[str]) -> str: + return "+".join(KEY_NAMES.get(key, key.upper() if len(key) == 1 else key) for key in keys) + + +def _shown_text(text: str) -> str: + return _cut(text.replace("\n", "[Enter]").replace("\t", "[Tab]")) + + +def _cut(text: str) -> str: + return text if len(text) <= CHIP_CHARS else text[:CHIP_CHARS - 1] + "…" + + +@functools.lru_cache(maxsize=16) +def _font(size: int) -> Any: + return ImageFont.load_default(size=size) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index a2f13c1da..27da96e12 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -6,15 +6,17 @@ import asyncio import base64 import contextlib +import io import json import math import time import uuid from collections.abc import Awaitable, Callable -from datetime import datetime +from datetime import datetime, timedelta from importlib import resources from typing import Any +import av from mcp import ClientSession from mcp.types import CallToolResult from pydantic import ValidationError @@ -36,7 +38,9 @@ GROUP_TOOLS, POINT_TOOLS, ) -from ..manifest import RecordingManifest +from ..annotate import COLOURS as MARK_COLOURS +from ..annotate import UNIT_EDGE +from ..manifest import ManifestSegment, RecordingManifest from ..recording import RECORDINGS_DIR from .client import Driver, image from .mp4 import read_mp4 @@ -64,6 +68,18 @@ NORM = 1000 #: How long the probe records, and the frame rate it asks for. RECORD_S = 2.0 +#: How long an annotated recording runs before and after the probe's click, so it keeps frames on both sides. +ANNOTATE_LEAD_S, ANNOTATE_TAIL_S = 1.0, 1.0 +#: How far, as the sum of its channels' differences, an encoded pixel may be from the colour drawn and still count. +NEAR_COLOUR = 90 +#: How many pixels near the mark's colour around the click make a frame draw its mark. +MARK_PIXELS = 20 +#: How far around the click, in pixels of a frame whose longer edge is the marks' UNIT_EDGE, the mark is looked for; +#: and the least, on a smaller one. +MARK_REACH, MARK_REACH_LEAST = 40, 8 +#: Where in the clicked cell its colour is read, left and up from its centre as a share of the cell: clear of the +#: mark's dot and of a pointer at the click, real or drawn, which points from there down and right. +CELL_INSIDE = 0.36 #: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. BUSY_DESK_ATTEMPTS = 2 @@ -164,6 +180,7 @@ async def full(self) -> None: for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording), + ("X", self._annotation), ("X", self._page_tools)): await self._guarded(check, section) finally: @@ -1083,12 +1100,48 @@ async def limited() -> str | None: await self.check("X", "a repeated request_id joins the first recording", joined, COMPUTER_RECORDING) await self.check("X", "a recording past max_concurrent is refused", limited, COMPUTER_RECORDING) finally: - for recording_id in made: - with contextlib.suppress(Exception): - await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) - await self.driver.run(*(WIN_REMOVE if self.windows else ("rm", "-rf")), f"{RECORDINGS_DIR}/{recording_id}") - await self.driver.run(*((*WIN_RMDIR, RECORDINGS_DIR) if self.windows else - ("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true"))) + await self._remove_recordings(made) + + async def _annotation(self) -> None: + if not self.driver.declares(COMPUTER_RECORDING): + return + params = self.driver.params(COMPUTER_RECORDING) + fps = min(RECORD_FPS, (params.get("fps") or {}).get("max") or RECORD_FPS) + made: list[str] = [] + + async def annotated() -> str | None: + """A recording with annotate draws the probe's click where it landed, from no later than the frame that + first shows the click's effect, and its first frame, captured before the click, draws none.""" + if "annotate" not in params: + raise NotApplicable("the card lists no annotate in computer-recording/v1's params") + started = await self.driver.invoke(COMPUTER_RECORDING, "start", { + "request_id": f"{PROBE_ROLE}-annotated-{uuid.uuid4().hex[:8]}", "fps": fps, "annotate": {}}) + made.append(started["recording_id"]) + await asyncio.sleep(ANNOTATE_LEAD_S) + frame, scale, img = await self._base() + size = self.layout["cell"] + x, y = self._at("grid", scale, size + size / 2, 6 * size + size / 2) + inside = (x - math.floor(size * CELL_INSIDE * scale), y - math.floor(size * CELL_INSIDE * scale)) + was = img.getpixel(inside) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + await asyncio.sleep(ANNOTATE_TAIL_S) + wire = await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": started["recording_id"]}) + return await self._annotated_click(wire, img.size, (x, y), inside, was) + + try: + await self.check("X", "an annotated recording draws a click where it landed, from the frame its effect " + "shows on", annotated, COMPUTER_RECORDING) + finally: + await self._remove_recordings(made) + + async def _remove_recordings(self, made: list[str]) -> None: + """End each recording the probe started, and remove what it kept on the computer.""" + for recording_id in made: + with contextlib.suppress(Exception): + await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) + await self.driver.run(*(WIN_REMOVE if self.windows else ("rm", "-rf")), f"{RECORDINGS_DIR}/{recording_id}") + await self.driver.run(*((*WIN_RMDIR, RECORDINGS_DIR) if self.windows else + ("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true"))) async def _segments_match(self, wire: dict, fps: int) -> str | None: """What is wrong with a recording without a grant, read back from the computer, or None.""" @@ -1110,22 +1163,9 @@ async def _segments_match(self, wire: dict, fps: int) -> str | None: if gap > 2 / fps: return f"{gap:.2f} s between segments {before.index} and {after.index}" for segment in segments: - path = f"{RECORDINGS_DIR}/{segment.path}" - out = await self.driver.run("python3", "-c", READ_DIGEST, path) - if out.get("exit_code") != 0: - return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}" - file = json.loads(out["stdout"]) - if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256): - return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ - f"the manifest says {segment.bytes}, {segment.sha256[:12]}" - data = b"" - while len(data) < file["size"]: - chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) - read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" - if not read: - return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \ - f"{chunk.get('stderr', '')[-200:]}" - data += read + data = await self._read_back(segment) + if isinstance(data, str): + return data mp4 = read_mp4(data) span = (_when(segment.ended_at) - _when(segment.started_at)).total_seconds() if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment.width, segment.height, segment.frames) @@ -1135,6 +1175,85 @@ async def _segments_match(self, wire: dict, fps: int) -> str | None: return None + async def _read_back(self, segment: ManifestSegment) -> bytes | str: + """A segment a recording without a grant kept on the computer, read back and matched to its manifest entry's + size and digest; what is wrong, as a string, when it can't be.""" + path = f"{RECORDINGS_DIR}/{segment.path}" + out = await self.driver.run("python3", "-c", READ_DIGEST, path) + if out.get("exit_code") != 0: + return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}" + file = json.loads(out["stdout"]) + if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256): + return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ + f"the manifest says {segment.bytes}, {segment.sha256[:12]}" + data = b"" + while len(data) < file["size"]: + chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) + read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" + if not read: + return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \ + f"{chunk.get('stderr', '')[-200:]}" + data += read + return data + + async def _annotated_click(self, wire: dict, size: tuple[int, int], point: tuple[int, int], + inside: tuple[int, int], was: tuple[int, int, int]) -> str | None: + """What is wrong with an annotated recording of one click at frame pixel ``point`` of a frame ``size``, the + clicked cell showing ``was`` at ``inside`` before it, or None; NotApplicable for a recording of a native + recorder's finished segments, which the base class keeps as they came. All its times are the computer's own.""" + manifest = RecordingManifest.model_validate(wire) + if manifest.annotate is None: + raise NotApplicable("its manifest names no annotate: the driver's recorder hands over finished segments, " + "which aren't drawn on") + if manifest.annotate != {}: + return f"its manifest's annotate is {manifest.annotate!r}, not the {{}} it was started with" + + def at(xy: tuple[int, int], shape: tuple[int, int]) -> tuple[int, int]: + return (min(shape[0] - 1, xy[0] * shape[0] // size[0]), min(shape[1] - 1, xy[1] * shape[1] // size[1])) + + def marked(picture: Any, shape: tuple[int, int]) -> bool: + cx, cy = at(point, shape) + reach = max(MARK_REACH_LEAST, round(MARK_REACH * max(shape) / UNIT_EDGE)) + near = sum(1 for px in range(max(0, cx - reach), min(shape[0], cx + reach)) + for py in range(max(0, cy - reach), min(shape[1], cy + reach)) + if _distance(picture.getpixel((px, py)), MARK_COLOURS[0]) < NEAR_COLOUR) + return near > MARK_PIXELS + + def changed(picture: Any, shape: tuple[int, int]) -> bool: + return _distance(picture.getpixel(at(inside, shape)), was) >= NEAR_COLOUR + + frames: list[tuple[datetime, bool, bool]] = [] # each frame's capture time, whether it draws the mark and + for segment in manifest.segments: # whether it shows the click's effect; not the frames, which run large + data = await self._read_back(segment) + if isinstance(data, str): + return data + with av.open(io.BytesIO(data)) as container: + first: float | None = None + for f in container.decode(container.streams.video[0]): + first = f.time if first is None else first + picture, shape = f.to_image(), (f.width, f.height) + frames.append((_when(segment.started_at) + timedelta(seconds=f.time - first), + marked(picture, shape), changed(picture, shape))) + if not frames: + return "it kept no frames" + first_marked = next((i for i, (_, mark, _) in enumerate(frames) if mark), None) + first_changed = next((i for i, (_, _, change) in enumerate(frames) if change), None) + if first_changed is None: + return f"the clicked cell never changed in its {len(frames)} frames, so the click can't be placed on them" + if first_marked is None: + return f"none of its {len(frames)} frames draws the click's mark at {point}" + if first_marked == 0: + return "its first frame, captured before the click, already draws a mark there" + if first_marked > first_changed: + late = (frames[first_marked][0] - frames[first_changed][0]).total_seconds() + return f"the mark first shows {late:.2f} s after the frame where the click's effect does" + return None + + +def _distance(a: tuple, b: tuple) -> int: + return sum(abs(x - y) for x, y in zip(a[:3], b[:3])) + + def _when(iso: str) -> datetime: return datetime.fromisoformat(iso.replace("Z", "+00:00")) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index bd5dc6856..4c9437787 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -6,6 +6,7 @@ import asyncio import logging import os +from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional @@ -14,12 +15,12 @@ from starlette.responses import Response from starlette.websockets import WebSocket -from ..agent_env_environment import (_CARD_CONFIG_ATTR, _TOOL_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication, - AgentEnvError, _error_response, _marked_members, add_websocket_route, - control_check, environment_card) +from ..agent_env_environment import (_CARD_CONFIG_ATTR, _ENV_NAME_TOKEN, _TOOL_ATTR, _TOOL_OPTIONS_ATTR, + AgentEnvEnvironment, AgentEnvFastMCPApplication, AgentEnvError, _error_response, + _marked_members, add_websocket_route, control_check, environment_card) from ..tool_context import ToolContext from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool -from . import _linux, apps, card, svc +from . import _linux, annotate, apps, card, svc from .checks import check_features, check_names from .dataplane import DataPlane from .errors import TargetUnavailable @@ -32,6 +33,7 @@ from .settings import TOOL_PREFIX_ENV, ComputerSettings, load_settings, tool_prefix_problem from .tools import COMPUTER_TOOLS, Tools +_log = logging.getLogger(__name__) #: The caller every call comes from until agents carry an identity of their own. DEFAULT_IDENTITY = "default" @@ -127,11 +129,36 @@ def declaration(cls, platform: Platform) -> EnvironmentCard: return AgentEnvFastMCPApplication(card, driver).environment_card def on_tool_call(self, context: ToolContext) -> Any: - """Note that an agent has acted, for the apps whose loads change what's on screen only before one has: a - subclass that has its own calls this one too.""" + """Note that an agent has acted, for the apps whose loads change what's on screen only before one has, and + draw a call to one of the driver's own or forwarded tools that may act on the recordings that draw inputs, + since it acts without ``act``, which marks the SDK's: a subclass that has its own calls this one too.""" apps.note_agent_acted(self._computer.apps) + if self._computer.recordings.annotating(): + try: + name = self._computer_prefix.own(context.tool) + if name not in COMPUTER_TOOLS and self._computer_tool_acts(name): + self._computer.recordings.note([annotate.tool_mark(name, context.arguments, role=context.role, + at=datetime.now(timezone.utc))]) + except Exception as e: # a mark is best effort: the call goes ahead without one + _log.warning("couldn't mark %s on an annotated recording: %s: %s", context.tool, type(e).__name__, e) return None + def _computer_tool_acts(self, name: str) -> bool: + """Whether the driver's own or forwarded tool ``name`` may act: its annotations don't say it only reads. A + tool named with ``{environment_name}`` is served under that name filled in from the card, as the SDK fills + it in when it mounts the tools.""" + acting = self.__dict__.get("_computer_acting") + if acting is None: + acting = {forwarded.tool.name: not _reads_only(forwarded.annotations) + for forwarded in self.forwarded_tools()} + environment_name = self.mounted_card().name + for member, marker in _marked_members(self, _TOOL_ATTR): + options = getattr(member, _TOOL_OPTIONS_ATTR, None) + acting[marker.name.replace(_ENV_NAME_TOKEN, environment_name)] = not _reads_only( + options.annotations if options is not None else None) + self._computer_acting = acting + return acting.get(name, True) + def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" return self._computer.has(capability) @@ -259,3 +286,10 @@ def deco(cls: type) -> type: cls.supports = list(supports) return environment_card(name=name)(cls) if name else cls return deco + + +def _reads_only(annotations: Any) -> bool: + """Whether a tool's MCP annotations, as a dict or a ``ToolAnnotations``, say it only reads.""" + if isinstance(annotations, dict): + return annotations.get("readOnlyHint") is True + return getattr(annotations, "readOnlyHint", None) is True diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 0dde59926..f3bf47361 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -129,12 +129,14 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st @extension(COMPUTER_RECORDING, op="start", description="Record the display: start returns a recording whose segments upload through its grant as " - "they close; end seals it and returns its manifest, uploaded beside them; end_all ends " - "every one not yet ended; list names those still recording; repeats answer the same.") + "they close, with annotate drawing each input over the frames; end seals it and returns " + "its manifest, uploaded beside them; end_all ends every one not yet ended; list names " + "those still recording; repeats answer the same.") async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, - upload: Optional[dict] = None) -> dict: - return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload) + upload: Optional[dict] = None, annotate: Optional[dict] = None) -> dict: + return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload, + annotate) @extension(COMPUTER_RECORDING, op="end") async def _computer_recording_end(self, recording_id: str, upload: Optional[dict] = None, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py index a5b851d97..f0a49a025 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py @@ -44,6 +44,8 @@ class RecordingManifest(BaseModel): ended_by: EndedBy segments: list[ManifestSegment] error: Optional[str] = None + #: The ``annotate`` the recording drew its inputs with; none for a plain recording. + annotate: Optional[dict] = None def wire(self) -> dict: """As JSON goes on the wire, without the fields it doesn't know.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index a6c7c846a..28d960194 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -28,6 +28,7 @@ from pydantic import ValidationError from ..transfers import NamespaceUploader, WriteNamespaceGrant +from . import annotate as annotating from .errors import InvalidArgument, LimitExceeded, Unsupported from .frames import to_pil from .manifest import ManifestSegment, RecordingManifest @@ -126,7 +127,8 @@ def recording_params(recording: Recording, streams: bool) -> dict: most = recording.max_fps if streams else min(recording.max_fps, MAX_POLLED_FPS) return {"formats": [MP4], "scopes": ["display"], "fps": {"default": min(DEFAULT_FPS, most), "max": most}, "cursor": True, "source": "stream" if streams else "capture", "max_segment_s": recording.max_segment_s, - "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s} + "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s, + "annotate": {"options": list(annotating.AnnotateOptions.model_fields)}} class Recordings: @@ -142,7 +144,8 @@ def __init__(self, runtime: ComputerRuntime) -> None: self._lock = asyncio.Lock() async def start(self, request_id: str, scope: str, window: Optional[str], fps: Optional[int], - max_edge: Optional[int], cursor: bool, upload: Optional[dict]) -> dict: + max_edge: Optional[int], cursor: bool, upload: Optional[dict], + annotate: Optional[dict] = None) -> dict: features: Recording = self.runtime.driver.recording if not (isinstance(request_id, str) and request_id): raise InvalidArgument("request_id is a non-empty string") @@ -152,6 +155,7 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O await self._live(self.running.get(answer["recording_id"])) return answer hints, grant = self._hints(features, scope, window, fps, max_edge, cursor, upload) + annotate = annotating.check_annotate(annotate) stopped = [rid for rid, r in self.running.items() if r.done] for rid in stopped[:max(0, len(stopped) - KEEP_ENDED)]: # stopped by itself and never ended self.running.pop(rid) @@ -166,7 +170,8 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O cursor = hints.cursor if streams else hints.cursor and runtime.has("pointer") source = (lambda hints: runtime.stream(hints)) if streams else \ (lambda hints: _poll(runtime, hints, cursor)) - recording = Video(recording_id, hints, features, root, grant, source, streams=streams, cursor=cursor) + recording = Video(recording_id, hints, features, root, grant, source, streams=streams, cursor=cursor, + annotate=annotate) self.running[recording_id] = recording recording.begin() answer = {"recording_id": recording_id, "started_at": _iso(recording.requested_at), "format": MP4} @@ -203,6 +208,16 @@ async def listed(self) -> list[dict]: return [{"recording_id": rid, "started_at": _iso(r.requested_at)} for rid, r in self.running.items() if not r.done] + def annotating(self) -> bool: + """Whether any recording still recording draws its inputs.""" + return any(r.annotations is not None and not r.done for r in self.running.values()) + + def note(self, marks: list[annotating.Mark]) -> None: + """Draw ``marks`` on every recording still recording that draws its inputs.""" + for recording in self.running.values(): + if recording.annotations is not None and not recording.done: + recording.annotations.add(marks) + async def end_all(self, reason: str) -> list[dict]: """End every recording not yet ended, each uploading under the grant it was started with; their manifests.""" return [await self.end(recording_id, None, reason) for recording_id in list(self.running)] @@ -227,11 +242,14 @@ class Video: """One recording's video: frames or segments from ``source(hints)``, encoded as H.264 segments in ``hints.directory``, each uploaded under ``grant`` as it closes, at ``/.mp4``, ``prefix`` its id by default; sealed by ``end``, which answers its manifest and, with ``manifest``, uploads it beside the segments. - ``streams`` says the source streams rather than polls, ``cursor`` that its frames draw the pointer.""" + ``streams`` says the source streams rather than polls, ``cursor`` that its frames draw the pointer, and + ``annotate``, when given, that each frame draws the inputs it shows (``annotate.Annotations``); a segment a native + recorder finished is kept as it came, so a recording that took one names no ``annotate`` in its manifest.""" def __init__(self, recording_id: str, hints: RecordHints, features: Recording, root: Path, grant: Optional[WriteNamespaceGrant], source: Callable[[RecordHints], AsyncIterator], *, - streams: bool, cursor: bool, prefix: Optional[str] = None, manifest: bool = True) -> None: + streams: bool, cursor: bool, prefix: Optional[str] = None, manifest: bool = True, + annotate: Optional[dict] = None) -> None: self.id = recording_id self.hints = hints self.features = features @@ -243,6 +261,9 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.requested_at = datetime.now(timezone.utc) self.streams = streams self.cursor = cursor + self.annotate = annotate + self.annotations = annotating.Annotations(pointer=hints.cursor and not cursor) if annotate is not None else None + self.native = False self.segments: list[_Segment] = [] self.encoder: Optional[_Encoder] = None self.ended_by: Optional[str] = None @@ -341,15 +362,17 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: recording_id=self.id, scope=self.hints.scope, format=MP4, fps=self.hints.fps, cursor=self.cursor, source="stream" if self.streams else "capture", started_at=entries[0].started_at if entries else None, ended_at=entries[-1].ended_at if entries else None, ended_by=self.ended_by, segments=entries, - error=self.error if self.ended_by == "error" else None).wire() + error=self.error if self.ended_by == "error" else None, + annotate=None if self.native else self.annotate).wire() if uploader is not None and self.uploads_manifest: # beside the segments, so a reader who didn't end the with contextlib.suppress(Exception): # recording finds it await uploader.upload(f"{self.prefix}/{MANIFEST}", json.dumps(manifest).encode()) return manifest async def _add_frame(self, item: RecordedFrame) -> None: - frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge) at = item.captured_at + marks = self.annotations.showing(at) if self.annotations is not None else [] + frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge, marks) if self.encoder is not None and ((frame.width, frame.height) != self.encoder.size or (at - self.encoder.last_at).total_seconds() > GAP_S or (at - self.encoder.started_at).total_seconds() >= self.hints.max_segment_s): @@ -362,6 +385,7 @@ async def _add_frame(self, item: RecordedFrame) -> None: self._check_disk() async def _add_segment(self, item: RecordedSegment) -> None: + self.native = True index = len(self.segments) target = self.hints.directory / f"{index:04d}.mp4" if Path(item.path) != target: @@ -528,8 +552,10 @@ def _draw_pointer(image: PILImage.Image, x: float, y: float) -> None: ImageDraw.Draw(image).polygon([(x + dx, y + dy) for dx, dy in arrow], fill=(0, 0, 0), outline=(255, 255, 255)) -def _video_frame(image: Any, max_edge: Optional[int]) -> Any: - """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``.""" +def _video_frame(image: Any, max_edge: Optional[int], marks: list = ()) -> Any: + """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``, with ``marks`` drawn.""" + if marks: + image = annotating.draw(image, marks) frame = image if isinstance(image, av.VideoFrame) else av.VideoFrame.from_image(image.convert("RGB")) width, height = frame.width, frame.height if max_edge is not None and max(width, height) > max_edge: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 2cbf236cb..855b8b03a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -5,15 +5,17 @@ import asyncio import contextlib +import logging from collections import OrderedDict from collections.abc import AsyncIterator +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Awaitable, Callable from mcp.types import CallToolResult from ..errors import AgentEnvError from ..tool_context import ToolContext -from . import apps +from . import annotate, apps from .errors import DeliveryFailed, DeviceUnavailable, DialogOpen, InputBusy, Internal from .frames import Frame, FrameStore, to_pil from .operations import DIALOG_BLOCKS, GROUPS, INPUT, Display, Image, PageState, ReadyCheck, resolve @@ -21,6 +23,8 @@ from .recording import RecordHints, Recordings from .results import EffectData, InputResult, send +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from .driver import ComputerDriverEnvironment @@ -117,13 +121,17 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra if operation in DIALOG_BLOCKS and self.has("page") and (dialog := (await self.page_now()).dialog): raise DialogOpen(f'a {dialog.type} dialog is open on the page: "{dialog.message}"; answer it with ' f"{self.driver._computer_prefix.served('computer_dialog')} first") + display = None if mapped and frame is not None: - self.frames.check_current(frame, await self.display_now()) + display = await self.display_now() + self.frames.check_current(frame, display) if at_pointer: sx, sy = await self.pointer() args, summary = (sx, sy, *args), f"{summary} at the pointer, screen ({sx}, {sy})" others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None + if self.recordings.annotating(): + await self._annotate(ctx, operation, args, move_to, repeat, hold, display) try: if move_to is not None: await self.call("pointer_move", *move_to) @@ -156,6 +164,24 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result() return send(result, new.encode(), new.captured_at) + async def _annotate(self, ctx: ToolContext, operation: str, args: tuple, move_to: tuple | None, repeat: int, + hold: tuple | None, display: Display | None) -> None: + """Mark an input on the recordings that draw them, as it is about to be delivered: one with a point on the + display it acts on, ``display`` when the call already read it, a press or release with none at the pointer. + Best effort: the input goes ahead whether or not its mark could be made.""" + try: + point, size = move_to, (1, 1) + if operation in annotate.POINTED: + display = display or await self.display_now() + size = (display.width, display.height) + if point is None and operation in ("pointer_down", "pointer_up") and self.has("pointer"): + point = await self.pointer() + self.recordings.note(annotate.input_marks( + operation, args, role=ctx.role, at=datetime.now(timezone.utc), size=size, point=point, + repeat=repeat, hold_ms=hold[0] if hold is not None else None)) + except Exception as e: + logger.warning("couldn't mark %s on an annotated recording: %s: %s", operation, type(e).__name__, e) + async def _hold_for(self, ms: int, keys: list[str]) -> None: """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" release = ("key_up", list(reversed(keys))) diff --git a/packages/agentenv-protocol/tests/computer_fake.py b/packages/agentenv-protocol/tests/computer_fake.py index 6133104cb..7e526c98d 100644 --- a/packages/agentenv-protocol/tests/computer_fake.py +++ b/packages/agentenv-protocol/tests/computer_fake.py @@ -15,13 +15,22 @@ class with no display. A click paints its button's colour at its point; everythi ComputerSettings, Display, Effect, + FileChooser, + Image, + PageDialog, + PageState, Platform, + Tab, capture, click, + dialog, display, drag, + history, key_down, key_up, + navigate, + page_state, pointer_down, pointer_move, pointer_position, @@ -29,7 +38,9 @@ class with no display. A click paints its button's colour at its point; everythi press, scroll, set_display, + tabs, type_text, + upload, ) LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") @@ -162,6 +173,54 @@ async def keys_up(self, keys): self.screen.held = [k for k in self.screen.held if k not in keys] +class PageFake(FakeDriver): + """A fake whose screen is a browser's page, with two tabs, the first active, and a log of the page calls.""" + + def __init__(self) -> None: + super().__init__() + self.tabs = [Tab("a", "https://mail.test/", "Inbox", True), Tab("b", "https://docs.test/", "Docs")] + self.open_dialog: PageDialog | None = None + self.chooser: FileChooser | None = None + self.calls: list = [] + self.captured: PageState | None = None + + @capture + async def grab(self): + image = await super().grab() + if self.captured is None: + return image + return Image(image.width, image.height, image.tobytes(), "RGB", self.captured) + + @navigate + async def go_to(self, url): + self.calls.append(("navigate", url)) + + @history + async def step(self, direction): + self.calls.append(("history", direction)) + + @tabs + async def tab_action(self, action, tab, url): + self.calls.append(("tabs", action, tab, url)) + if action == "switch": + self.tabs = [Tab(t.id, t.url, t.title, t.id == tab) for t in self.tabs] + return self.tabs + + @dialog + async def answer(self, accept, text): + self.calls.append(("dialog", accept, text)) + self.open_dialog = None + + @upload + async def choose(self, paths): + self.calls.append(("upload", paths)) + + @page_state + async def page_now(self): + active = next(t for t in self.tabs if t.active) + return PageState(active.id, active.url, active.title, self.open_dialog, self.chooser) + + class MinimalDriver(ComputerDriverEnvironment): """Only the required operations, synchronous ones: no pointer or keyboard group, no display modes.""" diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index 6ce87bd3a..751293485 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -40,6 +40,8 @@ ) from agentenv_protocol.computer import _linux as linux from agentenv_protocol.computer.conformance.checks import OWNER, STAGE +from agentenv_protocol.computer.conformance.desk import CELL +from agentenv_protocol.computer.conformance.desk import COLOURS as CELL_COLOURS from agentenv_protocol.computer.conformance.page_fixture import QUESTION, TITLE, TWO, TWO_TITLE CLIPBOARD = "agentenv-probe clipboard" @@ -94,8 +96,10 @@ def click(self, x: float, y: float, button: str, count: int) -> None: if not self.started: return if (at := self._in("grid", x, y)) is not None: - key = f"{int(at[1] // 50)},{int(at[0] // 50)}" + row, col = int(at[1] // CELL), int(at[0] // CELL) + key = f"{row},{col}" self.state["cells"][key] = (self.state["cells"].get(key, 0) + count) % 4 + self._fill_cell(row, col, CELL_COLOURS[self.state["cells"][key]]) self.state["right_clicks"] += button == "right" self.state["double_clicks"] += count >= 2 elif self._in("save", x, y) is not None: @@ -106,6 +110,16 @@ def click(self, x: float, y: float, button: str, count: int) -> None: if self._in(box, x, y) is not None: self.focus = box + def _fill_cell(self, row: int, col: int, colour: str) -> None: + """Repaint a grid cell its new colour, as the Fixture Desk does, but for the pixel the click painted.""" + x0, y0 = self.layout["grid"][0] + col * CELL, self.layout["grid"][1] + row * CELL + rgb = (int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16)) + pointer = self.driver.screen.pointer + for x in range(x0 + 1, x0 + CELL - 1): + for y in range(y0 + 1, y0 + CELL - 1): + if (x, y) != tuple(pointer): + self.driver.screen.paint(x, y, rgb) + def type(self, text: str) -> None: if self.focus is None: return diff --git a/packages/agentenv-protocol/tests/test_computer_page.py b/packages/agentenv-protocol/tests/test_computer_page.py index f417cbb30..862d86a9f 100644 --- a/packages/agentenv-protocol/tests/test_computer_page.py +++ b/packages/agentenv-protocol/tests/test_computer_page.py @@ -8,74 +8,17 @@ pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") -from computer_fake import FakeDriver, http_client, mcp_session # noqa: E402 +from computer_fake import FakeDriver, PageFake, http_client, mcp_session # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 FileChooser, - Image, PageDialog, PageState, - Tab, - capture, - dialog, - history, - navigate, - page_state, - tabs, - upload, ) PAGE_TOOLS = {"computer_navigate", "computer_history", "computer_tabs", "computer_dialog", "computer_upload"} -class PageFake(FakeDriver): - """A fake whose screen is a browser's page, with two tabs, the first active, and a log of the page calls.""" - - def __init__(self) -> None: - super().__init__() - self.tabs = [Tab("a", "https://mail.test/", "Inbox", True), Tab("b", "https://docs.test/", "Docs")] - self.open_dialog: PageDialog | None = None - self.chooser: FileChooser | None = None - self.calls: list = [] - self.captured: PageState | None = None - - @capture - async def grab(self): - image = await super().grab() - if self.captured is None: - return image - return Image(image.width, image.height, image.tobytes(), "RGB", self.captured) - - @navigate - async def go_to(self, url): - self.calls.append(("navigate", url)) - - @history - async def step(self, direction): - self.calls.append(("history", direction)) - - @tabs - async def tab_action(self, action, tab, url): - self.calls.append(("tabs", action, tab, url)) - if action == "switch": - self.tabs = [Tab(t.id, t.url, t.title, t.id == tab) for t in self.tabs] - return self.tabs - - @dialog - async def answer(self, accept, text): - self.calls.append(("dialog", accept, text)) - self.open_dialog = None - - @upload - async def choose(self, paths): - self.calls.append(("upload", paths)) - - @page_state - async def page_now(self): - active = next(t for t in self.tabs if t.active) - return PageState(active.id, active.url, active.title, self.open_dialog, self.chooser) - - @pytest.mark.asyncio async def test_a_page_driver_declares_computer_page_and_serves_its_tools_and_a_desktop_driver_neither(): async with http_client(PageFake()) as http: diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 59a3badb8..0c6fb236d 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -140,7 +140,8 @@ class Off(FakeDriver): assert set(extension["params"]["methods"]) == {"start", "end", "end_all", "list"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, - "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400} + "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400, + "annotate": {"options": []}} assert COMPUTER_RECORDING not in [e["uri"] for e in off["capabilities"]["extensions"]] diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py new file mode 100644 index 000000000..b5f527582 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -0,0 +1,474 @@ +"""computer-recording/v1 ``annotate``: a recording started with it draws each input where it landed, in the colour +of the role that sent it, on the frames captured after it; a call to the driver's own or forwarded tool that acts +shows as a chip; a plain recording draws nothing.""" +from __future__ import annotations + +import asyncio +import dataclasses +import os +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") +av = pytest.importorskip("av") + +from computer_fake import FakeDriver, MinimalDriver, PageFake, http_client, mcp_session # noqa: E402 +from mcp.types import CallToolResult, TextContent # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol import EnvironmentTool, ForwardedTool, tool # noqa: E402 +from agentenv_protocol.computer import ( # noqa: E402 + COMPUTER_RECORDING, + DeviceUnavailable, + FileChooser, + PageDialog, + RecordedSegment, + Recording, + record, +) +from agentenv_protocol.computer.annotate import ( # noqa: E402 + COLOURS, + FADE_S, + KEEP_S, + STYLE, + UNIT_EDGE, + Annotations, + PointerMove, + draw, + input_marks, + tool_mark, +) +from agentenv_protocol.computer.conformance import checks # noqa: E402 +from agentenv_protocol.computer.tools import COMPUTER_TOOLS # noqa: E402 +from agentenv_protocol.tool_context import ROLE_META_KEY # noqa: E402 + +START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" +T0 = datetime(2026, 11, 19, 12, tzinfo=timezone.utc) +CLICK = (400, 300) + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +class Recorded(FakeDriver): + recording = Recording(max_segment_s=60) + + +class Pointerless(MinimalDriver): + """A driver with no pointer group, so its recordings, polled from captures, can't draw the pointer themselves.""" + recording = Recording(max_segment_s=60) + + +class Segmented(FakeDriver): + """A native recorder that hands over a finished segment, which the base class keeps as it came.""" + recording = Recording(max_segment_s=60) + + @record + async def segments(self, hints): + path = hints.directory / "0000.mp4" + start = datetime.now(timezone.utc) + out = av.open(str(path), "w", format="mp4") + stream = out.add_stream("libx264", rate=10) + stream.width, stream.height, stream.pix_fmt = 160, 100, "yuv420p" + for i in range(3): + frame = av.VideoFrame.from_image(PILImage.new("RGB", (160, 100))).reformat(format="yuv420p") + frame.pts = i + for packet in stream.encode(frame): + out.mux(packet) + for packet in stream.encode(): + out.mux(packet) + out.close() + yield RecordedSegment(path, start, start + timedelta(seconds=0.2), 160, 100, 3) + await asyncio.sleep(3600) + + +class Templated(Recorded): + """A driver with tools of its own named after its environment: one that only reads, and one that acts whose name + ends as the first's does.""" + + @tool(name="{environment_name}_inspect", annotations={"readOnlyHint": True}) + async def inspect(self) -> str: + return "read" + + @tool(name="{environment_name}_foo_inspect") + async def poke(self) -> str: + return "poked" + + +class Forwarding(Recorded): + """A driver serving two tools of another server's: one that acts, one its annotations say only reads.""" + + def forwarded_tools(self): + schema = {"type": "object", "properties": {"text": {"type": "string"}}} + return [ForwardedTool(EnvironmentTool(name="click", description="Click an element.", inputSchema=schema), + self.answer), + ForwardedTool(EnvironmentTool(name="get_window_state", description="Read a window.", + inputSchema=schema), self.answer, {"readOnlyHint": True})] + + async def answer(self, **arguments): + return CallToolResult(content=[TextContent(type="text", text="done")]) + + +async def _clicked(driver, start: dict, meta: dict | None = None) -> tuple[datetime, dict, list]: + """A recording started with ``start`` around one click at CLICK: when the click was sent, the manifest, and each + frame with its capture time.""" + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, **start})).json() + await asyncio.sleep(0.5) + shot = await session.call_tool("computer_screenshot", {}) + sent = datetime.now(timezone.utc) + await session.call_tool("computer_click", {"x": CLICK[0], "y": CLICK[1], + "frame_id": shot.structuredContent["frame_id"]}, meta=meta) + await asyncio.sleep(0.5) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + return sent, manifest, _timed_frames(manifest) + + +def _timed_frames(manifest: dict) -> list[tuple[datetime, PILImage.Image]]: + frames = [] + for segment in manifest["segments"]: + started = datetime.fromisoformat(segment["started_at"].replace("Z", "+00:00")) + path = f"{os.environ['HOME']}/.agentenv-recordings/{segment['path']}" + with av.open(path) as container: + decoded = list(container.decode(container.streams.video[0])) + for frame in decoded: + frames.append((started + timedelta(seconds=frame.time - decoded[0].time), frame.to_image())) + return frames + + +def _marked(image: PILImage.Image, point: tuple[int, int], colour: tuple[int, int, int], reach: int = 40) -> int: + """How many pixels within ``reach`` of ``point`` are near ``colour``, as an encoded frame keeps it.""" + x, y = point + near = 0 + for px in range(max(0, x - reach), min(image.width, x + reach)): + for py in range(max(0, y - reach), min(image.height, y + reach)): + if sum(abs(a - b) for a, b in zip(image.getpixel((px, py)), colour)) < 90: + near += 1 + return near + + +def _split(frames: list, sent: datetime) -> tuple[list, list]: + """The frames captured before the click was sent, and those while its mark shows.""" + before = [image for at, image in frames if at < sent - timedelta(milliseconds=50)] + after = [image for at, image in frames if sent + timedelta(milliseconds=150) <= at <= sent + timedelta(seconds=0.5)] + assert before and after, f"{len(frames)} frames, {len(before)} before the click and {len(after)} after it" + return before, after + + +@pytest.mark.asyncio +async def test_an_annotated_recording_draws_a_click_where_it_landed_on_the_frames_after_it(): + sent, manifest, frames = await _clicked(Recorded(), {"annotate": {}}) + + before, after = _split(frames, sent) + assert manifest["annotate"] == {} + assert not any(_marked(image, CLICK, COLOURS[0]) for image in before) + assert all(_marked(image, CLICK, COLOURS[0]) > 20 for image in after) + + +@pytest.mark.asyncio +async def test_a_plain_recording_draws_nothing_and_its_manifest_names_no_annotate(): + sent, manifest, frames = await _clicked(Recorded(), {}) + + _before, after = _split(frames, sent) + assert "annotate" not in manifest + assert not any(_marked(image, CLICK, COLOURS[0]) for image in after) + + +@pytest.mark.asyncio +async def test_each_role_draws_in_its_own_colour(): + sent, _manifest, frames = await _clicked(Recorded(), {"annotate": {}}, meta={ROLE_META_KEY: "user"}) + + _before, after = _split(frames, sent) + assert all(_marked(image, CLICK, COLOURS[1]) > 20 and not _marked(image, CLICK, COLOURS[0]) for image in after) + + +@pytest.mark.asyncio +async def test_a_driver_tool_that_acts_shows_as_a_chip_and_one_that_only_reads_or_the_sdks_own_does_not(): + driver = Forwarding() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + await session.call_tool("click", {"text": "Save"}) + await session.call_tool("get_window_state", {"text": "Notes"}) + await session.call_tool("computer_screenshot", {}) + labels = [mark.label for mark in recording.annotations.marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert labels == ['click "Save"'] + + +@pytest.mark.asyncio +async def test_start_refuses_an_annotate_that_isnt_an_object_or_names_an_option_it_doesnt_take(): + async with http_client(Recorded()) as http: + option = await http.post(START, json={"request_id": "a", "annotate": {"events": True}}) + scalar = await http.post(START, json={"request_id": "b", "annotate": True}) + + assert option.status_code == scalar.status_code == 400 + assert option.json()["error"]["code"] == scalar.json()["error"]["code"] == "invalid_argument" + assert "it takes no options yet, not 'events'" in option.json()["error"]["message"] + + +@pytest.mark.asyncio +async def test_the_card_lists_annotate_and_its_options_with_the_recording_params(): + async with http_client(Recorded()) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + + [extension] = [e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING] + assert extension["params"]["annotate"] == {"options": []} + + +def test_each_input_is_marked_by_what_it_did_and_who_did_it(): + def labels(operation, args, role=None, **kw): + return [(m.kind, m.label) for m in input_marks(operation, args, role=role, at=T0, size=(1000, 500), **kw)] + + assert labels("click", (500, 250, "left", 2, ["Shift"])) == [("click", ""), ("chip", "Shift + click")] + assert labels("type_text", ("Q4 165\n",)) == [("chip", 'type "Q4 165[Enter]"')] + assert labels("press", (["Control", "s"],), repeat=3) == [("chip", "key Ctrl+S x3")] + assert labels("key_down", (["Shift"],), hold_ms=500) == [("chip", "hold Shift 500 ms")] + assert labels("pointer_down", ("left",)) == [("chip", "press left")] + assert labels("pointer_down", ("left",), point=(10, 20)) == [("pointer_down", "")] + assert labels("navigate", ("https://example.com",), role="user") == [("chip", "user: open https://example.com")] + assert labels("tabs", ("list", None, None)) == [] + [click] = input_marks("click", (500, 250, "right", 1, []), role=None, at=T0, size=(1000, 500)) + assert (click.points, click.button, click.role) == (((0.5, 0.5),), "right", "default") + [drag] = input_marks("drag", ([(0, 0), (1000, 500)], "left", 400, []), role=None, at=T0, size=(1000, 500)) + assert drag.moving_s == 0.4 and drag.until - drag.at == timedelta(seconds=1.2) + + +def test_a_tools_chip_shows_its_main_argument(): + def label(arguments): + return tool_mark("hotkey", arguments, role=None, at=T0).label + + assert label({"keys": ["cmd", "s"]}) == "hotkey cmd+S" + assert label({"text": "x" * 100}) == 'hotkey "' + "x" * 59 + '…"' + assert label({"window_id": 4}) == "hotkey" + assert tool_mark("click", {"element": 7}, role="executor", at=T0).label == "executor: click 7" + + +def test_a_mark_shows_from_its_input_until_it_ends_and_is_then_forgotten(): + annotations = Annotations() + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + + assert annotations.showing(T0 - timedelta(milliseconds=10)) == [] + assert len(annotations.showing(T0 + timedelta(milliseconds=500))) == 1 + assert annotations.showing(T0 + timedelta(seconds=1)) == [] and annotations.marks == [] + + +def test_a_mark_lands_at_its_point_on_a_frame_of_any_size(): + """A frame scaled down by max_edge, or captured at a HiDPI scale, keeps the point where the input landed.""" + annotations = Annotations() + annotations.add(input_marks("click", (500, 250, "left", 1, []), role=None, at=T0, size=(1000, 500))) + at = T0 + timedelta(milliseconds=400) + + for size in ((1000, 500), (400, 200), (2000, 1000)): + image = draw(PILImage.new("RGB", size, (255, 255, 255)), annotations.showing(at)) + cx, cy = size[0] // 2, size[1] // 2 + reach = round(STYLE.dot * max(size) / UNIT_EDGE) + 3 # the click's hollow dot, around its point + assert any(image.getpixel((cx + dx, cy + dy)) == COLOURS[0] + for dx in range(-reach, reach + 1) for dy in range(-reach, reach + 1)), size + assert image.getpixel((5, 5)) == (255, 255, 255) + + +def test_a_mark_no_frame_caught_while_it_lasted_shows_fully_on_the_first_frame_after_it_once(): + """At a low fps, or after a slow capture, the next frame may come after a click's mark is over.""" + annotations = Annotations() + [click] = input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100)) + annotations.add([click]) + + [(mark, _colour, age)] = annotations.showing(T0 + timedelta(seconds=3)) + span = (click.until - click.at).total_seconds() + assert mark is click and age == pytest.approx(span - FADE_S) + assert annotations.showing(T0 + timedelta(seconds=4)) == [] and annotations.marks == [] + + +def test_marks_no_frame_draws_are_kept_only_so_long(): + annotations = Annotations() + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + later = T0 + timedelta(seconds=KEEP_S + 1) + annotations.add(input_marks("click", (20, 20, "left", 1, []), role=None, at=later, size=(100, 100))) + + assert [mark.at for mark in annotations.marks] == [later] + + +@pytest.mark.asyncio +async def test_an_input_goes_ahead_when_its_mark_cant_be_made(): + """Reading the display for a mark fails: the scroll is still delivered, unmarked.""" + driver = Recorded() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + await session.call_tool("computer_screenshot", {}) + driver.fail["display"] = DeviceUnavailable("the display went away") + scrolled = await session.call_tool("computer_scroll", {"direction": "down"}) + marks = list(recording.annotations.marks) + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert not scrolled.isError and driver.screen.log[-1][0] == "scroll" and marks == [] + + +@pytest.mark.asyncio +async def test_a_recording_of_a_native_recorders_finished_segments_names_no_annotate(): + driver = Segmented() + async with http_client(driver) as http: + started = (await http.post(START, json={"request_id": "r", "annotate": {}})).json() + await asyncio.sleep(0.3) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + + assert manifest["segments"] and "annotate" not in manifest + + +def test_chips_past_the_most_a_frame_shows_wait_for_a_later_frame_rather_than_go_unseen(): + annotations = Annotations() + for n in range(6): + annotations.add([tool_mark(f"tool{n}", {}, role=None, at=T0 + timedelta(milliseconds=n))]) + + first = [mark.label for mark, _colour, _age in annotations.showing(T0 + timedelta(milliseconds=100))] + later = [mark.label for mark, _colour, _age in annotations.showing(T0 + timedelta(seconds=3))] + + assert first == ["tool2", "tool3", "tool4", "tool5"] and later == ["tool0", "tool1"] + assert annotations.showing(T0 + timedelta(seconds=4)) == [] and annotations.marks == [] + + +@pytest.mark.asyncio +async def test_the_probe_finds_annotation_not_applicable_to_a_native_recorders_finished_segments(): + wire = {"recording_id": "rec-1", "ended_by": "end", "segments": []} + with pytest.raises(checks.NotApplicable, match="hands over finished segments"): + await checks.Probe._annotated_click(None, wire, (100, 100), (10, 10), (20, 20), (255, 255, 255)) + + +@pytest.mark.asyncio +async def test_a_tool_named_after_its_environment_is_told_read_only_by_its_own_annotations(): + driver = Templated() + async with http_client(driver) as http, mcp_session(driver) as session: + names = [t.name for t in (await session.list_tools()).tools if t.name.endswith("_inspect")] + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + for name in names: + await session.call_tool(name, {}) + labels = [mark.label for mark in recording.annotations.marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + [poke] = [name for name in names if name.endswith("_foo_inspect")] + assert len(names) == 2 and labels == [poke] + + +def _pointed(image: PILImage.Image, tip: tuple[int, int], colour: tuple[int, int, int]) -> int: + """How many pixels near ``colour`` the agent's pointer drawn at ``tip`` would cover: its arrow, down and right.""" + x, y = tip + return sum(1 for px in range(x, min(image.width, x + 14)) for py in range(y + 2, min(image.height, y + 22)) + if sum(abs(a - b) for a, b in zip(image.getpixel((px, py)), colour)) < 90) + + +@pytest.mark.parametrize("driver, drawn", [(Pointerless, True), (Recorded, False)], + ids=["frames-without-a-pointer", "frames-with-the-pointer"]) +@pytest.mark.asyncio +async def test_the_agents_pointer_is_drawn_where_its_click_left_it_only_where_the_frames_cant_show_the_pointer( + driver, drawn): + """Once the click's own mark is over, a frame still shows the pointer there: drawn in the agent's colour where + the recording's frames leave the pointer out, and left to the frames' own where they draw it.""" + driver = driver() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + await asyncio.sleep(0.3) + shot = await session.call_tool("computer_screenshot", {}) + sent = datetime.now(timezone.utc) + await session.call_tool("computer_click", {"x": 200, "y": 150, "frame_id": shot.structuredContent["frame_id"]}) + await asyncio.sleep(1.6) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + + after = [image for at, image in _timed_frames(manifest) + if sent + timedelta(seconds=1.0) <= at <= sent + timedelta(seconds=1.5)] + assert after and all((_pointed(image, (200, 150), COLOURS[0]) > 10) is drawn for image in after) + + +def test_the_pointer_follows_the_last_input_before_each_frame_from_the_first_input_on(): + annotations = Annotations(pointer=True) + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + annotations.add(input_marks("pointer_move", (50, 50), role=None, at=T0 + timedelta(seconds=1), size=(100, 100))) + + def pointer(seconds): + return [(move.was, move.path[-1]) for move, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if isinstance(move, PointerMove)] + + assert pointer(-1) == [] + assert pointer(0.5) == [((0.1, 0.1), (0.1, 0.1))] + assert pointer(1.5) == [((0.1, 0.1), (0.5, 0.5))] and pointer(9) == [((0.1, 0.1), (0.5, 0.5))] + + +def test_each_role_and_pointer_has_a_pointer_the_last_mover_on_top_each_named_when_there_are_several(): + """The agent and a user simulator on one computer, and a second pointer of the agent's, as a lease would give it.""" + annotations = Annotations(pointer=True) + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + + def pointers(seconds): + return [(move.role, move.pointer, move.path[-1], move.faded, move.label) for move, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if isinstance(move, PointerMove)] + + assert pointers(0.5) == [("default", "default", (0.1, 0.1), False, "")] + annotations.add(input_marks("click", (50, 50, "left", 1, []), role="user", at=T0 + timedelta(seconds=1), + size=(100, 100))) + assert pointers(1.5) == [("default", "default", (0.1, 0.1), True, "agent"), + ("user", "default", (0.5, 0.5), False, "user")] + [second] = input_marks("pointer_move", (80, 20), role=None, at=T0 + timedelta(seconds=2), size=(100, 100)) + annotations.add([dataclasses.replace(second, pointer="p2")]) + assert pointers(2.5) == [("default", "default", (0.1, 0.1), True, "agent"), + ("user", "default", (0.5, 0.5), True, "user"), + ("default", "p2", (0.8, 0.2), False, "agent p2")] + + +#: Each input tool, a call to it, and the marks it makes, as (kind, label), in the order the calls go. +EVERY_INPUT = ( + ("computer_click", {"x": 10, "y": 10}, [("click", "")]), + ("computer_drag", {"path": [[10, 10], [60, 40]]}, [("drag", "")]), + ("computer_scroll", {"x": 10, "y": 10, "direction": "down"}, [("scroll", "")]), + ("computer_type", {"text": "hi"}, [("chip", 'type "hi"')]), + ("computer_key", {"keys": ["Control", "s"]}, [("chip", "key Ctrl+S")]), + ("computer_pointer_move", {"x": 20, "y": 20}, [("pointer_move", "")]), + ("computer_pointer_down", {}, [("pointer_down", "")]), + ("computer_pointer_up", {"x": 30, "y": 30}, [("pointer_up", "")]), + ("computer_keyboard_down", {"keys": ["Shift"]}, [("chip", "hold Shift")]), + ("computer_keyboard_up", {"keys": ["Shift"]}, [("chip", "release Shift")]), + ("computer_navigate", {"url": "https://x.test/"}, [("chip", "open https://x.test/")]), + ("computer_history", {"direction": "back"}, [("chip", "back")]), + ("computer_tabs", {"action": "switch", "tab": "b"}, [("chip", "tab switch b")]), + ("computer_dialog", {"accept": True}, [("chip", "dialog OK")]), + ("computer_upload", {"paths": ["a.pdf"]}, [("chip", "upload a.pdf")]), +) + + +@pytest.mark.asyncio +async def test_every_input_tool_is_marked_by_what_it_did_with_the_arguments_it_hands_its_operation(caplog): + """The marks are made from each operation's arguments as the tools build them, so a tool whose arguments change + shows here: by a wrong mark, or by the warning a mark that couldn't be made logs. A new input tool fails it until + it's listed.""" + driver = PageFake() + noted: list = [] + async with http_client(driver) as http, mcp_session(driver) as session: + served = {tool.name for tool in (await session.list_tools()).tools if tool.name in COMPUTER_TOOLS + and tool.annotations is not None and tool.annotations.readOnlyHint is False} + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recordings = driver._computer.recordings + note = recordings.note + recordings.note = lambda marks: (noted.append(list(marks)), note(marks)) + await session.call_tool("computer_screenshot", {}) + made = {} + with caplog.at_level("WARNING"): + for tool, args, _marks in EVERY_INPUT: + if tool == "computer_dialog": + driver.open_dialog = PageDialog("confirm", "Leave the page?") + if tool == "computer_upload": + driver.chooser = FileChooser() + noted.clear() + result = await session.call_tool(tool, args) + assert not result.isError, (tool, result.content[0].text) + made[tool] = [(mark.kind, mark.label) for marks in noted for mark in marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert served == {tool for tool, _args, _marks in EVERY_INPUT} + assert made == {tool: marks for tool, _args, marks in EVERY_INPUT} + assert "couldn't mark" not in caplog.text diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 591a04aaf..a6949f4df 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -12,10 +12,12 @@ import uuid from typing import ClassVar, Optional +from agentenv_protocol.computer.annotate import AnnotateOptions, options_problem from agentenv_protocol.computer.card import COMPUTER_RECORDING from agentenv_protocol.computer.manifest import ManifestSegment, RecordingManifest from agentenv_protocol.computer.recording import MANIFEST from agentenv_protocol.client import find_extension +from pydantic import ValidationError from agent_env.artifact import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse @@ -47,27 +49,50 @@ MAX_RECORDING_BYTES = 64 << 30 +class AnnotationSettings(AnnotateOptions): + """``start_computer_recording``'s ``annotation_settings``: whether the recording draws its inputs, as it does + unless ``is_enabled`` is false, and the options its driver's ``annotate`` takes, which the step passes on.""" + + is_enabled: bool = True + + class StartComputerRecordingTaskStep(TaskStep): - """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``. Best effort - by default.""" + """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``, its driver + drawing each input over the frames unless ``annotation_settings`` says ``is_enabled: false``; a driver that + can't, as one composed before agent-env drew inputs, records it plain. Best effort by default.""" type: ClassVar[str] = "start_computer_recording" entity_refs = (EntityRef.env("env_id"),) def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, environment_name: Optional[str] = None, - depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): + annotation_settings: Optional[dict] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = False): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) if not env_id or not name: raise ValueError("start_computer_recording needs an env_id and a name for the recording") self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor self.environment_name = environment_name + self.annotation_settings = annotation_settings + try: + self.annotation = AnnotationSettings.model_validate({} if annotation_settings is None else + annotation_settings) + except ValidationError as e: + raise ValueError(f"start_computer_recording's annotation_settings: " + f"{options_problem(e, AnnotationSettings)}") from None + + @property + def annotates(self) -> bool: + """Whether the recording draws each input, as it does unless its settings turn it off.""" + return self.annotation.is_enabled def to_dict(self) -> dict: data = {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor} if self.environment_name is not None: # left out otherwise, so a step without one serializes as before data["environment_name"] = self.environment_name + if self.annotation_settings is not None: # left out otherwise, so a step without them serializes as before + data["annotation_settings"] = self.annotation_settings return data @classmethod @@ -76,7 +101,8 @@ def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: base["fail_task_on_error"] = data.get("fail_task_on_error", False) return cls(**base, env_id=data["env_id"], name=data["name"], fps=data.get("fps"), max_edge=data.get("max_edge"), cursor=data.get("cursor", True), - environment_name=data.get("environment_name")) + environment_name=data.get("environment_name"), + annotation_settings=data.get("annotation_settings")) async def execute(self, context: TaskStepContext) -> TaskStepContext: recordings = context.metadata.setdefault(RECORDINGS_KEY, {}) @@ -94,9 +120,16 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: refuse_no_network(deployed) namespace = recording_namespace(request_id, _KEY_PREFIX) grant = _grant(deployed, namespace, self.environment_name) + annotated = self.annotates and _annotates(deployed, self.environment_name) + if self.annotates and not annotated: + logger.warning("start_computer_recording %s on %s: its driver can't draw inputs, its card listing no " + "annotate, as a computer composed before agent-env drew them; recording it plain", + self.name, self.env_id) + options = self.annotation.model_dump(exclude={"is_enabled"}, exclude_unset=True) started = await deployed.invoke(COMPUTER_RECORDING, "start", { "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, - "upload": grant}, environment_name=self.environment_name) + "upload": grant, **({"annotate": options} if annotated else {})}, + environment_name=self.environment_name) except Exception as e: recordings[self.name] = {"env_id": self.env_id, "state": "failed", "error": why(e)} logger.warning("start_computer_recording %s on %s: %s", self.name, self.env_id, e) @@ -105,7 +138,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context recordings[self.name] = { "env_id": self.env_id, "recording_id": started["recording_id"], "started_at": started["started_at"], - "namespace": namespace, "state": "open"} + "namespace": namespace, "state": "open", "annotated": annotated} if self.environment_name is not None: recordings[self.name]["environment_name"] = self.environment_name logger.info("start_computer_recording %s on %s: %s", self.name, self.env_id, started["recording_id"]) @@ -197,6 +230,8 @@ def _registered(context: TaskStepContext, name: str, entry: dict, manifest: Reco uploaded = [s for s in manifest.segments if s.uploaded] entry.update(state="ended", ended_by=manifest.ended_by, segments=len(uploaded), missing_segments=len(manifest.segments) - len(uploaded)) + if manifest.source is not None: # the driver's own says whether it drew; one rebuilt from segments can't + entry["annotated"] = manifest.annotate is not None logger.info("recording %s: %d segments, ended by %s", name, len(uploaded), manifest.ended_by) @@ -210,8 +245,7 @@ def _deployed(context: TaskStepContext, env_id: str): def _grant(deployed, namespace: str, environment_name: Optional[str] = None, method: str = "start") -> dict: """A grant for a computer recording's segments and manifest. EnvCapabilityUnsupported when the env's card, or its child's, declares no recording.""" - card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) - extension = find_extension(card or {}, COMPUTER_RECORDING) + extension = _recording_extension(deployed, environment_name) if extension is None: raise EnvCapabilityUnsupported(COMPUTER_RECORDING, method, deployed.env_id, getattr(deployed, "instance_id", None), environment_name) @@ -219,6 +253,16 @@ def _grant(deployed, namespace: str, environment_name: Optional[str] = None, met MAX_RECORDING_BYTES) +def _recording_extension(deployed, environment_name: Optional[str]) -> Optional[dict]: + card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) + return find_extension(card or {}, COMPUTER_RECORDING) + + +def _annotates(deployed, environment_name: Optional[str]) -> bool: + """Whether the env's driver, or its child's, draws inputs over a recording's frames: its card lists annotate.""" + return "annotate" in ((_recording_extension(deployed, environment_name) or {}).get("params") or {}) + + def _from_namespace(entry: dict) -> RecordingManifest: """What the recording's namespace holds, for a recording its driver didn't end for the run: the manifest the driver uploaded beside the segments when it ended it, or else one of the segments there.""" diff --git a/tst/integration/computer/desktop_example_test.py b/tst/integration/computer/desktop_example_test.py index fa3489bcd..8b494ffe3 100644 --- a/tst/integration/computer/desktop_example_test.py +++ b/tst/integration/computer/desktop_example_test.py @@ -51,6 +51,7 @@ def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state): context, manifest, frames = _recorded(ran.output, "desk", state) entry = context["metadata"]["computer_recordings"]["desk"] assert (entry["state"], entry["ended_by"], entry["missing_segments"]) == ("ended", "end", 0) + assert entry["annotated"] is False and "annotate" not in manifest assert manifest["source"] == "stream" and manifest["cursor"] is True and frames assert len(frames) == sum(s["frames"] for s in manifest["segments"]) drive = json.loads(context["metadata"]["verifications"]["drive"]["results"][0]["description"]) @@ -72,6 +73,7 @@ def test_an_agent_given_the_computers_skill_files_a_loaded_file_in_a_new_folder_ context, manifest, frames = _recorded(ran.output, "agent", state) entry = context["metadata"]["computer_recordings"]["agent"] assert (entry["state"], entry["missing_segments"]) == ("ended", 0) and frames + assert entry["annotated"] is True and manifest["annotate"] == {} assert "## Shortcuts" in context["metadata"]["env_skills"]["default-agent"]["desktop"]["desktop"] diff --git a/tst/integration/computer/recording/tasks/recording.json b/tst/integration/computer/recording/tasks/recording.json index ad000361f..5a67b77b1 100644 --- a/tst/integration/computer/recording/tasks/recording.json +++ b/tst/integration/computer/recording/tasks/recording.json @@ -1,7 +1,7 @@ [ {"id": "env", "type": "deploy_env", "env_id": "@local/agentenv-framework/desktop/desktop"}, {"id": "record", "type": "start_computer_recording", "env_id": "@local/agentenv-framework/desktop/desktop", - "name": "desk", "fps": 10, "depends_on": ["env"]}, + "name": "desk", "fps": 10, "annotation_settings": {"is_enabled": false}, "depends_on": ["env"]}, {"id": "drive", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "drive_desk", "verifier_id": "drive", "depends_on": ["record"]}, {"id": "end", "type": "end_computer_recording", "name": "desk", "depends_on": ["drive"]} diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 6b05e0feb..72bd76b2a 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -39,6 +39,7 @@ def __init__(self, store, env_id: str = "desktop", forgets: bool = False, garble self.store, self.forgets, self.garbles, self.drop_last = store, forgets, garbles, drop_last self.calls: list[tuple[str, dict]] = [] self.started: set[str] = set() + self.manifest: dict = {} # what else its manifest says, as a driver's own does async def invoke(self, uri, method, params=None, **_): assert uri == COMPUTER_RECORDING @@ -60,7 +61,7 @@ async def invoke(self, uri, method, params=None, **_): {"index": 1, "path": "rec-1/0001.mp4", "uploaded": not self.drop_last}] if not self.drop_last: self.store.put(f"{root}/rec-1/0001.mp4", b"mp4-1", content_type="video/mp4") - manifest = {"recording_id": "rec-1", "ended_by": params["reason"], "segments": segments} + manifest = {"recording_id": "rec-1", "ended_by": params["reason"], "segments": segments, **self.manifest} self.store.put(f"{root}/rec-1/manifest.json", json.dumps(manifest).encode(), content_type="application/json") return {"manifests": [manifest]} if method == "end_all" else manifest @@ -341,3 +342,61 @@ async def test_a_start_without_a_child_env_reads_the_envs_own_card_as_before(sto assert entry["error"].startswith("EnvCapabilityUnsupported: env 'shop' does not offer 'start' on ") context = await _start().execute(_context(_Computer(store))) assert "environment_name" not in context.metadata[RECORDINGS_KEY]["desk"] + + +_ANNOTATING = {**_CARD, "capabilities": {"extensions": [{**_CARD["capabilities"]["extensions"][0], "params": { + **_CARD["capabilities"]["extensions"][0]["params"], "annotate": {"options": []}}}]}} + + +@pytest.mark.asyncio +async def test_a_recording_draws_its_inputs_unless_its_annotation_settings_turn_it_off(store): + on, off = _Computer(store), _Computer(store) + on.environment_card = off.environment_card = _ANNOTATING + started = (await _start().execute(_context(on))).metadata[RECORDINGS_KEY]["desk"] + plain = (await _start("plain", annotation_settings={"is_enabled": False}).execute(_context(off))).metadata[ + RECORDINGS_KEY]["plain"] + + [(_, annotated)], [(_, unannotated)] = on.calls, off.calls + assert annotated["annotate"] == {} and started["annotated"] is True + assert "annotate" not in unannotated and plain["annotated"] is False + + +@pytest.mark.asyncio +async def test_a_driver_that_cant_draw_inputs_is_recorded_plain_with_a_warning_even_when_its_failures_are_fatal( + store, caplog): + """A computer composed before agent-env drew inputs lists no annotate on its card.""" + computer = _Computer(store) + with caplog.at_level("WARNING"): + context = await _start(fail_task_on_error=True).execute(_context(computer)) + entry = context.metadata[RECORDINGS_KEY]["desk"] + + [(_, started)] = computer.calls + assert entry["state"] == "open" and entry["annotated"] is False and "annotate" not in started + assert "can't draw inputs" in caplog.text and "recording it plain" in caplog.text + + +def test_annotation_settings_round_trip_only_when_set_and_take_a_boolean_is_enabled(): + off = _start(annotation_settings={"is_enabled": False}) + assert StartComputerRecordingTaskStep.from_dict(off.to_dict()).annotation_settings == {"is_enabled": False} + assert not off.annotates and _start().annotates and _start(annotation_settings={}).annotates + assert "annotation_settings" not in _start().to_dict() + for settings, problem in (({"is_enabled": "no"}, "is_enabled: input should be a valid boolean"), + ({"marks": []}, "it takes is_enabled, not 'marks'"), (True, "is an object of options"), + (False, "is an object of options"), ("", "is an object of options")): + with pytest.raises(ValueError, match=problem): + _start(annotation_settings=settings) + + +@pytest.mark.parametrize("manifest, annotated", [({"source": "stream", "annotate": {}}, True), + ({"source": "stream"}, False), ({}, True)], + ids=["drawn", "a-native-recorders-segments", "rebuilt-from-segments"]) +@pytest.mark.asyncio +async def test_a_recordings_entry_says_whether_its_video_was_drawn_on_as_its_drivers_own_manifest_says( + store, manifest, annotated): + """A driver that annotates can still hand over finished segments, which aren't drawn on and whose manifest names + no annotate; a manifest rebuilt from the segments alone can't say, so the start's answer stands.""" + computer = _Computer(store) + computer.environment_card, computer.manifest = _ANNOTATING, manifest + context = await _end().execute(await _start().execute(_context(computer))) + + assert context.metadata[RECORDINGS_KEY]["desk"]["annotated"] is annotated