From 89250deb90d2e601f9c4d0ac15380e913be8c171 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 11:00:13 -0800 Subject: [PATCH 01/10] feat(computers): an annotated recording draws each input over the frames computer-recording/v1 start takes annotate, an object of options (none yet, so {} draws everything). The driver then draws each computer_* input on the frames captured while it shows, in the colour of the role that sent it: a ring where a click landed, a trail for a drag, an arrow for a scroll, and a chip for keys, typed text and each driver tool that acts. The card lists annotate under the recording params, the manifest records it, start_computer_recording passes it through and refuses a driver that lists none, and the conformance probe checks a click is drawn where it landed from the frame its effect shows on. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 10 +- .../agentenv_protocol/computer/annotate.py | 328 ++++++++++++++++++ .../computer/conformance/checks.py | 152 ++++++-- .../src/agentenv_protocol/computer/driver.py | 37 +- .../agentenv_protocol/computer/extensions.py | 10 +- .../agentenv_protocol/computer/manifest.py | 2 + .../agentenv_protocol/computer/recording.py | 42 ++- .../src/agentenv_protocol/computer/runtime.py | 17 +- .../agentenv-protocol/tests/probe_fake.py | 16 +- .../tests/test_computer_recording.py | 3 +- .../tests/test_computer_recording_annotate.py | 215 ++++++++++++ .../task_steps/computer_recording.py | 33 +- tst/unit/task_step/computer_recording_test.py | 34 ++ 14 files changed, 844 insertions(+), 57 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py create mode 100644 packages/agentenv-protocol/tests/test_computer_recording_annotate.py diff --git a/AGENTS.md b/AGENTS.md index d6bbf6649..610cee9a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (with `annotate`, `{}` for no options, the computer's driver draws each input over the frames in its caller's colour, which a driver whose card lists no `annotate` refuses) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index d359234ed..aa960bef2 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -755,7 +755,7 @@ class MyDriver(ComputerDriverEnvironment): of its own, the latest 8 frames wait for the encoder, and it is closed when the recording stops. A driver without `@record` records by the base class polling `@capture`, with a pointer drawn at `@pointer_position`: the slow path, at most 15 fps. -- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns +- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?, annotate?}` returns `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment and returns the manifest; `end_all {reason?}` ends every recording not yet ended, each under the grant it started with, and returns `{manifests}`, as the framework does before it resets or tears a @@ -768,6 +768,14 @@ class MyDriver(ComputerDriverEnvironment): `/.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no grant they stay under `~/.agentenv-recordings/`. An ended recording's manifest uploads beside its segments, at `/manifest.json`, so a reader that didn't end it finds it there. +- **Annotations.** `start` with `annotate`, an object of options (`{}` for none; the card lists those a + driver takes under its params' `annotate.options`), draws each input over the frames captured while it + shows, in the colour of the role that sent it (the agent's first): a ring where a click landed (a + square for the right button, a ring for each click of a double or triple), a trail for a drag, an + arrow for a scroll, a cross for a move, and a chip for a key chord, typed text as it was typed, and + each of the driver's own or forwarded tools whose annotations don't say it only reads, by name and + main argument. Every `computer_*` input is marked as `act` delivers it; a segment a native recorder + finished is kept as it came. The manifest's `annotate` is the object the recording drew with. - **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path, `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size, SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py new file mode 100644 index 000000000..db2033286 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -0,0 +1,328 @@ +"""What an annotated recording (computer-recording/v1 ``start`` with ``annotate``) draws over its frames: where each +input landed and what it typed or pressed, in the colour of the role that sent it, on the frames captured while its +mark shows. ``ComputerRuntime.act`` marks every ``computer_*`` input as it is delivered; a call to one of the +driver's own or forwarded tools, which acts without it, shows as a chip naming the tool and its main argument.""" +from __future__ import annotations + +import functools +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Optional + +import av +from PIL import Image as PILImage +from PIL import ImageDraw, ImageFont + +from .errors import InvalidArgument + +#: The options ``annotate`` takes: none yet, so ``{}`` draws every mark. +OPTIONS: tuple[str, ...] = () +#: The role of a call that names none: the agent. +DEFAULT_ROLE = "default" +#: Each role's colour: the agent's first, then each other role's in the order it first acts on the recording. +COLOURS = ((255, 138, 0), (0, 184, 148), (175, 82, 222), (255, 59, 105), (52, 120, 246), (250, 204, 21)) +#: How long a mark shows from its input, in seconds; a drag's, after its moves. +SHOW_S = {"click": 0.8, "down": 0.8, "up": 0.8, "move": 0.5, "scroll": 0.9, "drag": 0.8, "chip": 1.6} +#: The most a chip of typed text shows, which grows with its length from the chip's own time. +MOST_TEXT_S = 4.0 +#: The most of a chip's text it shows; the rest is cut with an ellipsis. +CHIP_CHARS = 60 +#: The most chips shown at once, the newest lowest. +MOST_CHIPS = 4 +#: A frame's longer edge that marks are sized for; on a larger or smaller frame they scale with it. +UNIT_EDGE = 1280 +#: Arguments a tool's chip shows, the first one a call has. +MAIN_ARGUMENTS = ("text", "keys", "key", "url", "query", "label", "name", "element", "title", "code", "script") +#: Key names as chips show them. +KEY_NAMES = {"Control": "Ctrl", "Escape": "Esc", "ArrowLeft": "Left", "ArrowRight": "Right", "ArrowUp": "Up", + "ArrowDown": "Down", " ": "Space"} + + +@dataclass(frozen=True) +class Mark: + """One input on a recording: ``kind`` is click, down, up, move, scroll, drag or chip; ``points`` are fractions of + the display, so they land on any frame size; it shows from ``at`` until ``until``, a drag moving for its first + ``moving_s``.""" + + kind: str + at: datetime + until: datetime + role: str + points: tuple[tuple[float, float], ...] = () + button: str = "left" + count: int = 1 + direction: str = "" + label: str = "" + moving_s: float = 0.0 + + +def check_annotate(annotate: object) -> Optional[dict]: + """``start``'s ``annotate``: None, or an object of the options this driver takes.""" + if annotate is None: + return None + if not isinstance(annotate, dict): + raise InvalidArgument(f"annotate is an object of options, got {type(annotate).__name__}") + if unknown := sorted(set(annotate) - set(OPTIONS)): + raise InvalidArgument(f"annotate takes no options yet, got {unknown}; give {{}} to draw every input") + return dict(annotate) + + +class Annotations: + """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role.""" + + def __init__(self) -> None: + self.marks: list[Mark] = [] + self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} + + def add(self, marks: Sequence[Mark]) -> None: + for mark in marks: + self.colours.setdefault(mark.role, COLOURS[len(self.colours) % len(COLOURS)]) + self.marks.append(mark) + + def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int]]]: + """The marks a frame captured ``at`` shows, with their colours. A stream's frames come in the order they were + captured, so a mark over by then is over for every later frame too.""" + self.marks = [mark for mark in self.marks if mark.until > at] + return [(mark, self.colours[mark.role]) for mark in self.marks if mark.at <= at] + + +def input_marks(operation: str, args: tuple, *, role: Optional[str], at: datetime, size: tuple[int, int], + point: Optional[tuple] = None, repeat: int = 1, hold_ms: Optional[int] = None) -> list[Mark]: + """The marks of one ``ComputerRuntime.act`` input: ``args`` are the operation's, in screen units on a display of + ``size``; ``point`` is where an input with no point of its own acts (its ``move_to``, or the pointer).""" + role = role or DEFAULT_ROLE + width, height = size + + def where(x: float, y: float) -> tuple[float, float]: + return (x / width, y / height) + + def shown(kind: str, extra_s: float = 0.0) -> datetime: + return at + timedelta(seconds=SHOW_S[kind] + extra_s) + + def chip(label: str, seconds: Optional[float] = None) -> Mark: + until = at + timedelta(seconds=seconds) if seconds is not None else shown("chip") + return Mark("chip", at, until, role, label=label if role == DEFAULT_ROLE else f"{role}: {label}") + + held = args[-1] if operation in ("click", "drag", "scroll") and args and isinstance(args[-1], list) else [] + marks: list[Mark] = [] + if operation == "click": + x, y, button, count = args[:4] + marks.append(Mark("click", at, shown("click"), role, (where(x, y),), button=button, count=count)) + elif operation == "drag": + path, button, duration = args[:3] + marks.append(Mark("drag", at, shown("drag", duration / 1000), role, tuple(where(*p) for p in path), + button=button, moving_s=duration / 1000)) + elif operation == "scroll": + x, y, direction, amount = args[:4] + marks.append(Mark("scroll", at, shown("scroll"), role, (where(x, y),), direction=direction, count=amount)) + elif operation == "pointer_move": + marks.append(Mark("move", at, shown("move"), role, (where(*args[:2]),))) + elif operation in ("pointer_down", "pointer_up"): + kind = "down" if operation == "pointer_down" else "up" + if point is not None: + marks.append(Mark(kind, at, shown(kind), role, (where(*point[:2]),), button=args[0])) + else: + marks.append(chip(f"{'press' if kind == 'down' else 'release'} {args[0]}")) + elif operation == "type_text": + text = args[0] + marks.append(chip(f'type "{_shown_text(text)}"', min(MOST_TEXT_S, SHOW_S["chip"] + len(text) / 25))) + elif operation == "press": + times = f" x{repeat}" if repeat > 1 else "" + marks.append(chip(f"key {_chord(args[0])}{times}")) + elif operation == "key_down": + for_ms = f" {hold_ms} ms" if hold_ms is not None else "" + marks.append(chip(f"hold {_chord(args[0])}{for_ms}")) + elif operation == "key_up": + marks.append(chip(f"release {_chord(args[0])}")) + elif operation == "navigate": + marks.append(chip(f"open {_cut(args[0])}")) + elif operation == "history": + marks.append(chip(args[0])) + elif operation == "tabs": + action, tab, url = args + if action != "list": + marks.append(chip(" ".join(str(part) for part in ("tab", action, tab or _cut(url or "")) if part))) + elif operation == "dialog": + marks.append(chip("dialog OK" if args[0] else "dialog Cancel")) + elif operation == "upload": + marks.append(chip(f"upload {_cut(', '.join(args[0]))}")) + else: + marks.append(chip(operation)) + if held: + marks.append(chip(f"{_chord(held)} + {operation}")) + return marks + + +def tool_mark(tool: str, arguments: Mapping[str, Any], *, role: Optional[str], at: datetime) -> Mark: + """The chip of a call to one of the driver's own or forwarded tools: its name and its main argument.""" + role = role or DEFAULT_ROLE + main = next((arguments[name] for name in MAIN_ARGUMENTS if arguments.get(name) not in (None, "", [])), None) + if isinstance(main, list) and all(isinstance(part, str) for part in main): + shown = _chord(main) + elif isinstance(main, str): + shown = f'"{_shown_text(main)}"' if main.strip() else "" + else: + shown = "" if main is None else _cut(str(main)) + label = f"{tool} {shown}".strip() + return Mark("chip", at, at + timedelta(seconds=SHOW_S["chip"]), role, + label=label if role == DEFAULT_ROLE else f"{role}: {label}") + + +def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int]]], at: datetime) -> PILImage.Image: + """``image``, a Pillow image or a PyAV frame, with ``marks`` drawn as a frame captured ``at`` shows them.""" + base = (image.to_image() if isinstance(image, av.VideoFrame) else image).convert("RGBA") + overlay = PILImage.new("RGBA", base.size, (0, 0, 0, 0)) + pen = ImageDraw.Draw(overlay) + unit = max(base.size) / UNIT_EDGE + chips = [] + for mark, colour in marks: + age = (at - mark.at).total_seconds() + span = max((mark.until - mark.at).total_seconds(), 1e-3) + if mark.kind == "chip": + chips.append((mark, colour)) + continue + fade = 1.0 - max(0.0, (age - span * 0.6) / (span * 0.4)) + points = [(fx * base.width, fy * base.height) for fx, fy in mark.points] + _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, span, fade) + if chips: + _draw_chips(pen, chips[-MOST_CHIPS:], base.size, unit) + return PILImage.alpha_composite(base, overlay).convert("RGB") + + +def _draw_click(pen, mark, points, colour, unit, age, span, fade) -> None: + x, y = points[0] + grow = min(1.0, age / 0.35) + alpha = round(255 * fade) + for ring in range(max(1, min(mark.count, 3))): + r = unit * (12 + 18 * grow + 8 * ring) + box = (x - r, y - r, x + r, y + r) + shape = pen.rectangle if mark.button == "right" else pen.ellipse + shape(box, outline=(255, 255, 255, alpha), width=max(1, round(5 * unit))) + shape(box, outline=(*colour, alpha), width=max(1, round(3 * unit))) + dot = unit * 5 + pen.ellipse((x - dot, y - dot, x + dot, y + dot), fill=(*colour, alpha), outline=(255, 255, 255, alpha), + width=max(1, round(1.5 * unit))) + if mark.button not in ("left", "right"): + _label(pen, (x + unit * 16, y + unit * 16), mark.button, colour, unit, alpha) + + +def _draw_press(pen, mark, points, colour, unit, age, span, fade) -> None: + x, y = points[0] + alpha = round(255 * fade) + r = unit * (10 if mark.kind == "down" else 16) + fill = (*colour, round(alpha * 0.7)) if mark.kind == "down" else None + pen.ellipse((x - r, y - r, x + r, y + r), fill=fill, outline=(255, 255, 255, alpha), width=max(1, round(4 * unit))) + pen.ellipse((x - r, y - r, x + r, y + r), outline=(*colour, alpha), width=max(1, round(2.5 * unit))) + + +def _draw_move(pen, mark, points, colour, unit, age, span, fade) -> None: + x, y = points[0] + alpha = round(200 * fade) + arm = unit * 9 + for dx, dy in ((arm, 0), (0, arm)): + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(255, 255, 255, alpha), width=max(1, round(4 * unit))) + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*colour, alpha), width=max(1, round(2 * unit))) + + +def _draw_scroll(pen, mark, points, colour, unit, age, span, fade) -> None: + x, y = points[0] + alpha = round(255 * fade) + dx, dy = {"up": (0, -1), "down": (0, 1), "left": (-1, 0), "right": (1, 0)}.get(mark.direction, (0, 1)) + length = unit * 46 + end = (x + dx * length, y + dy * length) + for width, fill in ((7, (255, 255, 255, alpha)), (4, (*colour, alpha))): + pen.line((x, y, *end), fill=fill, width=max(1, round(width * unit))) + _arrowhead(pen, (x, y), end, unit, fill, width) + _label(pen, (end[0] + unit * 10, end[1] + unit * 4), f"x{mark.count}", colour, unit, alpha) + + +def _draw_drag(pen, mark, points, colour, unit, age, span, fade) -> None: + alpha = round(255 * fade) + shown = points if age >= mark.moving_s else _along(points, age / mark.moving_s) + for width, fill in ((7, (255, 255, 255, alpha)), (4, (*colour, alpha))): + if len(shown) > 1: + pen.line([coord for point in shown for coord in point], fill=fill, width=max(1, round(width * unit)), + joint="curve") + _arrowhead(pen, shown[-2], shown[-1], unit, fill, width) + sx, sy = points[0] + r = unit * 6 + pen.ellipse((sx - r, sy - r, sx + r, sy + r), fill=(*colour, alpha), outline=(255, 255, 255, alpha), + width=max(1, round(1.5 * unit))) + + +_DRAWERS = {"click": _draw_click, "down": _draw_press, "up": _draw_press, "move": _draw_move, "scroll": _draw_scroll, + "drag": _draw_drag} + + +def _draw_chips(pen, chips, size, unit) -> None: + """Chips stacked above the bottom edge, centred, the newest lowest.""" + font = _font(max(10, round(20 * unit))) + pad, gap = unit * 10, unit * 8 + y = size[1] - unit * 48 + for mark, colour in reversed(chips): + left, top, right, bottom = pen.textbbox((0, 0), mark.label, font=font) + w, h = right - left + 2 * pad, bottom - top + 2 * pad * 0.7 + x0 = (size[0] - w) / 2 + y0 = y - h + pen.rounded_rectangle((x0, y0, x0 + w, y), radius=unit * 8, fill=(*colour, 235), + outline=(255, 255, 255, 235), width=max(1, round(1.5 * unit))) + pen.text((x0 + pad - left, y0 + pad * 0.7 - top), mark.label, font=font, fill=_ink(colour)) + y = y0 - gap + + +def _label(pen, at: tuple[float, float], text: str, colour, unit, alpha) -> None: + font = _font(max(9, round(15 * unit))) + left, top, right, bottom = pen.textbbox(at, text, font=font) + pad = unit * 4 + pen.rounded_rectangle((left - pad, top - pad, right + pad, bottom + pad), radius=unit * 4, + fill=(*colour, round(alpha * 0.9))) + pen.text(at, text, font=font, fill=(*_ink(colour)[:3], alpha)) + + +def _arrowhead(pen, start, end, unit, fill, width) -> None: + angle = math.atan2(end[1] - start[1], end[0] - start[0]) + size = unit * (14 + width) + left = (end[0] - size * math.cos(angle - 0.5), end[1] - size * math.sin(angle - 0.5)) + right = (end[0] - size * math.cos(angle + 0.5), end[1] - size * math.sin(angle + 0.5)) + pen.polygon([end, left, right], fill=fill) + + +def _along(points: list[tuple[float, float]], fraction: float) -> list[tuple[float, float]]: + """The first ``fraction`` of the path through ``points``, by length.""" + legs = [math.dist(a, b) for a, b in zip(points, points[1:])] + left = sum(legs) * max(0.0, min(1.0, fraction)) + shown = [points[0]] + for (a, b), leg in zip(zip(points, points[1:]), legs): + if left >= leg: + shown.append(b) + left -= leg + continue + t = left / leg if leg else 0.0 + shown.append((a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t)) + break + return shown + + +def _ink(colour: tuple[int, int, int]) -> tuple[int, int, int, int]: + """Black or white, whichever reads on ``colour``.""" + r, g, b = colour + return (20, 20, 20, 255) if 0.299 * r + 0.587 * g + 0.114 * b > 160 else (255, 255, 255, 255) + + +def _chord(keys: Sequence[str]) -> str: + return "+".join(KEY_NAMES.get(key, key.upper() if len(key) == 1 else key) for key in keys) + + +def _shown_text(text: str) -> str: + return _cut(text.replace("\n", "[Enter]").replace("\t", "[Tab]")) + + +def _cut(text: str) -> str: + return text if len(text) <= CHIP_CHARS else text[:CHIP_CHARS - 1] + "…" + + +@functools.lru_cache(maxsize=16) +def _font(size: int) -> Any: + return ImageFont.load_default(size=size) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index a2f13c1da..d4f854aaa 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -6,15 +6,17 @@ import asyncio import base64 import contextlib +import io import json import math import time import uuid from collections.abc import Awaitable, Callable -from datetime import datetime +from datetime import datetime, timedelta from importlib import resources from typing import Any +import av from mcp import ClientSession from mcp.types import CallToolResult from pydantic import ValidationError @@ -36,7 +38,8 @@ GROUP_TOOLS, POINT_TOOLS, ) -from ..manifest import RecordingManifest +from ..annotate import COLOURS as MARK_COLOURS +from ..manifest import ManifestSegment, RecordingManifest from ..recording import RECORDINGS_DIR from .client import Driver, image from .mp4 import read_mp4 @@ -64,6 +67,10 @@ NORM = 1000 #: How long the probe records, and the frame rate it asks for. RECORD_S = 2.0 +#: How long an annotated recording runs before and after the probe's click, so it keeps frames on both sides. +ANNOTATE_LEAD_S, ANNOTATE_TAIL_S = 1.0, 1.0 +#: How far, as the sum of its channels' differences, an encoded pixel may be from the colour drawn and still count. +NEAR_COLOUR = 90 #: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. BUSY_DESK_ATTEMPTS = 2 @@ -164,6 +171,7 @@ async def full(self) -> None: for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording), + ("X", self._annotation), ("X", self._page_tools)): await self._guarded(check, section) finally: @@ -1083,12 +1091,48 @@ async def limited() -> str | None: await self.check("X", "a repeated request_id joins the first recording", joined, COMPUTER_RECORDING) await self.check("X", "a recording past max_concurrent is refused", limited, COMPUTER_RECORDING) finally: - for recording_id in made: - with contextlib.suppress(Exception): - await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) - await self.driver.run(*(WIN_REMOVE if self.windows else ("rm", "-rf")), f"{RECORDINGS_DIR}/{recording_id}") - await self.driver.run(*((*WIN_RMDIR, RECORDINGS_DIR) if self.windows else - ("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true"))) + await self._remove_recordings(made) + + async def _annotation(self) -> None: + if not self.driver.declares(COMPUTER_RECORDING): + return + params = self.driver.params(COMPUTER_RECORDING) + fps = min(RECORD_FPS, (params.get("fps") or {}).get("max") or RECORD_FPS) + made: list[str] = [] + + async def annotated() -> str | None: + """A recording with annotate draws the probe's click where it landed, from no later than the frame that + first shows the click's effect, and no frame before the click draws it.""" + if "annotate" not in params: + raise NotApplicable("the card lists no annotate in computer-recording/v1's params") + started = await self.driver.invoke(COMPUTER_RECORDING, "start", { + "request_id": f"{PROBE_ROLE}-annotated-{uuid.uuid4().hex[:8]}", "fps": fps, "annotate": {}}) + made.append(started["recording_id"]) + await asyncio.sleep(ANNOTATE_LEAD_S) + frame, scale, img = await self._base() + size = self.layout["cell"] + x, y = self._at("grid", scale, size + size / 2, 6 * size + size / 2) + inside = (x + math.floor(size * 0.36 * scale), y + math.floor(size * 0.36 * scale)) + was = img.getpixel(inside) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + await asyncio.sleep(ANNOTATE_TAIL_S) + wire = await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": started["recording_id"]}) + return await self._annotated_click(wire, img.size, (x, y), inside, was) + + try: + await self.check("X", "an annotated recording draws a click where it landed, from the frame its effect " + "shows on", annotated, COMPUTER_RECORDING) + finally: + await self._remove_recordings(made) + + async def _remove_recordings(self, made: list[str]) -> None: + """End each recording the probe started, and remove what it kept on the computer.""" + for recording_id in made: + with contextlib.suppress(Exception): + await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) + await self.driver.run(*(WIN_REMOVE if self.windows else ("rm", "-rf")), f"{RECORDINGS_DIR}/{recording_id}") + await self.driver.run(*((*WIN_RMDIR, RECORDINGS_DIR) if self.windows else + ("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true"))) async def _segments_match(self, wire: dict, fps: int) -> str | None: """What is wrong with a recording without a grant, read back from the computer, or None.""" @@ -1110,22 +1154,9 @@ async def _segments_match(self, wire: dict, fps: int) -> str | None: if gap > 2 / fps: return f"{gap:.2f} s between segments {before.index} and {after.index}" for segment in segments: - path = f"{RECORDINGS_DIR}/{segment.path}" - out = await self.driver.run("python3", "-c", READ_DIGEST, path) - if out.get("exit_code") != 0: - return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}" - file = json.loads(out["stdout"]) - if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256): - return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ - f"the manifest says {segment.bytes}, {segment.sha256[:12]}" - data = b"" - while len(data) < file["size"]: - chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) - read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" - if not read: - return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \ - f"{chunk.get('stderr', '')[-200:]}" - data += read + data = await self._read_back(segment) + if isinstance(data, str): + return data mp4 = read_mp4(data) span = (_when(segment.ended_at) - _when(segment.started_at)).total_seconds() if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment.width, segment.height, segment.frames) @@ -1135,6 +1166,79 @@ async def _segments_match(self, wire: dict, fps: int) -> str | None: return None + async def _read_back(self, segment: ManifestSegment) -> bytes | str: + """A segment a recording without a grant kept on the computer, read back and matched to its manifest entry's + size and digest; what is wrong, as a string, when it can't be.""" + path = f"{RECORDINGS_DIR}/{segment.path}" + out = await self.driver.run("python3", "-c", READ_DIGEST, path) + if out.get("exit_code") != 0: + return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}" + file = json.loads(out["stdout"]) + if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256): + return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ + f"the manifest says {segment.bytes}, {segment.sha256[:12]}" + data = b"" + while len(data) < file["size"]: + chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) + read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" + if not read: + return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \ + f"{chunk.get('stderr', '')[-200:]}" + data += read + return data + + async def _annotated_click(self, wire: dict, size: tuple[int, int], point: tuple[int, int], + inside: tuple[int, int], was: tuple[int, int, int]) -> str | None: + """What is wrong with an annotated recording of one click at frame pixel ``point`` of a frame ``size``, the + clicked cell showing ``was`` at ``inside`` before it, or None. All its times are the computer's own.""" + manifest = RecordingManifest.model_validate(wire) + if manifest.annotate != {}: + return f"its manifest's annotate is {manifest.annotate!r}, not the {{}} it was started with" + frames: list[tuple[datetime, Any, tuple[int, int]]] = [] + for segment in manifest.segments: + data = await self._read_back(segment) + if isinstance(data, str): + return data + with av.open(io.BytesIO(data)) as container: + decoded = list(container.decode(container.streams.video[0])) + for f in decoded: + frames.append((_when(segment.started_at) + timedelta(seconds=f.time - decoded[0].time), f.to_image(), + (f.width, f.height))) + if not frames: + return "it kept no frames" + + def at(xy: tuple[int, int], shape: tuple[int, int]) -> tuple[int, int]: + return (min(shape[0] - 1, xy[0] * shape[0] // size[0]), min(shape[1] - 1, xy[1] * shape[1] // size[1])) + + def marked(picture: Any, shape: tuple[int, int]) -> bool: + cx, cy = at(point, shape) + reach = max(8, round(40 * max(shape) / 1280)) + near = sum(1 for px in range(max(0, cx - reach), min(shape[0], cx + reach)) + for py in range(max(0, cy - reach), min(shape[1], cy + reach)) + if _distance(picture.getpixel((px, py)), MARK_COLOURS[0]) < NEAR_COLOUR) + return near > 20 + + def changed(picture: Any, shape: tuple[int, int]) -> bool: + return _distance(picture.getpixel(at(inside, shape)), was) >= NEAR_COLOUR + + first_marked = next((i for i, (_, picture, shape) in enumerate(frames) if marked(picture, shape)), None) + first_changed = next((i for i, (_, picture, shape) in enumerate(frames) if changed(picture, shape)), None) + if first_changed is None: + return f"the clicked cell never changed in its {len(frames)} frames, so the click can't be placed on them" + if first_marked is None: + return f"none of its {len(frames)} frames draws the click's mark at {point}" + if first_marked == 0: + return "its first frame, captured before the click, already draws a mark there" + if first_marked > first_changed: + late = (frames[first_marked][0] - frames[first_changed][0]).total_seconds() + return f"the mark first shows {late:.2f} s after the frame where the click's effect does" + return None + + +def _distance(a: tuple, b: tuple) -> int: + return sum(abs(x - y) for x, y in zip(a[:3], b[:3])) + + def _when(iso: str) -> datetime: return datetime.fromisoformat(iso.replace("Z", "+00:00")) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index bd5dc6856..70c5f7129 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -6,6 +6,7 @@ import asyncio import logging import os +from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional @@ -14,12 +15,12 @@ from starlette.responses import Response from starlette.websockets import WebSocket -from ..agent_env_environment import (_CARD_CONFIG_ATTR, _TOOL_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication, - AgentEnvError, _error_response, _marked_members, add_websocket_route, - control_check, environment_card) +from ..agent_env_environment import (_CARD_CONFIG_ATTR, _TOOL_ATTR, _TOOL_OPTIONS_ATTR, AgentEnvEnvironment, + AgentEnvFastMCPApplication, AgentEnvError, _error_response, _marked_members, + add_websocket_route, control_check, environment_card) from ..tool_context import ToolContext from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool -from . import _linux, apps, card, svc +from . import _linux, annotate, apps, card, svc from .checks import check_features, check_names from .dataplane import DataPlane from .errors import TargetUnavailable @@ -127,11 +128,29 @@ def declaration(cls, platform: Platform) -> EnvironmentCard: return AgentEnvFastMCPApplication(card, driver).environment_card def on_tool_call(self, context: ToolContext) -> Any: - """Note that an agent has acted, for the apps whose loads change what's on screen only before one has: a - subclass that has its own calls this one too.""" + """Note that an agent has acted, for the apps whose loads change what's on screen only before one has, and + draw a call to one of the driver's own or forwarded tools that may act on the recordings that draw inputs, + since it acts without ``act``, which marks the SDK's: a subclass that has its own calls this one too.""" apps.note_agent_acted(self._computer.apps) + if self._computer.recordings.annotating(): + name = self._computer_prefix.own(context.tool) + if name not in COMPUTER_TOOLS and self._computer_tool_acts(name): + self._computer.recordings.note([annotate.tool_mark(name, context.arguments, role=context.role, + at=datetime.now(timezone.utc))]) return None + def _computer_tool_acts(self, name: str) -> bool: + """Whether the driver's own or forwarded tool ``name`` may act: its annotations don't say it only reads.""" + acting = self.__dict__.get("_computer_acting") + if acting is None: + acting = {forwarded.tool.name: not _reads_only(forwarded.annotations) + for forwarded in self.forwarded_tools()} + for member, marker in _marked_members(self, _TOOL_ATTR): + options = getattr(member, _TOOL_OPTIONS_ATTR, None) + acting[marker.name] = not _reads_only(options.annotations if options is not None else None) + self._computer_acting = acting + return acting.get(name, True) + def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" return self._computer.has(capability) @@ -259,3 +278,9 @@ def deco(cls: type) -> type: cls.supports = list(supports) return environment_card(name=name)(cls) if name else cls return deco + + +def _reads_only(annotations: Any) -> bool: + """Whether a tool's MCP annotations, as a dict or a ``ToolAnnotations``, say it only reads.""" + hint = annotations.get("readOnlyHint") if isinstance(annotations, dict) else getattr(annotations, "readOnlyHint", None) + return hint is True diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 0dde59926..f3bf47361 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -129,12 +129,14 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st @extension(COMPUTER_RECORDING, op="start", description="Record the display: start returns a recording whose segments upload through its grant as " - "they close; end seals it and returns its manifest, uploaded beside them; end_all ends " - "every one not yet ended; list names those still recording; repeats answer the same.") + "they close, with annotate drawing each input over the frames; end seals it and returns " + "its manifest, uploaded beside them; end_all ends every one not yet ended; list names " + "those still recording; repeats answer the same.") async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, - upload: Optional[dict] = None) -> dict: - return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload) + upload: Optional[dict] = None, annotate: Optional[dict] = None) -> dict: + return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload, + annotate) @extension(COMPUTER_RECORDING, op="end") async def _computer_recording_end(self, recording_id: str, upload: Optional[dict] = None, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py index a5b851d97..f0a49a025 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py @@ -44,6 +44,8 @@ class RecordingManifest(BaseModel): ended_by: EndedBy segments: list[ManifestSegment] error: Optional[str] = None + #: The ``annotate`` the recording drew its inputs with; none for a plain recording. + annotate: Optional[dict] = None def wire(self) -> dict: """As JSON goes on the wire, without the fields it doesn't know.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index a6c7c846a..b9f86e0c1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -28,6 +28,7 @@ from pydantic import ValidationError from ..transfers import NamespaceUploader, WriteNamespaceGrant +from . import annotate as annotating from .errors import InvalidArgument, LimitExceeded, Unsupported from .frames import to_pil from .manifest import ManifestSegment, RecordingManifest @@ -126,7 +127,8 @@ def recording_params(recording: Recording, streams: bool) -> dict: most = recording.max_fps if streams else min(recording.max_fps, MAX_POLLED_FPS) return {"formats": [MP4], "scopes": ["display"], "fps": {"default": min(DEFAULT_FPS, most), "max": most}, "cursor": True, "source": "stream" if streams else "capture", "max_segment_s": recording.max_segment_s, - "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s} + "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s, + "annotate": {"options": list(annotating.OPTIONS)}} class Recordings: @@ -142,7 +144,8 @@ def __init__(self, runtime: ComputerRuntime) -> None: self._lock = asyncio.Lock() async def start(self, request_id: str, scope: str, window: Optional[str], fps: Optional[int], - max_edge: Optional[int], cursor: bool, upload: Optional[dict]) -> dict: + max_edge: Optional[int], cursor: bool, upload: Optional[dict], + annotate: Optional[dict] = None) -> dict: features: Recording = self.runtime.driver.recording if not (isinstance(request_id, str) and request_id): raise InvalidArgument("request_id is a non-empty string") @@ -152,6 +155,7 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O await self._live(self.running.get(answer["recording_id"])) return answer hints, grant = self._hints(features, scope, window, fps, max_edge, cursor, upload) + annotate = annotating.check_annotate(annotate) stopped = [rid for rid, r in self.running.items() if r.done] for rid in stopped[:max(0, len(stopped) - KEEP_ENDED)]: # stopped by itself and never ended self.running.pop(rid) @@ -166,7 +170,8 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O cursor = hints.cursor if streams else hints.cursor and runtime.has("pointer") source = (lambda hints: runtime.stream(hints)) if streams else \ (lambda hints: _poll(runtime, hints, cursor)) - recording = Video(recording_id, hints, features, root, grant, source, streams=streams, cursor=cursor) + recording = Video(recording_id, hints, features, root, grant, source, streams=streams, cursor=cursor, + annotate=annotate) self.running[recording_id] = recording recording.begin() answer = {"recording_id": recording_id, "started_at": _iso(recording.requested_at), "format": MP4} @@ -203,6 +208,16 @@ async def listed(self) -> list[dict]: return [{"recording_id": rid, "started_at": _iso(r.requested_at)} for rid, r in self.running.items() if not r.done] + def annotating(self) -> bool: + """Whether any recording still recording draws its inputs.""" + return any(r.annotations is not None and not r.done for r in self.running.values()) + + def note(self, marks: list[annotating.Mark]) -> None: + """Draw ``marks`` on every recording still recording that draws its inputs.""" + for recording in self.running.values(): + if recording.annotations is not None and not recording.done: + recording.annotations.add(marks) + async def end_all(self, reason: str) -> list[dict]: """End every recording not yet ended, each uploading under the grant it was started with; their manifests.""" return [await self.end(recording_id, None, reason) for recording_id in list(self.running)] @@ -227,11 +242,14 @@ class Video: """One recording's video: frames or segments from ``source(hints)``, encoded as H.264 segments in ``hints.directory``, each uploaded under ``grant`` as it closes, at ``/.mp4``, ``prefix`` its id by default; sealed by ``end``, which answers its manifest and, with ``manifest``, uploads it beside the segments. - ``streams`` says the source streams rather than polls, ``cursor`` that its frames draw the pointer.""" + ``streams`` says the source streams rather than polls, ``cursor`` that its frames draw the pointer, and + ``annotate``, when given, that each frame draws the inputs it shows (``annotate.Annotations``); a segment a native + recorder finished is kept as it came.""" def __init__(self, recording_id: str, hints: RecordHints, features: Recording, root: Path, grant: Optional[WriteNamespaceGrant], source: Callable[[RecordHints], AsyncIterator], *, - streams: bool, cursor: bool, prefix: Optional[str] = None, manifest: bool = True) -> None: + streams: bool, cursor: bool, prefix: Optional[str] = None, manifest: bool = True, + annotate: Optional[dict] = None) -> None: self.id = recording_id self.hints = hints self.features = features @@ -243,6 +261,8 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.requested_at = datetime.now(timezone.utc) self.streams = streams self.cursor = cursor + self.annotate = annotate + self.annotations = annotating.Annotations() if annotate is not None else None self.segments: list[_Segment] = [] self.encoder: Optional[_Encoder] = None self.ended_by: Optional[str] = None @@ -341,15 +361,16 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: recording_id=self.id, scope=self.hints.scope, format=MP4, fps=self.hints.fps, cursor=self.cursor, source="stream" if self.streams else "capture", started_at=entries[0].started_at if entries else None, ended_at=entries[-1].ended_at if entries else None, ended_by=self.ended_by, segments=entries, - error=self.error if self.ended_by == "error" else None).wire() + error=self.error if self.ended_by == "error" else None, annotate=self.annotate).wire() if uploader is not None and self.uploads_manifest: # beside the segments, so a reader who didn't end the with contextlib.suppress(Exception): # recording finds it await uploader.upload(f"{self.prefix}/{MANIFEST}", json.dumps(manifest).encode()) return manifest async def _add_frame(self, item: RecordedFrame) -> None: - frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge) at = item.captured_at + marks = self.annotations.showing(at) if self.annotations is not None else [] + frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge, marks, at) if self.encoder is not None and ((frame.width, frame.height) != self.encoder.size or (at - self.encoder.last_at).total_seconds() > GAP_S or (at - self.encoder.started_at).total_seconds() >= self.hints.max_segment_s): @@ -528,8 +549,11 @@ def _draw_pointer(image: PILImage.Image, x: float, y: float) -> None: ImageDraw.Draw(image).polygon([(x + dx, y + dy) for dx, dy in arrow], fill=(0, 0, 0), outline=(255, 255, 255)) -def _video_frame(image: Any, max_edge: Optional[int]) -> Any: - """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``.""" +def _video_frame(image: Any, max_edge: Optional[int], marks: list = (), at: Optional[datetime] = None) -> Any: + """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``, with ``marks`` drawn as a + frame captured ``at`` shows them.""" + if marks: + image = annotating.draw(image, marks, at) frame = image if isinstance(image, av.VideoFrame) else av.VideoFrame.from_image(image.convert("RGB")) width, height = frame.width, frame.height if max_edge is not None and max(width, height) > max_edge: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 2cbf236cb..597333008 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -7,13 +7,14 @@ import contextlib from collections import OrderedDict from collections.abc import AsyncIterator +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Awaitable, Callable from mcp.types import CallToolResult from ..errors import AgentEnvError from ..tool_context import ToolContext -from . import apps +from . import annotate, apps from .errors import DeliveryFailed, DeviceUnavailable, DialogOpen, InputBusy, Internal from .frames import Frame, FrameStore, to_pil from .operations import DIALOG_BLOCKS, GROUPS, INPUT, Display, Image, PageState, ReadyCheck, resolve @@ -124,6 +125,8 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra args, summary = (sx, sy, *args), f"{summary} at the pointer, screen ({sx}, {sy})" others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None + if self.recordings.annotating(): + await self._annotate(ctx, operation, args, move_to, repeat, hold) try: if move_to is not None: await self.call("pointer_move", *move_to) @@ -156,6 +159,18 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result() return send(result, new.encode(), new.captured_at) + async def _annotate(self, ctx: ToolContext, operation: str, args: tuple, move_to: tuple | None, repeat: int, + hold: tuple | None) -> None: + """Mark an input on the recordings that draw them, as it is about to be delivered: on the display it acts on, + at its point, or for a press or release with none at the pointer.""" + display = await self.display_now() + point = move_to + if point is None and operation in ("pointer_down", "pointer_up") and self.has("pointer"): + point = await self.pointer() + self.recordings.note(annotate.input_marks( + operation, args, role=ctx.role, at=datetime.now(timezone.utc), size=(display.width, display.height), + point=point, repeat=repeat, hold_ms=hold[0] if hold is not None else None)) + async def _hold_for(self, ms: int, keys: list[str]) -> None: """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" release = ("key_up", list(reversed(keys))) diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index 6ce87bd3a..751293485 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -40,6 +40,8 @@ ) from agentenv_protocol.computer import _linux as linux from agentenv_protocol.computer.conformance.checks import OWNER, STAGE +from agentenv_protocol.computer.conformance.desk import CELL +from agentenv_protocol.computer.conformance.desk import COLOURS as CELL_COLOURS from agentenv_protocol.computer.conformance.page_fixture import QUESTION, TITLE, TWO, TWO_TITLE CLIPBOARD = "agentenv-probe clipboard" @@ -94,8 +96,10 @@ def click(self, x: float, y: float, button: str, count: int) -> None: if not self.started: return if (at := self._in("grid", x, y)) is not None: - key = f"{int(at[1] // 50)},{int(at[0] // 50)}" + row, col = int(at[1] // CELL), int(at[0] // CELL) + key = f"{row},{col}" self.state["cells"][key] = (self.state["cells"].get(key, 0) + count) % 4 + self._fill_cell(row, col, CELL_COLOURS[self.state["cells"][key]]) self.state["right_clicks"] += button == "right" self.state["double_clicks"] += count >= 2 elif self._in("save", x, y) is not None: @@ -106,6 +110,16 @@ def click(self, x: float, y: float, button: str, count: int) -> None: if self._in(box, x, y) is not None: self.focus = box + def _fill_cell(self, row: int, col: int, colour: str) -> None: + """Repaint a grid cell its new colour, as the Fixture Desk does, but for the pixel the click painted.""" + x0, y0 = self.layout["grid"][0] + col * CELL, self.layout["grid"][1] + row * CELL + rgb = (int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16)) + pointer = self.driver.screen.pointer + for x in range(x0 + 1, x0 + CELL - 1): + for y in range(y0 + 1, y0 + CELL - 1): + if (x, y) != tuple(pointer): + self.driver.screen.paint(x, y, rgb) + def type(self, text: str) -> None: if self.focus is None: return diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 59a3badb8..0c6fb236d 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -140,7 +140,8 @@ class Off(FakeDriver): assert set(extension["params"]["methods"]) == {"start", "end", "end_all", "list"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, - "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400} + "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400, + "annotate": {"options": []}} assert COMPUTER_RECORDING not in [e["uri"] for e in off["capabilities"]["extensions"]] diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py new file mode 100644 index 000000000..7202de0ee --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -0,0 +1,215 @@ +"""computer-recording/v1 ``annotate``: a recording started with it draws each input where it landed, in the colour +of the role that sent it, on the frames captured after it; a call to the driver's own or forwarded tool that acts +shows as a chip; a plain recording draws nothing.""" +from __future__ import annotations + +import asyncio +import os +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") +av = pytest.importorskip("av") + +from computer_fake import FakeDriver, http_client, mcp_session # noqa: E402 +from mcp.types import CallToolResult, TextContent # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol import EnvironmentTool, ForwardedTool # noqa: E402 +from agentenv_protocol.computer import COMPUTER_RECORDING, Recording # noqa: E402 +from agentenv_protocol.computer.annotate import ( # noqa: E402 + COLOURS, + Annotations, + draw, + input_marks, + tool_mark, +) +from agentenv_protocol.tool_context import ROLE_META_KEY # noqa: E402 + +START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" +T0 = datetime(2026, 11, 19, 12, tzinfo=timezone.utc) +CLICK = (400, 300) + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +class Recorded(FakeDriver): + recording = Recording(max_segment_s=60) + + +class Forwarding(Recorded): + """A driver serving two tools of another server's: one that acts, one its annotations say only reads.""" + + def forwarded_tools(self): + schema = {"type": "object", "properties": {"text": {"type": "string"}}} + return [ForwardedTool(EnvironmentTool(name="click", description="Click an element.", inputSchema=schema), + self.answer), + ForwardedTool(EnvironmentTool(name="get_window_state", description="Read a window.", + inputSchema=schema), self.answer, {"readOnlyHint": True})] + + async def answer(self, **arguments): + return CallToolResult(content=[TextContent(type="text", text="done")]) + + +async def _clicked(driver, start: dict, meta: dict | None = None) -> tuple[datetime, dict, list]: + """A recording started with ``start`` around one click at CLICK: when the click was sent, the manifest, and each + frame with its capture time.""" + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, **start})).json() + await asyncio.sleep(0.5) + shot = await session.call_tool("computer_screenshot", {}) + sent = datetime.now(timezone.utc) + await session.call_tool("computer_click", {"x": CLICK[0], "y": CLICK[1], + "frame_id": shot.structuredContent["frame_id"]}, meta=meta) + await asyncio.sleep(0.5) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + return sent, manifest, _timed_frames(manifest) + + +def _timed_frames(manifest: dict) -> list[tuple[datetime, PILImage.Image]]: + frames = [] + for segment in manifest["segments"]: + started = datetime.fromisoformat(segment["started_at"].replace("Z", "+00:00")) + path = f"{os.environ['HOME']}/.agentenv-recordings/{segment['path']}" + with av.open(path) as container: + decoded = list(container.decode(container.streams.video[0])) + for frame in decoded: + frames.append((started + timedelta(seconds=frame.time - decoded[0].time), frame.to_image())) + return frames + + +def _marked(image: PILImage.Image, point: tuple[int, int], colour: tuple[int, int, int], reach: int = 40) -> int: + """How many pixels within ``reach`` of ``point`` are near ``colour``, as an encoded frame keeps it.""" + x, y = point + near = 0 + for px in range(max(0, x - reach), min(image.width, x + reach)): + for py in range(max(0, y - reach), min(image.height, y + reach)): + if sum(abs(a - b) for a, b in zip(image.getpixel((px, py)), colour)) < 90: + near += 1 + return near + + +def _split(frames: list, sent: datetime) -> tuple[list, list]: + """The frames captured before the click was sent, and those while its mark shows.""" + before = [image for at, image in frames if at < sent - timedelta(milliseconds=50)] + after = [image for at, image in frames if sent + timedelta(milliseconds=150) <= at <= sent + timedelta(seconds=0.5)] + assert before and after, f"{len(frames)} frames, {len(before)} before the click and {len(after)} after it" + return before, after + + +@pytest.mark.asyncio +async def test_an_annotated_recording_draws_a_click_where_it_landed_on_the_frames_after_it(): + sent, manifest, frames = await _clicked(Recorded(), {"annotate": {}}) + + before, after = _split(frames, sent) + assert manifest["annotate"] == {} + assert not any(_marked(image, CLICK, COLOURS[0]) for image in before) + assert all(_marked(image, CLICK, COLOURS[0]) > 20 for image in after) + + +@pytest.mark.asyncio +async def test_a_plain_recording_draws_nothing_and_its_manifest_names_no_annotate(): + sent, manifest, frames = await _clicked(Recorded(), {}) + + _before, after = _split(frames, sent) + assert "annotate" not in manifest + assert not any(_marked(image, CLICK, COLOURS[0]) for image in after) + + +@pytest.mark.asyncio +async def test_each_role_draws_in_its_own_colour(): + sent, _manifest, frames = await _clicked(Recorded(), {"annotate": {}}, meta={ROLE_META_KEY: "user"}) + + _before, after = _split(frames, sent) + assert all(_marked(image, CLICK, COLOURS[1]) > 20 and not _marked(image, CLICK, COLOURS[0]) for image in after) + + +@pytest.mark.asyncio +async def test_a_driver_tool_that_acts_shows_as_a_chip_and_one_that_only_reads_or_the_sdks_own_does_not(): + driver = Forwarding() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + await session.call_tool("click", {"text": "Save"}) + await session.call_tool("get_window_state", {"text": "Notes"}) + await session.call_tool("computer_screenshot", {}) + labels = [mark.label for mark in recording.annotations.marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert labels == ['click "Save"'] + + +@pytest.mark.asyncio +async def test_start_refuses_an_annotate_that_isnt_an_object_or_names_an_option_it_doesnt_take(): + async with http_client(Recorded()) as http: + option = await http.post(START, json={"request_id": "a", "annotate": {"events": True}}) + scalar = await http.post(START, json={"request_id": "b", "annotate": True}) + + assert option.status_code == scalar.status_code == 400 + assert option.json()["error"]["code"] == scalar.json()["error"]["code"] == "invalid_argument" + assert "takes no options yet, got ['events']" in option.json()["error"]["message"] + + +@pytest.mark.asyncio +async def test_the_card_lists_annotate_and_its_options_with_the_recording_params(): + async with http_client(Recorded()) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + + [extension] = [e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING] + assert extension["params"]["annotate"] == {"options": []} + + +def test_each_input_is_marked_by_what_it_did_and_who_did_it(): + def labels(operation, args, role=None, **kw): + return [(m.kind, m.label) for m in input_marks(operation, args, role=role, at=T0, size=(1000, 500), **kw)] + + assert labels("click", (500, 250, "left", 2, ["Shift"])) == [("click", ""), ("chip", "Shift + click")] + assert labels("type_text", ("Q4 165\n",)) == [("chip", 'type "Q4 165[Enter]"')] + assert labels("press", (["Control", "s"],), repeat=3) == [("chip", "key Ctrl+S x3")] + assert labels("key_down", (["Shift"],), hold_ms=500) == [("chip", "hold Shift 500 ms")] + assert labels("pointer_down", ("left",)) == [("chip", "press left")] + assert labels("pointer_down", ("left",), point=(10, 20)) == [("down", "")] + assert labels("navigate", ("https://example.com",), role="user") == [("chip", "user: open https://example.com")] + assert labels("tabs", ("list", None, None)) == [] + [click] = input_marks("click", (500, 250, "right", 1, []), role=None, at=T0, size=(1000, 500)) + assert (click.points, click.button, click.role) == (((0.5, 0.5),), "right", "default") + [drag] = input_marks("drag", ([(0, 0), (1000, 500)], "left", 400, []), role=None, at=T0, size=(1000, 500)) + assert drag.moving_s == 0.4 and drag.until - drag.at == timedelta(seconds=1.2) + + +def test_a_tools_chip_shows_its_main_argument(): + def label(arguments): + return tool_mark("hotkey", arguments, role=None, at=T0).label + + assert label({"keys": ["cmd", "s"]}) == "hotkey cmd+S" + assert label({"text": "x" * 100}) == 'hotkey "' + "x" * 59 + '…"' + assert label({"window_id": 4}) == "hotkey" + assert tool_mark("click", {"element": 7}, role="executor", at=T0).label == "executor: click 7" + + +def test_a_mark_shows_from_its_input_until_it_ends_and_is_then_forgotten(): + annotations = Annotations() + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + + assert annotations.showing(T0 - timedelta(milliseconds=10)) == [] + assert len(annotations.showing(T0 + timedelta(milliseconds=500))) == 1 + assert annotations.showing(T0 + timedelta(seconds=1)) == [] and annotations.marks == [] + + +def test_a_mark_lands_at_its_point_on_a_frame_of_any_size(): + """A frame scaled down by max_edge, or captured at a HiDPI scale, keeps the point where the input landed.""" + annotations = Annotations() + annotations.add(input_marks("click", (500, 250, "left", 1, []), role=None, at=T0, size=(1000, 500))) + at = T0 + timedelta(milliseconds=400) + + for size in ((1000, 500), (400, 200), (2000, 1000)): + image = draw(PILImage.new("RGB", size, (255, 255, 255)), annotations.showing(at), at) + cx, cy = size[0] // 2, size[1] // 2 + assert any(image.getpixel((cx + dx, cy + dy)) == COLOURS[0] for dx in range(-3, 4) for dy in range(-3, 4)), size + assert image.getpixel((5, 5)) == (255, 255, 255) diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 591a04aaf..4bbb5c5db 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -48,26 +48,33 @@ class StartComputerRecordingTaskStep(TaskStep): - """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``. Best effort - by default.""" + """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``; with + ``annotate``, an object of options (``{}`` for none), its driver draws each input over the frames. Best effort by + default.""" type: ClassVar[str] = "start_computer_recording" entity_refs = (EntityRef.env("env_id"),) def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, environment_name: Optional[str] = None, - depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): + annotate: Optional[dict] = None, depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = False): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) if not env_id or not name: raise ValueError("start_computer_recording needs an env_id and a name for the recording") + if annotate is not None and not isinstance(annotate, dict): + raise ValueError(f"start_computer_recording's annotate is an object of options, {{}} for none; got " + f"{annotate!r}") self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor - self.environment_name = environment_name + self.environment_name, self.annotate = environment_name, annotate def to_dict(self) -> dict: data = {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor} if self.environment_name is not None: # left out otherwise, so a step without one serializes as before data["environment_name"] = self.environment_name + if self.annotate is not None: + data["annotate"] = self.annotate return data @classmethod @@ -76,7 +83,7 @@ def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: base["fail_task_on_error"] = data.get("fail_task_on_error", False) return cls(**base, env_id=data["env_id"], name=data["name"], fps=data.get("fps"), max_edge=data.get("max_edge"), cursor=data.get("cursor", True), - environment_name=data.get("environment_name")) + environment_name=data.get("environment_name"), annotate=data.get("annotate")) async def execute(self, context: TaskStepContext) -> TaskStepContext: recordings = context.metadata.setdefault(RECORDINGS_KEY, {}) @@ -93,10 +100,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = _deployed(context, self.env_id) refuse_no_network(deployed) namespace = recording_namespace(request_id, _KEY_PREFIX) - grant = _grant(deployed, namespace, self.environment_name) + grant = _grant(deployed, namespace, self.environment_name, annotate=self.annotate is not None) started = await deployed.invoke(COMPUTER_RECORDING, "start", { "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, - "upload": grant}, environment_name=self.environment_name) + "upload": grant, **({"annotate": self.annotate} if self.annotate is not None else {})}, + environment_name=self.environment_name) except Exception as e: recordings[self.name] = {"env_id": self.env_id, "state": "failed", "error": why(e)} logger.warning("start_computer_recording %s on %s: %s", self.name, self.env_id, e) @@ -207,14 +215,21 @@ def _deployed(context: TaskStepContext, env_id: str): return deployed -def _grant(deployed, namespace: str, environment_name: Optional[str] = None, method: str = "start") -> dict: +def _grant(deployed, namespace: str, environment_name: Optional[str] = None, method: str = "start", + annotate: bool = False) -> dict: """A grant for a computer recording's segments and manifest. EnvCapabilityUnsupported when the env's card, or its - child's, declares no recording.""" + child's, declares no recording; with ``annotate``, a RuntimeError when it declares no ``annotate``, as a driver + from before agent-env drew inputs does.""" card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) extension = find_extension(card or {}, COMPUTER_RECORDING) if extension is None: raise EnvCapabilityUnsupported(COMPUTER_RECORDING, method, deployed.env_id, getattr(deployed, "instance_id", None), environment_name) + if annotate and "annotate" not in (extension.get("params") or {}): + who = f"{deployed.env_id}'s {environment_name}" if environment_name else deployed.env_id + raise RuntimeError(f"{who} can't annotate a recording: its driver's computer-recording/v1 lists no annotate, " + "as a driver composed before agent-env drew inputs; compose it again, or record it " + "without annotate") return recording_grant(deployed, extension.get("params") or {}, namespace, MAX_SEGMENTS + 1, _MAX_SEGMENT_BYTES, MAX_RECORDING_BYTES) diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 6b05e0feb..80653ccf7 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -341,3 +341,37 @@ async def test_a_start_without_a_child_env_reads_the_envs_own_card_as_before(sto assert entry["error"].startswith("EnvCapabilityUnsupported: env 'shop' does not offer 'start' on ") context = await _start().execute(_context(_Computer(store))) assert "environment_name" not in context.metadata[RECORDINGS_KEY]["desk"] + + +_ANNOTATING = {**_CARD, "capabilities": {"extensions": [{**_CARD["capabilities"]["extensions"][0], "params": { + **_CARD["capabilities"]["extensions"][0]["params"], "annotate": {"options": []}}}]}} + + +@pytest.mark.asyncio +async def test_annotate_reaches_the_driver_only_when_a_step_asks_for_it(store): + computer, plain = _Computer(store), _Computer(store) + computer.environment_card = plain.environment_card = _ANNOTATING + await _start(annotate={}).execute(_context(computer)) + await _start().execute(_context(plain)) + + [(_, annotated)], [(_, unannotated)] = computer.calls, plain.calls + assert annotated["annotate"] == {} and "annotate" not in unannotated + + +@pytest.mark.asyncio +async def test_annotating_on_a_driver_whose_card_lists_no_annotate_fails_the_start_best_effort_or_fatally(store): + computer = _Computer(store) + context = await _start(annotate={}).execute(_context(computer)) + entry = context.metadata[RECORDINGS_KEY]["desk"] + + assert entry["state"] == "failed" and "can't annotate a recording" in entry["error"] and computer.calls == [] + with pytest.raises(RuntimeError, match="lists no annotate"): + await _start(annotate={}, fail_task_on_error=True).execute(_context(_Computer(store))) + + +def test_annotate_round_trips_only_when_set_and_is_an_object(): + annotated = _start(annotate={}) + assert StartComputerRecordingTaskStep.from_dict(annotated.to_dict()).annotate == {} + assert "annotate" not in _start().to_dict() + with pytest.raises(ValueError, match="annotate is an object of options"): + _start(annotate=True) From cb4a539ba775259e3a4961688c99ac8acfbe22fc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 11:19:37 -0800 Subject: [PATCH 02/10] fix(computers): an annotated recording's marks show on a late frame and never stop an input A mark shows on the first frame after its input however late it comes, so a low fps or a slow capture can't miss it; making a mark is best effort, reusing the display a mapped input read; a recording of a native recorder's finished segments names no annotate, its marks kept only so long; the probe keeps each frame's findings, not the frames. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 6 +- .../agentenv_protocol/computer/annotate.py | 54 +++++++++--- .../computer/conformance/checks.py | 34 ++++---- .../src/agentenv_protocol/computer/driver.py | 12 ++- .../agentenv_protocol/computer/recording.py | 16 ++-- .../src/agentenv_protocol/computer/runtime.py | 35 +++++--- .../tests/test_computer_recording_annotate.py | 83 ++++++++++++++++++- 7 files changed, 185 insertions(+), 55 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index aa960bef2..a7674b59d 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -774,8 +774,10 @@ class MyDriver(ComputerDriverEnvironment): square for the right button, a ring for each click of a double or triple), a trail for a drag, an arrow for a scroll, a cross for a move, and a chip for a key chord, typed text as it was typed, and each of the driver's own or forwarded tools whose annotations don't say it only reads, by name and - main argument. Every `computer_*` input is marked as `act` delivers it; a segment a native recorder - finished is kept as it came. The manifest's `annotate` is the object the recording drew with. + main argument. Every `computer_*` input is marked as `act` delivers it, best effort, and shows on + every frame while it lasts and on the first one after it however late, as at a low fps. A segment a + native recorder finished is kept as it came. The manifest's `annotate` is the object the recording + drew with, and is left out for one that took a native recorder's segments. - **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path, `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size, SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py index db2033286..d6140859f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -25,12 +25,20 @@ COLOURS = ((255, 138, 0), (0, 184, 148), (175, 82, 222), (255, 59, 105), (52, 120, 246), (250, 204, 21)) #: How long a mark shows from its input, in seconds; a drag's, after its moves. SHOW_S = {"click": 0.8, "down": 0.8, "up": 0.8, "move": 0.5, "scroll": 0.9, "drag": 0.8, "chip": 1.6} +#: The last part of a mark's time, in seconds, over which it fades out. +FADE_S = 0.3 +#: How long a mark no frame has drawn is kept, in seconds: past it, the recording's frames have stopped coming. +KEEP_S = 30.0 +#: The inputs whose marks are drawn at a point of the display. +POINTED = ("click", "drag", "scroll", "pointer_move", "pointer_down", "pointer_up") #: The most a chip of typed text shows, which grows with its length from the chip's own time. MOST_TEXT_S = 4.0 #: The most of a chip's text it shows; the rest is cut with an ellipsis. CHIP_CHARS = 60 #: The most chips shown at once, the newest lowest. MOST_CHIPS = 4 +#: How far above the bottom edge, as a fraction of the frame's height, the lowest chip ends. +CHIPS_ABOVE = 0.1 #: A frame's longer edge that marks are sized for; on a larger or smaller frame they scale with it. UNIT_EDGE = 1280 #: Arguments a tool's chip shows, the first one a call has. @@ -70,22 +78,43 @@ def check_annotate(annotate: object) -> Optional[dict]: class Annotations: - """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role.""" + """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role. Each + mark shows on every frame captured while it lasts, and on the first one after its input however late that comes, + as at a low fps or from a slow capture.""" def __init__(self) -> None: self.marks: list[Mark] = [] + self.drawn: set[int] = set() self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} def add(self, marks: Sequence[Mark]) -> None: for mark in marks: self.colours.setdefault(mark.role, COLOURS[len(self.colours) % len(COLOURS)]) + stale = mark.at - timedelta(seconds=KEEP_S) + self._keep([kept for kept in self.marks if kept.until > stale]) self.marks.append(mark) - def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int]]]: - """The marks a frame captured ``at`` shows, with their colours. A stream's frames come in the order they were - captured, so a mark over by then is over for every later frame too.""" - self.marks = [mark for mark in self.marks if mark.until > at] - return [(mark, self.colours[mark.role]) for mark in self.marks if mark.at <= at] + def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: + """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a + mark over by then that no frame has drawn yet is drawn once, at the age it is last fully shown. A stream's + frames come in the order they were captured, so a mark that is over and drawn is over for every later one.""" + shown = [] + for mark in self.marks: + if mark.at > at: + continue + age = (at - mark.at).total_seconds() + if at >= mark.until: + if id(mark) in self.drawn: + continue + age = max(0.0, (mark.until - mark.at).total_seconds() - FADE_S) + shown.append((mark, self.colours[mark.role], age)) + self.drawn.add(id(mark)) + self._keep([mark for mark in self.marks if mark.until > at or id(mark) not in self.drawn]) + return shown + + def _keep(self, marks: list[Mark]) -> None: + self.marks = marks + self.drawn &= {id(mark) for mark in marks} def input_marks(operation: str, args: tuple, *, role: Optional[str], at: datetime, size: tuple[int, int], @@ -170,20 +199,19 @@ def tool_mark(tool: str, arguments: Mapping[str, Any], *, role: Optional[str], a label=label if role == DEFAULT_ROLE else f"{role}: {label}") -def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int]]], at: datetime) -> PILImage.Image: - """``image``, a Pillow image or a PyAV frame, with ``marks`` drawn as a frame captured ``at`` shows them.""" +def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int], float]]) -> PILImage.Image: + """``image``, a Pillow image or a PyAV frame, with ``marks`` drawn, each in its colour at its age.""" base = (image.to_image() if isinstance(image, av.VideoFrame) else image).convert("RGBA") overlay = PILImage.new("RGBA", base.size, (0, 0, 0, 0)) pen = ImageDraw.Draw(overlay) unit = max(base.size) / UNIT_EDGE chips = [] - for mark, colour in marks: - age = (at - mark.at).total_seconds() + for mark, colour, age in marks: span = max((mark.until - mark.at).total_seconds(), 1e-3) if mark.kind == "chip": chips.append((mark, colour)) continue - fade = 1.0 - max(0.0, (age - span * 0.6) / (span * 0.4)) + fade = 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) points = [(fx * base.width, fy * base.height) for fx, fy in mark.points] _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, span, fade) if chips: @@ -257,10 +285,10 @@ def _draw_drag(pen, mark, points, colour, unit, age, span, fade) -> None: def _draw_chips(pen, chips, size, unit) -> None: - """Chips stacked above the bottom edge, centred, the newest lowest.""" + """Chips stacked centred above the bottom edge, clear of a panel or dock there, the newest lowest.""" font = _font(max(10, round(20 * unit))) pad, gap = unit * 10, unit * 8 - y = size[1] - unit * 48 + y = size[1] - max(unit * 48, size[1] * CHIPS_ABOVE) for mark, colour in reversed(chips): left, top, right, bottom = pen.textbbox((0, 0), mark.label, font=font) w, h = right - left + 2 * pad, bottom - top + 2 * pad * 0.7 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index d4f854aaa..5e7147b82 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -1102,7 +1102,7 @@ async def _annotation(self) -> None: async def annotated() -> str | None: """A recording with annotate draws the probe's click where it landed, from no later than the frame that - first shows the click's effect, and no frame before the click draws it.""" + first shows the click's effect, and its first frame, captured before the click, draws none.""" if "annotate" not in params: raise NotApplicable("the card lists no annotate in computer-recording/v1's params") started = await self.driver.invoke(COMPUTER_RECORDING, "start", { @@ -1192,20 +1192,10 @@ async def _annotated_click(self, wire: dict, size: tuple[int, int], point: tuple """What is wrong with an annotated recording of one click at frame pixel ``point`` of a frame ``size``, the clicked cell showing ``was`` at ``inside`` before it, or None. All its times are the computer's own.""" manifest = RecordingManifest.model_validate(wire) + if manifest.annotate is None: + return "its manifest names no annotate: a recorder that hands over finished segments isn't drawn on" if manifest.annotate != {}: return f"its manifest's annotate is {manifest.annotate!r}, not the {{}} it was started with" - frames: list[tuple[datetime, Any, tuple[int, int]]] = [] - for segment in manifest.segments: - data = await self._read_back(segment) - if isinstance(data, str): - return data - with av.open(io.BytesIO(data)) as container: - decoded = list(container.decode(container.streams.video[0])) - for f in decoded: - frames.append((_when(segment.started_at) + timedelta(seconds=f.time - decoded[0].time), f.to_image(), - (f.width, f.height))) - if not frames: - return "it kept no frames" def at(xy: tuple[int, int], shape: tuple[int, int]) -> tuple[int, int]: return (min(shape[0] - 1, xy[0] * shape[0] // size[0]), min(shape[1] - 1, xy[1] * shape[1] // size[1])) @@ -1221,8 +1211,22 @@ def marked(picture: Any, shape: tuple[int, int]) -> bool: def changed(picture: Any, shape: tuple[int, int]) -> bool: return _distance(picture.getpixel(at(inside, shape)), was) >= NEAR_COLOUR - first_marked = next((i for i, (_, picture, shape) in enumerate(frames) if marked(picture, shape)), None) - first_changed = next((i for i, (_, picture, shape) in enumerate(frames) if changed(picture, shape)), None) + frames: list[tuple[datetime, bool, bool]] = [] # each frame's capture time, whether it draws the mark and + for segment in manifest.segments: # whether it shows the click's effect; not the frames, which run large + data = await self._read_back(segment) + if isinstance(data, str): + return data + with av.open(io.BytesIO(data)) as container: + first: float | None = None + for f in container.decode(container.streams.video[0]): + first = f.time if first is None else first + picture, shape = f.to_image(), (f.width, f.height) + frames.append((_when(segment.started_at) + timedelta(seconds=f.time - first), + marked(picture, shape), changed(picture, shape))) + if not frames: + return "it kept no frames" + first_marked = next((i for i, (_, mark, _) in enumerate(frames) if mark), None) + first_changed = next((i for i, (_, _, change) in enumerate(frames) if change), None) if first_changed is None: return f"the clicked cell never changed in its {len(frames)} frames, so the click can't be placed on them" if first_marked is None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 70c5f7129..92c23c796 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -33,6 +33,7 @@ from .settings import TOOL_PREFIX_ENV, ComputerSettings, load_settings, tool_prefix_problem from .tools import COMPUTER_TOOLS, Tools +_log = logging.getLogger(__name__) #: The caller every call comes from until agents carry an identity of their own. DEFAULT_IDENTITY = "default" @@ -133,10 +134,13 @@ def on_tool_call(self, context: ToolContext) -> Any: since it acts without ``act``, which marks the SDK's: a subclass that has its own calls this one too.""" apps.note_agent_acted(self._computer.apps) if self._computer.recordings.annotating(): - name = self._computer_prefix.own(context.tool) - if name not in COMPUTER_TOOLS and self._computer_tool_acts(name): - self._computer.recordings.note([annotate.tool_mark(name, context.arguments, role=context.role, - at=datetime.now(timezone.utc))]) + try: + name = self._computer_prefix.own(context.tool) + if name not in COMPUTER_TOOLS and self._computer_tool_acts(name): + self._computer.recordings.note([annotate.tool_mark(name, context.arguments, role=context.role, + at=datetime.now(timezone.utc))]) + except Exception as e: # a mark is best effort: the call goes ahead without one + _log.warning("couldn't mark %s on an annotated recording: %s: %s", context.tool, type(e).__name__, e) return None def _computer_tool_acts(self, name: str) -> bool: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index b9f86e0c1..4baa6a341 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -244,7 +244,7 @@ class Video: default; sealed by ``end``, which answers its manifest and, with ``manifest``, uploads it beside the segments. ``streams`` says the source streams rather than polls, ``cursor`` that its frames draw the pointer, and ``annotate``, when given, that each frame draws the inputs it shows (``annotate.Annotations``); a segment a native - recorder finished is kept as it came.""" + recorder finished is kept as it came, so a recording that took one names no ``annotate`` in its manifest.""" def __init__(self, recording_id: str, hints: RecordHints, features: Recording, root: Path, grant: Optional[WriteNamespaceGrant], source: Callable[[RecordHints], AsyncIterator], *, @@ -263,6 +263,7 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.cursor = cursor self.annotate = annotate self.annotations = annotating.Annotations() if annotate is not None else None + self.native = False self.segments: list[_Segment] = [] self.encoder: Optional[_Encoder] = None self.ended_by: Optional[str] = None @@ -361,7 +362,8 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: recording_id=self.id, scope=self.hints.scope, format=MP4, fps=self.hints.fps, cursor=self.cursor, source="stream" if self.streams else "capture", started_at=entries[0].started_at if entries else None, ended_at=entries[-1].ended_at if entries else None, ended_by=self.ended_by, segments=entries, - error=self.error if self.ended_by == "error" else None, annotate=self.annotate).wire() + error=self.error if self.ended_by == "error" else None, + annotate=None if self.native else self.annotate).wire() if uploader is not None and self.uploads_manifest: # beside the segments, so a reader who didn't end the with contextlib.suppress(Exception): # recording finds it await uploader.upload(f"{self.prefix}/{MANIFEST}", json.dumps(manifest).encode()) @@ -370,7 +372,7 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: async def _add_frame(self, item: RecordedFrame) -> None: at = item.captured_at marks = self.annotations.showing(at) if self.annotations is not None else [] - frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge, marks, at) + frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge, marks) if self.encoder is not None and ((frame.width, frame.height) != self.encoder.size or (at - self.encoder.last_at).total_seconds() > GAP_S or (at - self.encoder.started_at).total_seconds() >= self.hints.max_segment_s): @@ -383,6 +385,7 @@ async def _add_frame(self, item: RecordedFrame) -> None: self._check_disk() async def _add_segment(self, item: RecordedSegment) -> None: + self.native = True index = len(self.segments) target = self.hints.directory / f"{index:04d}.mp4" if Path(item.path) != target: @@ -549,11 +552,10 @@ def _draw_pointer(image: PILImage.Image, x: float, y: float) -> None: ImageDraw.Draw(image).polygon([(x + dx, y + dy) for dx, dy in arrow], fill=(0, 0, 0), outline=(255, 255, 255)) -def _video_frame(image: Any, max_edge: Optional[int], marks: list = (), at: Optional[datetime] = None) -> Any: - """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``, with ``marks`` drawn as a - frame captured ``at`` shows them.""" +def _video_frame(image: Any, max_edge: Optional[int], marks: list = ()) -> Any: + """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``, with ``marks`` drawn.""" if marks: - image = annotating.draw(image, marks, at) + image = annotating.draw(image, marks) frame = image if isinstance(image, av.VideoFrame) else av.VideoFrame.from_image(image.convert("RGB")) width, height = frame.width, frame.height if max_edge is not None and max(width, height) > max_edge: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 597333008..855b8b03a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -5,6 +5,7 @@ import asyncio import contextlib +import logging from collections import OrderedDict from collections.abc import AsyncIterator from datetime import datetime, timezone @@ -22,6 +23,8 @@ from .recording import RecordHints, Recordings from .results import EffectData, InputResult, send +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from .driver import ComputerDriverEnvironment @@ -118,15 +121,17 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra if operation in DIALOG_BLOCKS and self.has("page") and (dialog := (await self.page_now()).dialog): raise DialogOpen(f'a {dialog.type} dialog is open on the page: "{dialog.message}"; answer it with ' f"{self.driver._computer_prefix.served('computer_dialog')} first") + display = None if mapped and frame is not None: - self.frames.check_current(frame, await self.display_now()) + display = await self.display_now() + self.frames.check_current(frame, display) if at_pointer: sx, sy = await self.pointer() args, summary = (sx, sy, *args), f"{summary} at the pointer, screen ({sx}, {sy})" others = self.queue.others_since(frame.inputs, identity) if frame is not None else 0 effect = None if self.recordings.annotating(): - await self._annotate(ctx, operation, args, move_to, repeat, hold) + await self._annotate(ctx, operation, args, move_to, repeat, hold, display) try: if move_to is not None: await self.call("pointer_move", *move_to) @@ -160,16 +165,22 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra return send(result, new.encode(), new.captured_at) async def _annotate(self, ctx: ToolContext, operation: str, args: tuple, move_to: tuple | None, repeat: int, - hold: tuple | None) -> None: - """Mark an input on the recordings that draw them, as it is about to be delivered: on the display it acts on, - at its point, or for a press or release with none at the pointer.""" - display = await self.display_now() - point = move_to - if point is None and operation in ("pointer_down", "pointer_up") and self.has("pointer"): - point = await self.pointer() - self.recordings.note(annotate.input_marks( - operation, args, role=ctx.role, at=datetime.now(timezone.utc), size=(display.width, display.height), - point=point, repeat=repeat, hold_ms=hold[0] if hold is not None else None)) + hold: tuple | None, display: Display | None) -> None: + """Mark an input on the recordings that draw them, as it is about to be delivered: one with a point on the + display it acts on, ``display`` when the call already read it, a press or release with none at the pointer. + Best effort: the input goes ahead whether or not its mark could be made.""" + try: + point, size = move_to, (1, 1) + if operation in annotate.POINTED: + display = display or await self.display_now() + size = (display.width, display.height) + if point is None and operation in ("pointer_down", "pointer_up") and self.has("pointer"): + point = await self.pointer() + self.recordings.note(annotate.input_marks( + operation, args, role=ctx.role, at=datetime.now(timezone.utc), size=size, point=point, + repeat=repeat, hold_ms=hold[0] if hold is not None else None)) + except Exception as e: + logger.warning("couldn't mark %s on an annotated recording: %s: %s", operation, type(e).__name__, e) async def _hold_for(self, ms: int, keys: list[str]) -> None: """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index 7202de0ee..976ad4b57 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -18,9 +18,17 @@ from PIL import Image as PILImage # noqa: E402 from agentenv_protocol import EnvironmentTool, ForwardedTool # noqa: E402 -from agentenv_protocol.computer import COMPUTER_RECORDING, Recording # noqa: E402 +from agentenv_protocol.computer import ( # noqa: E402 + COMPUTER_RECORDING, + DeviceUnavailable, + RecordedSegment, + Recording, + record, +) from agentenv_protocol.computer.annotate import ( # noqa: E402 COLOURS, + FADE_S, + KEEP_S, Annotations, draw, input_marks, @@ -43,6 +51,29 @@ class Recorded(FakeDriver): recording = Recording(max_segment_s=60) +class Segmented(FakeDriver): + """A native recorder that hands over a finished segment, which the base class keeps as it came.""" + recording = Recording(max_segment_s=60) + + @record + async def segments(self, hints): + path = hints.directory / "0000.mp4" + start = datetime.now(timezone.utc) + out = av.open(str(path), "w", format="mp4") + stream = out.add_stream("libx264", rate=10) + stream.width, stream.height, stream.pix_fmt = 160, 100, "yuv420p" + for i in range(3): + frame = av.VideoFrame.from_image(PILImage.new("RGB", (160, 100))).reformat(format="yuv420p") + frame.pts = i + for packet in stream.encode(frame): + out.mux(packet) + for packet in stream.encode(): + out.mux(packet) + out.close() + yield RecordedSegment(path, start, start + timedelta(seconds=0.2), 160, 100, 3) + await asyncio.sleep(3600) + + class Forwarding(Recorded): """A driver serving two tools of another server's: one that acts, one its annotations say only reads.""" @@ -209,7 +240,55 @@ def test_a_mark_lands_at_its_point_on_a_frame_of_any_size(): at = T0 + timedelta(milliseconds=400) for size in ((1000, 500), (400, 200), (2000, 1000)): - image = draw(PILImage.new("RGB", size, (255, 255, 255)), annotations.showing(at), at) + image = draw(PILImage.new("RGB", size, (255, 255, 255)), annotations.showing(at)) cx, cy = size[0] // 2, size[1] // 2 assert any(image.getpixel((cx + dx, cy + dy)) == COLOURS[0] for dx in range(-3, 4) for dy in range(-3, 4)), size assert image.getpixel((5, 5)) == (255, 255, 255) + + +def test_a_mark_no_frame_caught_while_it_lasted_shows_fully_on_the_first_frame_after_it_once(): + """At a low fps, or after a slow capture, the next frame may come after a click's mark is over.""" + annotations = Annotations() + [click] = input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100)) + annotations.add([click]) + + [(mark, _colour, age)] = annotations.showing(T0 + timedelta(seconds=3)) + span = (click.until - click.at).total_seconds() + assert mark is click and age == pytest.approx(span - FADE_S) + assert annotations.showing(T0 + timedelta(seconds=4)) == [] and annotations.marks == [] + + +def test_marks_no_frame_draws_are_kept_only_so_long(): + annotations = Annotations() + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + later = T0 + timedelta(seconds=KEEP_S + 1) + annotations.add(input_marks("click", (20, 20, "left", 1, []), role=None, at=later, size=(100, 100))) + + assert [mark.at for mark in annotations.marks] == [later] + + +@pytest.mark.asyncio +async def test_an_input_goes_ahead_when_its_mark_cant_be_made(): + """Reading the display for a mark fails: the scroll is still delivered, unmarked.""" + driver = Recorded() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + await session.call_tool("computer_screenshot", {}) + driver.fail["display"] = DeviceUnavailable("the display went away") + scrolled = await session.call_tool("computer_scroll", {"direction": "down"}) + marks = list(recording.annotations.marks) + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert not scrolled.isError and driver.screen.log[-1][0] == "scroll" and marks == [] + + +@pytest.mark.asyncio +async def test_a_recording_of_a_native_recorders_finished_segments_names_no_annotate(): + driver = Segmented() + async with http_client(driver) as http: + started = (await http.post(START, json={"request_id": "r", "annotate": {}})).json() + await asyncio.sleep(0.3) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + + assert manifest["segments"] and "annotate" not in manifest From a0e7e48cafc8a1d5c29d861b16ebeddc44b28915 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 11:33:41 -0800 Subject: [PATCH 03/10] fix(computers): annotation's probe check doesn't apply to native segments, and every chip shows The probe finds the annotate check not applicable to a recording of a native recorder's finished segments, which aren't drawn on; chips past the most a frame shows wait for a later frame instead of counting as drawn; the drawing's sizes and timings are named in one Style. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/annotate.py | 158 ++++++++++++------ .../computer/conformance/checks.py | 6 +- .../tests/test_computer_recording_annotate.py | 20 +++ 3 files changed, 128 insertions(+), 56 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py index d6140859f..3d20da942 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -98,7 +98,7 @@ def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float] """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a mark over by then that no frame has drawn yet is drawn once, at the age it is last fully shown. A stream's frames come in the order they were captured, so a mark that is over and drawn is over for every later one.""" - shown = [] + due = [] for mark in self.marks: if mark.at > at: continue @@ -107,8 +107,11 @@ def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float] if id(mark) in self.drawn: continue age = max(0.0, (mark.until - mark.at).total_seconds() - FADE_S) - shown.append((mark, self.colours[mark.role], age)) - self.drawn.add(id(mark)) + due.append((mark, self.colours[mark.role], age)) + chips = [mark for mark, _colour, _age in due if mark.kind == "chip"] + waiting = {id(mark) for mark in chips[:-MOST_CHIPS]} # older chips than a frame shows wait for a later one + shown = [entry for entry in due if id(entry[0]) not in waiting] + self.drawn |= {id(mark) for mark, _colour, _age in shown} self._keep([mark for mark in self.marks if mark.until > at or id(mark) not in self.drawn]) return shown @@ -213,71 +216,119 @@ def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int], float]]) continue fade = 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) points = [(fx * base.width, fy * base.height) for fx, fy in mark.points] - _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, span, fade) + _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, fade) if chips: - _draw_chips(pen, chips[-MOST_CHIPS:], base.size, unit) + _draw_chips(pen, chips, base.size, unit) return PILImage.alpha_composite(base, overlay).convert("RGB") -def _draw_click(pen, mark, points, colour, unit, age, span, fade) -> None: +@dataclass(frozen=True) +class Style: + """How marks look: sizes in pixels of a frame whose longer edge is UNIT_EDGE, scaled with the frame; each coloured + stroke over a wider white halo, so a mark reads on any background.""" + + stroke: float = 3 # a ring's and a cross's coloured stroke + halo: float = 5 # the white band under a ring's stroke + line: float = 4 # a trail's and an arrow's coloured stroke + line_halo: float = 7 + outline: float = 1.5 # a dot's white outline + ring_grow_s: float = 0.35 # how long a click's ring takes to grow to its full size, in seconds + ring_from: float = 12 # a click's ring's radius as it starts + ring_grows: float = 18 # and how much it grows + ring_apart: float = 8 # between the rings of a double or triple click + most_rings: int = 3 + dot: float = 5 # the dot at a click's point + press_down: float = 10 # a press's filled ring's radius + press_up: float = 16 # a release's ring's radius + press_fill: float = 0.7 # a press's fill opacity, as a share of its stroke's + cross: float = 9 # each arm of a move's cross + cross_alpha: int = 200 # a move's cross is fainter than the marks of inputs that act + arrow: float = 46 # a scroll's arrow + arrowhead: float = 14 # an arrowhead's sides, beside the stroke's own width + arrowhead_angle: float = 0.5 # radians either side of its line + start_dot: float = 6 # the dot where a drag starts + chip_text: float = 20 + chip_pad: float = 10 # beside a chip's text; above and below it, CHIP_PAD_HIGH of that + chip_gap: float = 8 # between stacked chips + chip_radius: float = 8 + chip_alpha: int = 235 + chip_low: float = 48 # the least the lowest chip ends above the bottom edge + label_text: float = 15 # a scroll's count and a button's name beside its mark + label_pad: float = 4 + label_from: float = 16 # a button's name, right and down from its click + label_alpha: float = 0.9 + least_text: int = 9 # the smallest text, in pixels, on any frame + + +STYLE = Style() +#: A chip's padding above and below its text, as a share of the padding beside it. +CHIP_PAD_HIGH = 0.7 +WHITE = (255, 255, 255) + + +def _width(width: float, unit: float) -> int: + return max(1, round(width * unit)) + + +def _draw_click(pen, mark, points, colour, unit, age, fade) -> None: x, y = points[0] - grow = min(1.0, age / 0.35) + grow = min(1.0, age / STYLE.ring_grow_s) alpha = round(255 * fade) - for ring in range(max(1, min(mark.count, 3))): - r = unit * (12 + 18 * grow + 8 * ring) + for ring in range(max(1, min(mark.count, STYLE.most_rings))): + r = unit * (STYLE.ring_from + STYLE.ring_grows * grow + STYLE.ring_apart * ring) box = (x - r, y - r, x + r, y + r) shape = pen.rectangle if mark.button == "right" else pen.ellipse - shape(box, outline=(255, 255, 255, alpha), width=max(1, round(5 * unit))) - shape(box, outline=(*colour, alpha), width=max(1, round(3 * unit))) - dot = unit * 5 - pen.ellipse((x - dot, y - dot, x + dot, y + dot), fill=(*colour, alpha), outline=(255, 255, 255, alpha), - width=max(1, round(1.5 * unit))) + shape(box, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + shape(box, outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) + dot = unit * STYLE.dot + pen.ellipse((x - dot, y - dot, x + dot, y + dot), fill=(*colour, alpha), outline=(*WHITE, alpha), + width=_width(STYLE.outline, unit)) if mark.button not in ("left", "right"): - _label(pen, (x + unit * 16, y + unit * 16), mark.button, colour, unit, alpha) + _label(pen, (x + unit * STYLE.label_from, y + unit * STYLE.label_from), mark.button, colour, unit, alpha) -def _draw_press(pen, mark, points, colour, unit, age, span, fade) -> None: +def _draw_press(pen, mark, points, colour, unit, age, fade) -> None: x, y = points[0] alpha = round(255 * fade) - r = unit * (10 if mark.kind == "down" else 16) - fill = (*colour, round(alpha * 0.7)) if mark.kind == "down" else None - pen.ellipse((x - r, y - r, x + r, y + r), fill=fill, outline=(255, 255, 255, alpha), width=max(1, round(4 * unit))) - pen.ellipse((x - r, y - r, x + r, y + r), outline=(*colour, alpha), width=max(1, round(2.5 * unit))) + r = unit * (STYLE.press_down if mark.kind == "down" else STYLE.press_up) + fill = (*colour, round(alpha * STYLE.press_fill)) if mark.kind == "down" else None + pen.ellipse((x - r, y - r, x + r, y + r), fill=fill, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + pen.ellipse((x - r, y - r, x + r, y + r), outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) -def _draw_move(pen, mark, points, colour, unit, age, span, fade) -> None: +def _draw_move(pen, mark, points, colour, unit, age, fade) -> None: x, y = points[0] - alpha = round(200 * fade) - arm = unit * 9 + alpha = round(STYLE.cross_alpha * fade) + arm = unit * STYLE.cross for dx, dy in ((arm, 0), (0, arm)): - pen.line((x - dx, y - dy, x + dx, y + dy), fill=(255, 255, 255, alpha), width=max(1, round(4 * unit))) - pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*colour, alpha), width=max(1, round(2 * unit))) + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*WHITE, alpha), width=_width(STYLE.halo, unit)) + pen.line((x - dx, y - dy, x + dx, y + dy), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) -def _draw_scroll(pen, mark, points, colour, unit, age, span, fade) -> None: +def _draw_scroll(pen, mark, points, colour, unit, age, fade) -> None: x, y = points[0] alpha = round(255 * fade) dx, dy = {"up": (0, -1), "down": (0, 1), "left": (-1, 0), "right": (1, 0)}.get(mark.direction, (0, 1)) - length = unit * 46 - end = (x + dx * length, y + dy * length) - for width, fill in ((7, (255, 255, 255, alpha)), (4, (*colour, alpha))): - pen.line((x, y, *end), fill=fill, width=max(1, round(width * unit))) + end = (x + dx * unit * STYLE.arrow, y + dy * unit * STYLE.arrow) + for width, fill in ((STYLE.line_halo, (*WHITE, alpha)), (STYLE.line, (*colour, alpha))): + pen.line((x, y, *end), fill=fill, width=_width(width, unit)) _arrowhead(pen, (x, y), end, unit, fill, width) - _label(pen, (end[0] + unit * 10, end[1] + unit * 4), f"x{mark.count}", colour, unit, alpha) + _label(pen, (end[0] + unit * STYLE.label_pad * 2, end[1] + unit * STYLE.label_pad), f"x{mark.count}", colour, + unit, alpha) -def _draw_drag(pen, mark, points, colour, unit, age, span, fade) -> None: +def _draw_drag(pen, mark, points, colour, unit, age, fade) -> None: alpha = round(255 * fade) shown = points if age >= mark.moving_s else _along(points, age / mark.moving_s) - for width, fill in ((7, (255, 255, 255, alpha)), (4, (*colour, alpha))): + for width, fill in ((STYLE.line_halo, (*WHITE, alpha)), (STYLE.line, (*colour, alpha))): if len(shown) > 1: - pen.line([coord for point in shown for coord in point], fill=fill, width=max(1, round(width * unit)), + pen.line([coord for point in shown for coord in point], fill=fill, width=_width(width, unit), joint="curve") _arrowhead(pen, shown[-2], shown[-1], unit, fill, width) sx, sy = points[0] - r = unit * 6 - pen.ellipse((sx - r, sy - r, sx + r, sy + r), fill=(*colour, alpha), outline=(255, 255, 255, alpha), - width=max(1, round(1.5 * unit))) + r = unit * STYLE.start_dot + pen.ellipse((sx - r, sy - r, sx + r, sy + r), fill=(*colour, alpha), outline=(*WHITE, alpha), + width=_width(STYLE.outline, unit)) _DRAWERS = {"click": _draw_click, "down": _draw_press, "up": _draw_press, "move": _draw_move, "scroll": _draw_scroll, @@ -286,35 +337,34 @@ def _draw_drag(pen, mark, points, colour, unit, age, span, fade) -> None: def _draw_chips(pen, chips, size, unit) -> None: """Chips stacked centred above the bottom edge, clear of a panel or dock there, the newest lowest.""" - font = _font(max(10, round(20 * unit))) - pad, gap = unit * 10, unit * 8 - y = size[1] - max(unit * 48, size[1] * CHIPS_ABOVE) + font = _font(max(STYLE.least_text, round(STYLE.chip_text * unit))) + pad, high, gap = unit * STYLE.chip_pad, unit * STYLE.chip_pad * CHIP_PAD_HIGH, unit * STYLE.chip_gap + y = size[1] - max(unit * STYLE.chip_low, size[1] * CHIPS_ABOVE) for mark, colour in reversed(chips): left, top, right, bottom = pen.textbbox((0, 0), mark.label, font=font) - w, h = right - left + 2 * pad, bottom - top + 2 * pad * 0.7 - x0 = (size[0] - w) / 2 - y0 = y - h - pen.rounded_rectangle((x0, y0, x0 + w, y), radius=unit * 8, fill=(*colour, 235), - outline=(255, 255, 255, 235), width=max(1, round(1.5 * unit))) - pen.text((x0 + pad - left, y0 + pad * 0.7 - top), mark.label, font=font, fill=_ink(colour)) + w, h = right - left + 2 * pad, bottom - top + 2 * high + x0, y0 = (size[0] - w) / 2, y - h + pen.rounded_rectangle((x0, y0, x0 + w, y), radius=unit * STYLE.chip_radius, fill=(*colour, STYLE.chip_alpha), + outline=(*WHITE, STYLE.chip_alpha), width=_width(STYLE.outline, unit)) + pen.text((x0 + pad - left, y0 + high - top), mark.label, font=font, fill=_ink(colour)) y = y0 - gap def _label(pen, at: tuple[float, float], text: str, colour, unit, alpha) -> None: - font = _font(max(9, round(15 * unit))) + font = _font(max(STYLE.least_text, round(STYLE.label_text * unit))) left, top, right, bottom = pen.textbbox(at, text, font=font) - pad = unit * 4 - pen.rounded_rectangle((left - pad, top - pad, right + pad, bottom + pad), radius=unit * 4, - fill=(*colour, round(alpha * 0.9))) + pad = unit * STYLE.label_pad + pen.rounded_rectangle((left - pad, top - pad, right + pad, bottom + pad), radius=pad, + fill=(*colour, round(alpha * STYLE.label_alpha))) pen.text(at, text, font=font, fill=(*_ink(colour)[:3], alpha)) def _arrowhead(pen, start, end, unit, fill, width) -> None: angle = math.atan2(end[1] - start[1], end[0] - start[0]) - size = unit * (14 + width) - left = (end[0] - size * math.cos(angle - 0.5), end[1] - size * math.sin(angle - 0.5)) - right = (end[0] - size * math.cos(angle + 0.5), end[1] - size * math.sin(angle + 0.5)) - pen.polygon([end, left, right], fill=fill) + size = unit * (STYLE.arrowhead + width) + sides = [(end[0] - size * math.cos(angle + turn), end[1] - size * math.sin(angle + turn)) + for turn in (-STYLE.arrowhead_angle, STYLE.arrowhead_angle)] + pen.polygon([end, *sides], fill=fill) def _along(points: list[tuple[float, float]], fraction: float) -> list[tuple[float, float]]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 5e7147b82..ef144594a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -1190,10 +1190,12 @@ async def _read_back(self, segment: ManifestSegment) -> bytes | str: async def _annotated_click(self, wire: dict, size: tuple[int, int], point: tuple[int, int], inside: tuple[int, int], was: tuple[int, int, int]) -> str | None: """What is wrong with an annotated recording of one click at frame pixel ``point`` of a frame ``size``, the - clicked cell showing ``was`` at ``inside`` before it, or None. All its times are the computer's own.""" + clicked cell showing ``was`` at ``inside`` before it, or None; NotApplicable for a recording of a native + recorder's finished segments, which the base class keeps as they came. All its times are the computer's own.""" manifest = RecordingManifest.model_validate(wire) if manifest.annotate is None: - return "its manifest names no annotate: a recorder that hands over finished segments isn't drawn on" + raise NotApplicable("its manifest names no annotate: the driver's recorder hands over finished segments, " + "which aren't drawn on") if manifest.annotate != {}: return f"its manifest's annotate is {manifest.annotate!r}, not the {{}} it was started with" diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index 976ad4b57..9ffc0b3cd 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -34,6 +34,7 @@ input_marks, tool_mark, ) +from agentenv_protocol.computer.conformance import checks # noqa: E402 from agentenv_protocol.tool_context import ROLE_META_KEY # noqa: E402 START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" @@ -292,3 +293,22 @@ async def test_a_recording_of_a_native_recorders_finished_segments_names_no_anno manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() assert manifest["segments"] and "annotate" not in manifest + + +def test_chips_past_the_most_a_frame_shows_wait_for_a_later_frame_rather_than_go_unseen(): + annotations = Annotations() + for n in range(6): + annotations.add([tool_mark(f"tool{n}", {}, role=None, at=T0 + timedelta(milliseconds=n))]) + + first = [mark.label for mark, _colour, _age in annotations.showing(T0 + timedelta(milliseconds=100))] + later = [mark.label for mark, _colour, _age in annotations.showing(T0 + timedelta(seconds=3))] + + assert first == ["tool2", "tool3", "tool4", "tool5"] and later == ["tool0", "tool1"] + assert annotations.showing(T0 + timedelta(seconds=4)) == [] and annotations.marks == [] + + +@pytest.mark.asyncio +async def test_the_probe_finds_annotation_not_applicable_to_a_native_recorders_finished_segments(): + wire = {"recording_id": "rec-1", "ended_by": "end", "segments": []} + with pytest.raises(checks.NotApplicable, match="hands over finished segments"): + await checks.Probe._annotated_click(None, wire, (100, 100), (10, 10), (20, 20), (255, 255, 255)) From 288f310598b6776c12caac7e213166df17efb9c2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 13:00:06 -0800 Subject: [PATCH 04/10] fix(computers): a tool named after its environment is told read-only by its own annotations, and the probe's settings are named Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 15 ++++++-- .../src/agentenv_protocol/computer/driver.py | 34 +++++++++++++------ .../tests/test_computer_recording_annotate.py | 30 +++++++++++++++- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index ef144594a..fe4253216 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -39,6 +39,7 @@ POINT_TOOLS, ) from ..annotate import COLOURS as MARK_COLOURS +from ..annotate import UNIT_EDGE from ..manifest import ManifestSegment, RecordingManifest from ..recording import RECORDINGS_DIR from .client import Driver, image @@ -71,6 +72,14 @@ ANNOTATE_LEAD_S, ANNOTATE_TAIL_S = 1.0, 1.0 #: How far, as the sum of its channels' differences, an encoded pixel may be from the colour drawn and still count. NEAR_COLOUR = 90 +#: How many pixels near the mark's colour around the click make a frame draw its mark. +MARK_PIXELS = 20 +#: How far around the click, in pixels of a frame whose longer edge is the marks' UNIT_EDGE, the mark is looked for; +#: and the least, on a smaller one. +MARK_REACH, MARK_REACH_LEAST = 40, 8 +#: Where in the clicked cell its colour is read, right and down from its centre as a share of the cell: clear of the +#: pointer drawn at the click and of the mark's dot. +CELL_INSIDE = 0.36 #: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. BUSY_DESK_ATTEMPTS = 2 @@ -1112,7 +1121,7 @@ async def annotated() -> str | None: frame, scale, img = await self._base() size = self.layout["cell"] x, y = self._at("grid", scale, size + size / 2, 6 * size + size / 2) - inside = (x + math.floor(size * 0.36 * scale), y + math.floor(size * 0.36 * scale)) + inside = (x + math.floor(size * CELL_INSIDE * scale), y + math.floor(size * CELL_INSIDE * scale)) was = img.getpixel(inside) await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) await asyncio.sleep(ANNOTATE_TAIL_S) @@ -1204,11 +1213,11 @@ def at(xy: tuple[int, int], shape: tuple[int, int]) -> tuple[int, int]: def marked(picture: Any, shape: tuple[int, int]) -> bool: cx, cy = at(point, shape) - reach = max(8, round(40 * max(shape) / 1280)) + reach = max(MARK_REACH_LEAST, round(MARK_REACH * max(shape) / UNIT_EDGE)) near = sum(1 for px in range(max(0, cx - reach), min(shape[0], cx + reach)) for py in range(max(0, cy - reach), min(shape[1], cy + reach)) if _distance(picture.getpixel((px, py)), MARK_COLOURS[0]) < NEAR_COLOUR) - return near > 20 + return near > MARK_PIXELS def changed(picture: Any, shape: tuple[int, int]) -> bool: return _distance(picture.getpixel(at(inside, shape)), was) >= NEAR_COLOUR diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 92c23c796..1ec3e7c93 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -6,6 +6,7 @@ import asyncio import logging import os +import re from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional @@ -15,9 +16,9 @@ from starlette.responses import Response from starlette.websockets import WebSocket -from ..agent_env_environment import (_CARD_CONFIG_ATTR, _TOOL_ATTR, _TOOL_OPTIONS_ATTR, AgentEnvEnvironment, - AgentEnvFastMCPApplication, AgentEnvError, _error_response, _marked_members, - add_websocket_route, control_check, environment_card) +from ..agent_env_environment import (_CARD_CONFIG_ATTR, _ENV_NAME_TOKEN, _TOOL_ATTR, _TOOL_OPTIONS_ATTR, + AgentEnvEnvironment, AgentEnvFastMCPApplication, AgentEnvError, _error_response, + _marked_members, add_websocket_route, control_check, environment_card) from ..tool_context import ToolContext from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool from . import _linux, annotate, apps, card, svc @@ -144,16 +145,26 @@ def on_tool_call(self, context: ToolContext) -> Any: return None def _computer_tool_acts(self, name: str) -> bool: - """Whether the driver's own or forwarded tool ``name`` may act: its annotations don't say it only reads.""" - acting = self.__dict__.get("_computer_acting") - if acting is None: + """Whether the driver's own or forwarded tool ``name`` may act: its annotations don't say it only reads. A + tool named with ``{environment_name}`` is served under that name filled in, so it's matched by the rest.""" + known = self.__dict__.get("_computer_acting") + if known is None: acting = {forwarded.tool.name: not _reads_only(forwarded.annotations) for forwarded in self.forwarded_tools()} + templated = [] for member, marker in _marked_members(self, _TOOL_ATTR): options = getattr(member, _TOOL_OPTIONS_ATTR, None) - acting[marker.name] = not _reads_only(options.annotations if options is not None else None) - self._computer_acting = acting - return acting.get(name, True) + acts = not _reads_only(options.annotations if options is not None else None) + if _ENV_NAME_TOKEN in marker.name: + parts = map(re.escape, marker.name.split(_ENV_NAME_TOKEN)) + templated.append((re.compile(".+".join(parts)), acts)) + else: + acting[marker.name] = acts + known = self._computer_acting = (acting, templated) + acting, templated = known + if name in acting: + return acting[name] + return next((acts for pattern, acts in templated if pattern.fullmatch(name)), True) def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" @@ -286,5 +297,6 @@ def deco(cls: type) -> type: def _reads_only(annotations: Any) -> bool: """Whether a tool's MCP annotations, as a dict or a ``ToolAnnotations``, say it only reads.""" - hint = annotations.get("readOnlyHint") if isinstance(annotations, dict) else getattr(annotations, "readOnlyHint", None) - return hint is True + if isinstance(annotations, dict): + return annotations.get("readOnlyHint") is True + return getattr(annotations, "readOnlyHint", None) is True diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index 9ffc0b3cd..fa0ec2cbf 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -17,7 +17,7 @@ from mcp.types import CallToolResult, TextContent # noqa: E402 from PIL import Image as PILImage # noqa: E402 -from agentenv_protocol import EnvironmentTool, ForwardedTool # noqa: E402 +from agentenv_protocol import EnvironmentTool, ForwardedTool, tool # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 COMPUTER_RECORDING, DeviceUnavailable, @@ -75,6 +75,18 @@ async def segments(self, hints): await asyncio.sleep(3600) +class Templated(Recorded): + """A driver with tools of its own named after its environment: one that only reads, one that acts.""" + + @tool(name="{environment_name}_inspect", annotations={"readOnlyHint": True}) + async def inspect(self) -> str: + return "read" + + @tool(name="{environment_name}_poke") + async def poke(self) -> str: + return "poked" + + class Forwarding(Recorded): """A driver serving two tools of another server's: one that acts, one its annotations say only reads.""" @@ -312,3 +324,19 @@ async def test_the_probe_finds_annotation_not_applicable_to_a_native_recorders_f wire = {"recording_id": "rec-1", "ended_by": "end", "segments": []} with pytest.raises(checks.NotApplicable, match="hands over finished segments"): await checks.Probe._annotated_click(None, wire, (100, 100), (10, 10), (20, 20), (255, 255, 255)) + + +@pytest.mark.asyncio +async def test_a_tool_named_after_its_environment_is_told_read_only_by_its_own_annotations(): + driver = Templated() + async with http_client(driver) as http, mcp_session(driver) as session: + names = [t.name for t in (await session.list_tools()).tools if t.name.endswith(("_inspect", "_poke"))] + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recording = driver._computer.recordings.running[started["recording_id"]] + for name in names: + await session.call_tool(name, {}) + labels = [mark.label for mark in recording.annotations.marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + [poke] = [name for name in names if name.endswith("_poke")] + assert len(names) == 2 and labels == [poke] From 04992c8789bc59cf331ff57e2c4747cb2b74fdd3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 14:54:08 -0800 Subject: [PATCH 05/10] feat(computers): an annotated recording draws the agent's pointer where its frames can't show it Frames polled from captures that leave the pointer out, as cua-driver's, now show it where the agent's last input left it, along a drag's path as it moves, after a fading line from where it jumped; a frame that draws the pointer itself gets no second one. A click's dot is hollow, so the pointer's tip shows through, and the probe reads the clicked cell's colour up and left of the click, clear of a pointer there. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 10 +++- .../agentenv_protocol/computer/annotate.py | 60 ++++++++++++++++--- .../computer/conformance/checks.py | 6 +- .../agentenv_protocol/computer/recording.py | 2 +- .../tests/test_computer_recording_annotate.py | 56 ++++++++++++++++- 5 files changed, 116 insertions(+), 18 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 6854a7011..6120ebd03 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -770,11 +770,15 @@ class MyDriver(ComputerDriverEnvironment): segments, at `/manifest.json`, so a reader that didn't end it finds it there. - **Annotations.** `start` with `annotate`, an object of options (`{}` for none; the card lists those a driver takes under its params' `annotate.options`), draws each input over the frames captured while it - shows, in the colour of the role that sent it (the agent's first): a ring where a click landed (a - square for the right button, a ring for each click of a double or triple), a trail for a drag, an + shows, in the colour of the role that sent it (the agent's first): a ring where a click landed, round a + hollow dot that leaves the pointer's tip in sight (a square for the right button, a ring for each click + of a double or triple), a trail for a drag, an arrow for a scroll, a cross for a move, and a chip for a key chord, typed text as it was typed, and each of the driver's own or forwarded tools whose annotations don't say it only reads, by name and - main argument. Every `computer_*` input is marked as `act` delivers it, best effort, and shows on + main argument. Where the frames can't show the pointer themselves, as frames polled from captures that + leave it out do (cua-driver's), the agent's pointer is drawn where its last input left it, along a + drag's path as it moves, after a fading line from where it jumped. Every `computer_*` input is marked + as `act` delivers it, best effort, and shows on every frame while it lasts and on the first one after it however late, as at a low fps. A segment a native recorder finished is kept as it came. The manifest's `annotate` is the object the recording drew with, and is left out for one that took a native recorder's segments. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py index 3d20da942..db690ff93 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -1,7 +1,9 @@ """What an annotated recording (computer-recording/v1 ``start`` with ``annotate``) draws over its frames: where each input landed and what it typed or pressed, in the colour of the role that sent it, on the frames captured while its mark shows. ``ComputerRuntime.act`` marks every ``computer_*`` input as it is delivered; a call to one of the -driver's own or forwarded tools, which acts without it, shows as a chip naming the tool and its main argument.""" +driver's own or forwarded tools, which acts without it, shows as a chip naming the tool and its main argument. Where +the frames can't show the pointer themselves, as frames from screenshots that leave it out, the agent's pointer is +drawn where its inputs left it.""" from __future__ import annotations import functools @@ -29,6 +31,10 @@ FADE_S = 0.3 #: How long a mark no frame has drawn is kept, in seconds: past it, the recording's frames have stopped coming. KEEP_S = 30.0 +#: The marks of inputs that leave the pointer at their last point, a drag's after moving it along its path. +MOVES_POINTER = ("click", "drag", "scroll", "move", "down", "up") +#: How long after the pointer jumps a line shows where it came from, in seconds. +JUMP_S = 0.4 #: The inputs whose marks are drawn at a point of the display. POINTED = ("click", "drag", "scroll", "pointer_move", "pointer_down", "pointer_up") #: The most a chip of typed text shows, which grows with its length from the chip's own time. @@ -80,12 +86,15 @@ def check_annotate(annotate: object) -> Optional[dict]: class Annotations: """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role. Each mark shows on every frame captured while it lasts, and on the first one after its input however late that comes, - as at a low fps or from a slow capture.""" + as at a low fps or from a slow capture. With ``pointer``, as for frames that can't show the pointer themselves, + each frame also draws the pointer where the last input before it left it, from the recording's first input on.""" - def __init__(self) -> None: + def __init__(self, pointer: bool = False) -> None: self.marks: list[Mark] = [] self.drawn: set[int] = set() self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} + self.pointer = pointer + self.moves: list[Mark] = [] # the pointer's, each a "pointer" mark: where it was, then its path def add(self, marks: Sequence[Mark]) -> None: for mark in marks: @@ -93,6 +102,19 @@ def add(self, marks: Sequence[Mark]) -> None: stale = mark.at - timedelta(seconds=KEEP_S) self._keep([kept for kept in self.marks if kept.until > stale]) self.marks.append(mark) + if self.pointer and mark.kind in MOVES_POINTER and mark.points: + path = mark.points if mark.kind == "drag" else mark.points[-1:] + was = self.moves[-1].points[-1] if self.moves else path[0] + self.moves.append(Mark("pointer", mark.at, mark.at, mark.role, (was, *path), moving_s=mark.moving_s)) + + def _pointer(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: + """The pointer a frame captured ``at`` shows: the last move before it, as its age; none before the first.""" + current = next((i for i in range(len(self.moves) - 1, -1, -1) if self.moves[i].at <= at), None) + if current is None: + return [] + self.moves = self.moves[current:] # the frames come in order, so an earlier move won't show again + move = self.moves[0] + return [(move, self.colours[move.role], (at - move.at).total_seconds())] def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a @@ -113,7 +135,7 @@ def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float] shown = [entry for entry in due if id(entry[0]) not in waiting] self.drawn |= {id(mark) for mark, _colour, _age in shown} self._keep([mark for mark in self.marks if mark.until > at or id(mark) not in self.drawn]) - return shown + return shown + (self._pointer(at) if self.pointer else []) def _keep(self, marks: list[Mark]) -> None: self.marks = marks @@ -214,7 +236,7 @@ def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int], float]]) if mark.kind == "chip": chips.append((mark, colour)) continue - fade = 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) + fade = 1.0 if mark.kind == "pointer" else 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) points = [(fx * base.width, fy * base.height) for fx, fy in mark.points] _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, fade) if chips: @@ -237,7 +259,11 @@ class Style: ring_grows: float = 18 # and how much it grows ring_apart: float = 8 # between the rings of a double or triple click most_rings: int = 3 - dot: float = 5 # the dot at a click's point + dot: float = 5 # the hollow dot at a click's point, which leaves the pointer's tip there in sight + dot_stroke: float = 2 + pointer: float = 1.6 # the agent's pointer, as a multiple of the usual arrow cursor's size + pointer_outline: float = 2 + jump_alpha: int = 170 # the line from where the pointer jumped, at its start press_down: float = 10 # a press's filled ring's radius press_up: float = 16 # a release's ring's radius press_fill: float = 0.7 # a press's fill opacity, as a share of its stroke's @@ -281,8 +307,7 @@ def _draw_click(pen, mark, points, colour, unit, age, fade) -> None: shape(box, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) shape(box, outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) dot = unit * STYLE.dot - pen.ellipse((x - dot, y - dot, x + dot, y + dot), fill=(*colour, alpha), outline=(*WHITE, alpha), - width=_width(STYLE.outline, unit)) + pen.ellipse((x - dot, y - dot, x + dot, y + dot), outline=(*colour, alpha), width=_width(STYLE.dot_stroke, unit)) if mark.button not in ("left", "right"): _label(pen, (x + unit * STYLE.label_from, y + unit * STYLE.label_from), mark.button, colour, unit, alpha) @@ -331,8 +356,25 @@ def _draw_drag(pen, mark, points, colour, unit, age, fade) -> None: width=_width(STYLE.outline, unit)) +#: The usual arrow cursor, its tip at (0, 0), in pixels of a UNIT_EDGE frame. +ARROW = ((0, 0), (0, 17), (4, 13), (7, 20), (10, 19), (7, 12), (12, 12)) + + +def _draw_pointer(pen, mark, points, colour, unit, age, fade) -> None: + """The agent's pointer: along a drag's path while it moves, else where its input left it, after a fading line + from where it jumped.""" + was, path = points[0], points[1:] + tip = _along(path, age / mark.moving_s)[-1] if len(path) > 1 and age < mark.moving_s else path[-1] + if age < JUMP_S and math.dist(was, path[0]) > unit * STYLE.cross: + alpha = round(STYLE.jump_alpha * (1 - age / JUMP_S)) + pen.line((*was, *path[0]), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) + scale = unit * STYLE.pointer + arrow = [(tip[0] + dx * scale, tip[1] + dy * scale) for dx, dy in ARROW] + pen.polygon(arrow, fill=(*colour, 255), outline=(*WHITE, 255), width=_width(STYLE.pointer_outline, unit)) + + _DRAWERS = {"click": _draw_click, "down": _draw_press, "up": _draw_press, "move": _draw_move, "scroll": _draw_scroll, - "drag": _draw_drag} + "drag": _draw_drag, "pointer": _draw_pointer} def _draw_chips(pen, chips, size, unit) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index fe4253216..27da96e12 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -77,8 +77,8 @@ #: How far around the click, in pixels of a frame whose longer edge is the marks' UNIT_EDGE, the mark is looked for; #: and the least, on a smaller one. MARK_REACH, MARK_REACH_LEAST = 40, 8 -#: Where in the clicked cell its colour is read, right and down from its centre as a share of the cell: clear of the -#: pointer drawn at the click and of the mark's dot. +#: Where in the clicked cell its colour is read, left and up from its centre as a share of the cell: clear of the +#: mark's dot and of a pointer at the click, real or drawn, which points from there down and right. CELL_INSIDE = 0.36 #: How many times a check whose effect a busy desk can miss is tried before it fails: a desk starved through a drag #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. @@ -1121,7 +1121,7 @@ async def annotated() -> str | None: frame, scale, img = await self._base() size = self.layout["cell"] x, y = self._at("grid", scale, size + size / 2, 6 * size + size / 2) - inside = (x + math.floor(size * CELL_INSIDE * scale), y + math.floor(size * CELL_INSIDE * scale)) + inside = (x - math.floor(size * CELL_INSIDE * scale), y - math.floor(size * CELL_INSIDE * scale)) was = img.getpixel(inside) await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) await asyncio.sleep(ANNOTATE_TAIL_S) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 4baa6a341..2ff0ca085 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -262,7 +262,7 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.streams = streams self.cursor = cursor self.annotate = annotate - self.annotations = annotating.Annotations() if annotate is not None else None + self.annotations = annotating.Annotations(pointer=hints.cursor and not cursor) if annotate is not None else None self.native = False self.segments: list[_Segment] = [] self.encoder: Optional[_Encoder] = None diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index fa0ec2cbf..92d8ed54c 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -13,7 +13,7 @@ pytest.importorskip("PIL") av = pytest.importorskip("av") -from computer_fake import FakeDriver, http_client, mcp_session # noqa: E402 +from computer_fake import FakeDriver, MinimalDriver, http_client, mcp_session # noqa: E402 from mcp.types import CallToolResult, TextContent # noqa: E402 from PIL import Image as PILImage # noqa: E402 @@ -29,6 +29,8 @@ COLOURS, FADE_S, KEEP_S, + STYLE, + UNIT_EDGE, Annotations, draw, input_marks, @@ -52,6 +54,11 @@ class Recorded(FakeDriver): recording = Recording(max_segment_s=60) +class Pointerless(MinimalDriver): + """A driver with no pointer group, so its recordings, polled from captures, can't draw the pointer themselves.""" + recording = Recording(max_segment_s=60) + + class Segmented(FakeDriver): """A native recorder that hands over a finished segment, which the base class keeps as it came.""" recording = Recording(max_segment_s=60) @@ -255,7 +262,9 @@ def test_a_mark_lands_at_its_point_on_a_frame_of_any_size(): for size in ((1000, 500), (400, 200), (2000, 1000)): image = draw(PILImage.new("RGB", size, (255, 255, 255)), annotations.showing(at)) cx, cy = size[0] // 2, size[1] // 2 - assert any(image.getpixel((cx + dx, cy + dy)) == COLOURS[0] for dx in range(-3, 4) for dy in range(-3, 4)), size + reach = round(STYLE.dot * max(size) / UNIT_EDGE) + 3 # the click's hollow dot, around its point + assert any(image.getpixel((cx + dx, cy + dy)) == COLOURS[0] + for dx in range(-reach, reach + 1) for dy in range(-reach, reach + 1)), size assert image.getpixel((5, 5)) == (255, 255, 255) @@ -340,3 +349,46 @@ async def test_a_tool_named_after_its_environment_is_told_read_only_by_its_own_a [poke] = [name for name in names if name.endswith("_poke")] assert len(names) == 2 and labels == [poke] + + +def _pointed(image: PILImage.Image, tip: tuple[int, int], colour: tuple[int, int, int]) -> int: + """How many pixels near ``colour`` the agent's pointer drawn at ``tip`` would cover: its arrow, down and right.""" + x, y = tip + return sum(1 for px in range(x, min(image.width, x + 14)) for py in range(y + 2, min(image.height, y + 22)) + if sum(abs(a - b) for a, b in zip(image.getpixel((px, py)), colour)) < 90) + + +@pytest.mark.parametrize("driver, drawn", [(Pointerless, True), (Recorded, False)], + ids=["frames-without-a-pointer", "frames-with-the-pointer"]) +@pytest.mark.asyncio +async def test_the_agents_pointer_is_drawn_where_its_click_left_it_only_where_the_frames_cant_show_the_pointer( + driver, drawn): + """Once the click's own mark is over, a frame still shows the pointer there: drawn in the agent's colour where + the recording's frames leave the pointer out, and left to the frames' own where they draw it.""" + driver = driver() + async with http_client(driver) as http, mcp_session(driver) as session: + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + await asyncio.sleep(0.3) + shot = await session.call_tool("computer_screenshot", {}) + sent = datetime.now(timezone.utc) + await session.call_tool("computer_click", {"x": 200, "y": 150, "frame_id": shot.structuredContent["frame_id"]}) + await asyncio.sleep(1.6) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + + after = [image for at, image in _timed_frames(manifest) + if sent + timedelta(seconds=1.0) <= at <= sent + timedelta(seconds=1.5)] + assert after and all((_pointed(image, (200, 150), COLOURS[0]) > 10) is drawn for image in after) + + +def test_the_pointer_follows_the_last_input_before_each_frame_from_the_first_input_on(): + annotations = Annotations(pointer=True) + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + annotations.add(input_marks("pointer_move", (50, 50), role=None, at=T0 + timedelta(seconds=1), size=(100, 100))) + + def pointer(seconds): + return [(mark.points[0], mark.points[-1]) for mark, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if mark.kind == "pointer"] + + assert pointer(-1) == [] + assert pointer(0.5) == [((0.1, 0.1), (0.1, 0.1))] + assert pointer(1.5) == [((0.1, 0.1), (0.5, 0.5))] and pointer(9) == [((0.1, 0.1), (0.5, 0.5))] From 0276b3748848f30bdf3dc28b5a57dde5a23d0cf7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:11:38 -0800 Subject: [PATCH 06/10] fix(computers): a driver tool named after its environment is looked up by its exact served name The hook fills in {environment_name} from the mounted card, as the SDK does when it mounts the tools, so one tool's name can't match another's. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/driver.py | 25 +++++++------------ .../tests/test_computer_recording_annotate.py | 9 ++++--- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 1ec3e7c93..4c9437787 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -6,7 +6,6 @@ import asyncio import logging import os -import re from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional @@ -146,25 +145,19 @@ def on_tool_call(self, context: ToolContext) -> Any: def _computer_tool_acts(self, name: str) -> bool: """Whether the driver's own or forwarded tool ``name`` may act: its annotations don't say it only reads. A - tool named with ``{environment_name}`` is served under that name filled in, so it's matched by the rest.""" - known = self.__dict__.get("_computer_acting") - if known is None: + tool named with ``{environment_name}`` is served under that name filled in from the card, as the SDK fills + it in when it mounts the tools.""" + acting = self.__dict__.get("_computer_acting") + if acting is None: acting = {forwarded.tool.name: not _reads_only(forwarded.annotations) for forwarded in self.forwarded_tools()} - templated = [] + environment_name = self.mounted_card().name for member, marker in _marked_members(self, _TOOL_ATTR): options = getattr(member, _TOOL_OPTIONS_ATTR, None) - acts = not _reads_only(options.annotations if options is not None else None) - if _ENV_NAME_TOKEN in marker.name: - parts = map(re.escape, marker.name.split(_ENV_NAME_TOKEN)) - templated.append((re.compile(".+".join(parts)), acts)) - else: - acting[marker.name] = acts - known = self._computer_acting = (acting, templated) - acting, templated = known - if name in acting: - return acting[name] - return next((acts for pattern, acts in templated if pattern.fullmatch(name)), True) + acting[marker.name.replace(_ENV_NAME_TOKEN, environment_name)] = not _reads_only( + options.annotations if options is not None else None) + self._computer_acting = acting + return acting.get(name, True) def has(self, capability: str) -> bool: """Whether an operation, or a whole group (``pointer``, ``keyboard``), works on this platform.""" diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index 92d8ed54c..cc6f9de5c 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -83,13 +83,14 @@ async def segments(self, hints): class Templated(Recorded): - """A driver with tools of its own named after its environment: one that only reads, one that acts.""" + """A driver with tools of its own named after its environment: one that only reads, and one that acts whose name + ends as the first's does.""" @tool(name="{environment_name}_inspect", annotations={"readOnlyHint": True}) async def inspect(self) -> str: return "read" - @tool(name="{environment_name}_poke") + @tool(name="{environment_name}_foo_inspect") async def poke(self) -> str: return "poked" @@ -339,7 +340,7 @@ async def test_the_probe_finds_annotation_not_applicable_to_a_native_recorders_f async def test_a_tool_named_after_its_environment_is_told_read_only_by_its_own_annotations(): driver = Templated() async with http_client(driver) as http, mcp_session(driver) as session: - names = [t.name for t in (await session.list_tools()).tools if t.name.endswith(("_inspect", "_poke"))] + names = [t.name for t in (await session.list_tools()).tools if t.name.endswith("_inspect")] started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() recording = driver._computer.recordings.running[started["recording_id"]] for name in names: @@ -347,7 +348,7 @@ async def test_a_tool_named_after_its_environment_is_told_read_only_by_its_own_a labels = [mark.label for mark in recording.annotations.marks] await http.post(END, json={"recording_id": started["recording_id"]}) - [poke] = [name for name in names if name.endswith("_poke")] + [poke] = [name for name in names if name.endswith("_foo_inspect")] assert len(names) == 2 and labels == [poke] From bd63a90556e0f3621cec6f17ad3e261c1060c98f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:20:14 -0800 Subject: [PATCH 07/10] feat(computers): an annotated recording draws a pointer for each role and pointer Where the frames can't show the pointer, each role, and each pointer one acts through, gets a pointer where its last input left it: the one that moved last at full strength, the others fainter, each named once there are several. Inputs carry the pointer they act through, the computer's own until agents can drive more. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 5 +- .../agentenv_protocol/computer/annotate.py | 62 +++++++++++++------ .../tests/test_computer_recording_annotate.py | 22 +++++++ 3 files changed, 69 insertions(+), 20 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 6120ebd03..b6e8eab75 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -776,8 +776,9 @@ class MyDriver(ComputerDriverEnvironment): arrow for a scroll, a cross for a move, and a chip for a key chord, typed text as it was typed, and each of the driver's own or forwarded tools whose annotations don't say it only reads, by name and main argument. Where the frames can't show the pointer themselves, as frames polled from captures that - leave it out do (cua-driver's), the agent's pointer is drawn where its last input left it, along a - drag's path as it moves, after a fading line from where it jumped. Every `computer_*` input is marked + leave it out do (cua-driver's), a pointer is drawn for each role and pointer that has acted, where its + last input left it, along a drag's path as it moves, after a fading line from where it jumped: the one + that moved last at full strength, the others fainter, each named once there are several. Every `computer_*` input is marked as `act` delivers it, best effort, and shows on every frame while it lasts and on the first one after it however late, as at a low fps. A segment a native recorder finished is kept as it came. The manifest's `annotate` is the object the recording diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py index db690ff93..b8e4c8248 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -6,6 +6,7 @@ drawn where its inputs left it.""" from __future__ import annotations +import dataclasses import functools import math from collections.abc import Mapping, Sequence @@ -23,6 +24,8 @@ OPTIONS: tuple[str, ...] = () #: The role of a call that names none: the agent. DEFAULT_ROLE = "default" +#: The pointer an input acts through when it names none: the computer's own, the only one the SDK drives yet. +DEFAULT_POINTER = "default" #: Each role's colour: the agent's first, then each other role's in the order it first acts on the recording. COLOURS = ((255, 138, 0), (0, 184, 148), (175, 82, 222), (255, 59, 105), (52, 120, 246), (250, 204, 21)) #: How long a mark shows from its input, in seconds; a drag's, after its moves. @@ -56,9 +59,10 @@ @dataclass(frozen=True) class Mark: - """One input on a recording: ``kind`` is click, down, up, move, scroll, drag or chip; ``points`` are fractions of - the display, so they land on any frame size; it shows from ``at`` until ``until``, a drag moving for its first - ``moving_s``.""" + """One input on a recording: ``kind`` is click, down, up, move, scroll, drag or chip, or pointer for where a + pointer was and then went; ``points`` are fractions of the display, so they land on any frame size; it shows from + ``at`` until ``until``, a drag moving for its first ``moving_s``. ``pointer`` is the pointer it acted through; + ``faded``, a pointer drawn fainter, as one that didn't move last.""" kind: str at: datetime @@ -70,6 +74,8 @@ class Mark: direction: str = "" label: str = "" moving_s: float = 0.0 + pointer: str = DEFAULT_POINTER + faded: bool = False def check_annotate(annotate: object) -> Optional[dict]: @@ -87,14 +93,15 @@ class Annotations: """One annotated recording's marks, from their inputs until they stop showing, and the colour of each role. Each mark shows on every frame captured while it lasts, and on the first one after its input however late that comes, as at a low fps or from a slow capture. With ``pointer``, as for frames that can't show the pointer themselves, - each frame also draws the pointer where the last input before it left it, from the recording's first input on.""" + each frame also draws a pointer for each role and pointer that has acted, where its last input before the frame + left it: the one that moved last at full strength, the others fainter, each named once there are several.""" def __init__(self, pointer: bool = False) -> None: self.marks: list[Mark] = [] self.drawn: set[int] = set() self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} self.pointer = pointer - self.moves: list[Mark] = [] # the pointer's, each a "pointer" mark: where it was, then its path + self.moves: dict[tuple[str, str], list[Mark]] = {} # by role and pointer: each move, where it was and went def add(self, marks: Sequence[Mark]) -> None: for mark in marks: @@ -103,18 +110,26 @@ def add(self, marks: Sequence[Mark]) -> None: self._keep([kept for kept in self.marks if kept.until > stale]) self.marks.append(mark) if self.pointer and mark.kind in MOVES_POINTER and mark.points: + moves = self.moves.setdefault((mark.role, mark.pointer), []) path = mark.points if mark.kind == "drag" else mark.points[-1:] - was = self.moves[-1].points[-1] if self.moves else path[0] - self.moves.append(Mark("pointer", mark.at, mark.at, mark.role, (was, *path), moving_s=mark.moving_s)) - - def _pointer(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: - """The pointer a frame captured ``at`` shows: the last move before it, as its age; none before the first.""" - current = next((i for i in range(len(self.moves) - 1, -1, -1) if self.moves[i].at <= at), None) - if current is None: - return [] - self.moves = self.moves[current:] # the frames come in order, so an earlier move won't show again - move = self.moves[0] - return [(move, self.colours[move.role], (at - move.at).total_seconds())] + was = moves[-1].points[-1] if moves else path[0] + moves.append(Mark("pointer", mark.at, mark.at, mark.role, (was, *path), moving_s=mark.moving_s, + pointer=mark.pointer)) + + def _pointers(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: + """The pointers a frame captured ``at`` shows, each as its last move before the frame at its age, the one that + moved last drawn over the others; none for a role or pointer that hadn't acted by then.""" + current = [] + for key, moves in self.moves.items(): + last = next((i for i in range(len(moves) - 1, -1, -1) if moves[i].at <= at), None) + if last is not None: + self.moves[key] = moves[last:] # the frames come in order, so an earlier move won't show again + current.append(moves[last]) + current.sort(key=lambda move: move.at) + several = len(current) > 1 + shown = [dataclasses.replace(move, faded=move is not current[-1], + label=_pointer_label(move) if several else "") for move in current] + return [(move, self.colours[move.role], (at - move.at).total_seconds()) for move in shown] def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a @@ -135,7 +150,7 @@ def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float] shown = [entry for entry in due if id(entry[0]) not in waiting] self.drawn |= {id(mark) for mark, _colour, _age in shown} self._keep([mark for mark in self.marks if mark.until > at or id(mark) not in self.drawn]) - return shown + (self._pointer(at) if self.pointer else []) + return shown + (self._pointers(at) if self.pointer else []) def _keep(self, marks: list[Mark]) -> None: self.marks = marks @@ -263,6 +278,7 @@ class Style: dot_stroke: float = 2 pointer: float = 1.6 # the agent's pointer, as a multiple of the usual arrow cursor's size pointer_outline: float = 2 + pointer_faded_alpha: int = 120 # a pointer that didn't move last jump_alpha: int = 170 # the line from where the pointer jumped, at its start press_down: float = 10 # a press's filled ring's radius press_up: float = 16 # a release's ring's radius @@ -370,7 +386,11 @@ def _draw_pointer(pen, mark, points, colour, unit, age, fade) -> None: pen.line((*was, *path[0]), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) scale = unit * STYLE.pointer arrow = [(tip[0] + dx * scale, tip[1] + dy * scale) for dx, dy in ARROW] - pen.polygon(arrow, fill=(*colour, 255), outline=(*WHITE, 255), width=_width(STYLE.pointer_outline, unit)) + alpha = STYLE.pointer_faded_alpha if mark.faded else 255 + pen.polygon(arrow, fill=(*colour, alpha), outline=(*WHITE, alpha), width=_width(STYLE.pointer_outline, unit)) + if mark.label: + corner = max(dx for dx, _dy in ARROW), max(dy for _dx, dy in ARROW) + _label(pen, (tip[0] + corner[0] * scale, tip[1] + corner[1] * scale), mark.label, colour, unit, alpha) _DRAWERS = {"click": _draw_click, "down": _draw_press, "up": _draw_press, "move": _draw_move, "scroll": _draw_scroll, @@ -425,6 +445,12 @@ def _along(points: list[tuple[float, float]], fraction: float) -> list[tuple[flo return shown +def _pointer_label(move: Mark) -> str: + """A pointer's name beside it: its role's, the agent's as agent, and its own when it isn't the default.""" + role = "agent" if move.role == DEFAULT_ROLE else move.role + return role if move.pointer == DEFAULT_POINTER else f"{role} {move.pointer}" + + def _ink(colour: tuple[int, int, int]) -> tuple[int, int, int, int]: """Black or white, whichever reads on ``colour``.""" r, g, b = colour diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index cc6f9de5c..e4b8f6f13 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import dataclasses import os from datetime import datetime, timedelta, timezone @@ -393,3 +394,24 @@ def pointer(seconds): assert pointer(-1) == [] assert pointer(0.5) == [((0.1, 0.1), (0.1, 0.1))] assert pointer(1.5) == [((0.1, 0.1), (0.5, 0.5))] and pointer(9) == [((0.1, 0.1), (0.5, 0.5))] + + +def test_each_role_and_pointer_has_a_pointer_the_last_mover_on_top_each_named_when_there_are_several(): + """The agent and a user simulator on one computer, and a second pointer of the agent's, as a lease would give it.""" + annotations = Annotations(pointer=True) + annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) + + def pointers(seconds): + return [(mark.role, mark.pointer, mark.points[-1], mark.faded, mark.label) for mark, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if mark.kind == "pointer"] + + assert pointers(0.5) == [("default", "default", (0.1, 0.1), False, "")] + annotations.add(input_marks("click", (50, 50, "left", 1, []), role="user", at=T0 + timedelta(seconds=1), + size=(100, 100))) + assert pointers(1.5) == [("default", "default", (0.1, 0.1), True, "agent"), + ("user", "default", (0.5, 0.5), False, "user")] + [second] = input_marks("pointer_move", (80, 20), role=None, at=T0 + timedelta(seconds=2), size=(100, 100)) + annotations.add([dataclasses.replace(second, pointer="p2")]) + assert pointers(2.5) == [("default", "default", (0.1, 0.1), True, "agent"), + ("user", "default", (0.5, 0.5), True, "user"), + ("default", "p2", (0.8, 0.2), False, "agent p2")] From 5744a9c37f33083d2c4fc910fa896a4344563b16 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 16:11:37 -0800 Subject: [PATCH 08/10] feat(task_step): start_computer_recording annotates by default, annotation_settings turning it off A recording draws its inputs unless its step's annotation_settings says is_enabled: false. A computer whose driver can't, as one composed before agent-env drew inputs, is recorded plain with a warning rather than failing, and the run's recording entry says whether it was annotated. The scripted desktop recording, which compares its frames with the screenshots, turns it off. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../task_steps/computer_recording.py | 77 +++++++++++++------ .../computer/desktop_example_test.py | 2 + .../computer/recording/tasks/recording.json | 2 +- tst/unit/task_step/computer_recording_test.py | 44 ++++++----- 5 files changed, 82 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 11aed51fb..b015bd651 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (with `annotate`, `{}` for no options, the computer's driver draws each input over the frames in its caller's colour, which a driver whose card lists no `annotate` refuses) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (its driver draws each input over the frames in its caller's colour, and a pointer per role where the frames lack one, unless `annotation_settings` says `is_enabled: false`; a driver whose card lists no `annotate` records it plain, with a warning, and the run's recording entry says `annotated`) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 4bbb5c5db..c5ceb4de7 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -48,33 +48,36 @@ class StartComputerRecordingTaskStep(TaskStep): - """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``; with - ``annotate``, an object of options (``{}`` for none), its driver draws each input over the frames. Best effort by - default.""" + """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``, its driver + drawing each input over the frames unless ``annotation_settings`` says ``is_enabled: false``; a driver that + can't, as one composed before agent-env drew inputs, records it plain. Best effort by default.""" type: ClassVar[str] = "start_computer_recording" entity_refs = (EntityRef.env("env_id"),) def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, environment_name: Optional[str] = None, - annotate: Optional[dict] = None, depends_on: Optional[list[TaskStepDependency]] = None, + annotation_settings: Optional[dict] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) if not env_id or not name: raise ValueError("start_computer_recording needs an env_id and a name for the recording") - if annotate is not None and not isinstance(annotate, dict): - raise ValueError(f"start_computer_recording's annotate is an object of options, {{}} for none; got " - f"{annotate!r}") self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor - self.environment_name, self.annotate = environment_name, annotate + self.environment_name = environment_name + self.annotation_settings = _checked_annotation_settings(annotation_settings) + + @property + def annotates(self) -> bool: + """Whether the recording draws each input, as it does unless its settings turn it off.""" + return (self.annotation_settings or {}).get("is_enabled", True) def to_dict(self) -> dict: data = {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor} if self.environment_name is not None: # left out otherwise, so a step without one serializes as before data["environment_name"] = self.environment_name - if self.annotate is not None: - data["annotate"] = self.annotate + if self.annotation_settings is not None: # left out otherwise, so a step without them serializes as before + data["annotation_settings"] = self.annotation_settings return data @classmethod @@ -83,7 +86,8 @@ def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: base["fail_task_on_error"] = data.get("fail_task_on_error", False) return cls(**base, env_id=data["env_id"], name=data["name"], fps=data.get("fps"), max_edge=data.get("max_edge"), cursor=data.get("cursor", True), - environment_name=data.get("environment_name"), annotate=data.get("annotate")) + environment_name=data.get("environment_name"), + annotation_settings=data.get("annotation_settings")) async def execute(self, context: TaskStepContext) -> TaskStepContext: recordings = context.metadata.setdefault(RECORDINGS_KEY, {}) @@ -100,10 +104,16 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = _deployed(context, self.env_id) refuse_no_network(deployed) namespace = recording_namespace(request_id, _KEY_PREFIX) - grant = _grant(deployed, namespace, self.environment_name, annotate=self.annotate is not None) + grant = _grant(deployed, namespace, self.environment_name) + annotated = self.annotates and _annotates(deployed, self.environment_name) + if self.annotates and not annotated: + logger.warning("start_computer_recording %s on %s: its driver can't draw inputs, its card listing no " + "annotate, as a computer composed before agent-env drew them; recording it plain", + self.name, self.env_id) + options = {key: value for key, value in (self.annotation_settings or {}).items() if key != "is_enabled"} started = await deployed.invoke(COMPUTER_RECORDING, "start", { "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, - "upload": grant, **({"annotate": self.annotate} if self.annotate is not None else {})}, + "upload": grant, **({"annotate": options} if annotated else {})}, environment_name=self.environment_name) except Exception as e: recordings[self.name] = {"env_id": self.env_id, "state": "failed", "error": why(e)} @@ -113,7 +123,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context recordings[self.name] = { "env_id": self.env_id, "recording_id": started["recording_id"], "started_at": started["started_at"], - "namespace": namespace, "state": "open"} + "namespace": namespace, "state": "open", "annotated": annotated} if self.environment_name is not None: recordings[self.name]["environment_name"] = self.environment_name logger.info("start_computer_recording %s on %s: %s", self.name, self.env_id, started["recording_id"]) @@ -215,25 +225,42 @@ def _deployed(context: TaskStepContext, env_id: str): return deployed -def _grant(deployed, namespace: str, environment_name: Optional[str] = None, method: str = "start", - annotate: bool = False) -> dict: +def _grant(deployed, namespace: str, environment_name: Optional[str] = None, method: str = "start") -> dict: """A grant for a computer recording's segments and manifest. EnvCapabilityUnsupported when the env's card, or its - child's, declares no recording; with ``annotate``, a RuntimeError when it declares no ``annotate``, as a driver - from before agent-env drew inputs does.""" - card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) - extension = find_extension(card or {}, COMPUTER_RECORDING) + child's, declares no recording.""" + extension = _recording_extension(deployed, environment_name) if extension is None: raise EnvCapabilityUnsupported(COMPUTER_RECORDING, method, deployed.env_id, getattr(deployed, "instance_id", None), environment_name) - if annotate and "annotate" not in (extension.get("params") or {}): - who = f"{deployed.env_id}'s {environment_name}" if environment_name else deployed.env_id - raise RuntimeError(f"{who} can't annotate a recording: its driver's computer-recording/v1 lists no annotate, " - "as a driver composed before agent-env drew inputs; compose it again, or record it " - "without annotate") return recording_grant(deployed, extension.get("params") or {}, namespace, MAX_SEGMENTS + 1, _MAX_SEGMENT_BYTES, MAX_RECORDING_BYTES) +def _recording_extension(deployed, environment_name: Optional[str]) -> Optional[dict]: + card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) + return find_extension(card or {}, COMPUTER_RECORDING) + + +def _annotates(deployed, environment_name: Optional[str]) -> bool: + """Whether the env's driver, or its child's, draws inputs over a recording's frames: its card lists annotate.""" + return "annotate" in ((_recording_extension(deployed, environment_name) or {}).get("params") or {}) + + +def _checked_annotation_settings(settings: object) -> Optional[dict]: + """``annotation_settings``: None, or an object whose ``is_enabled``, true unless given, is a boolean.""" + if settings is None: + return None + if not isinstance(settings, dict): + raise ValueError(f"start_computer_recording's annotation_settings is an object, such as " + f'{{"is_enabled": false}}; got {settings!r}') + if unknown := sorted(set(settings) - {"is_enabled"}): + raise ValueError(f"start_computer_recording's annotation_settings takes is_enabled, got {unknown}") + if not isinstance(settings.get("is_enabled", True), bool): + raise ValueError(f"start_computer_recording's annotation_settings.is_enabled is true or false, got " + f"{settings['is_enabled']!r}") + return dict(settings) + + def _from_namespace(entry: dict) -> RecordingManifest: """What the recording's namespace holds, for a recording its driver didn't end for the run: the manifest the driver uploaded beside the segments when it ended it, or else one of the segments there.""" diff --git a/tst/integration/computer/desktop_example_test.py b/tst/integration/computer/desktop_example_test.py index fa3489bcd..8b494ffe3 100644 --- a/tst/integration/computer/desktop_example_test.py +++ b/tst/integration/computer/desktop_example_test.py @@ -51,6 +51,7 @@ def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state): context, manifest, frames = _recorded(ran.output, "desk", state) entry = context["metadata"]["computer_recordings"]["desk"] assert (entry["state"], entry["ended_by"], entry["missing_segments"]) == ("ended", "end", 0) + assert entry["annotated"] is False and "annotate" not in manifest assert manifest["source"] == "stream" and manifest["cursor"] is True and frames assert len(frames) == sum(s["frames"] for s in manifest["segments"]) drive = json.loads(context["metadata"]["verifications"]["drive"]["results"][0]["description"]) @@ -72,6 +73,7 @@ def test_an_agent_given_the_computers_skill_files_a_loaded_file_in_a_new_folder_ context, manifest, frames = _recorded(ran.output, "agent", state) entry = context["metadata"]["computer_recordings"]["agent"] assert (entry["state"], entry["missing_segments"]) == ("ended", 0) and frames + assert entry["annotated"] is True and manifest["annotate"] == {} assert "## Shortcuts" in context["metadata"]["env_skills"]["default-agent"]["desktop"]["desktop"] diff --git a/tst/integration/computer/recording/tasks/recording.json b/tst/integration/computer/recording/tasks/recording.json index ad000361f..5a67b77b1 100644 --- a/tst/integration/computer/recording/tasks/recording.json +++ b/tst/integration/computer/recording/tasks/recording.json @@ -1,7 +1,7 @@ [ {"id": "env", "type": "deploy_env", "env_id": "@local/agentenv-framework/desktop/desktop"}, {"id": "record", "type": "start_computer_recording", "env_id": "@local/agentenv-framework/desktop/desktop", - "name": "desk", "fps": 10, "depends_on": ["env"]}, + "name": "desk", "fps": 10, "annotation_settings": {"is_enabled": false}, "depends_on": ["env"]}, {"id": "drive", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "drive_desk", "verifier_id": "drive", "depends_on": ["record"]}, {"id": "end", "type": "end_computer_recording", "name": "desk", "depends_on": ["drive"]} diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 80653ccf7..c8910b5e5 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -348,30 +348,38 @@ async def test_a_start_without_a_child_env_reads_the_envs_own_card_as_before(sto @pytest.mark.asyncio -async def test_annotate_reaches_the_driver_only_when_a_step_asks_for_it(store): - computer, plain = _Computer(store), _Computer(store) - computer.environment_card = plain.environment_card = _ANNOTATING - await _start(annotate={}).execute(_context(computer)) - await _start().execute(_context(plain)) +async def test_a_recording_draws_its_inputs_unless_its_annotation_settings_turn_it_off(store): + on, off = _Computer(store), _Computer(store) + on.environment_card = off.environment_card = _ANNOTATING + started = (await _start().execute(_context(on))).metadata[RECORDINGS_KEY]["desk"] + plain = (await _start("plain", annotation_settings={"is_enabled": False}).execute(_context(off))).metadata[ + RECORDINGS_KEY]["plain"] - [(_, annotated)], [(_, unannotated)] = computer.calls, plain.calls - assert annotated["annotate"] == {} and "annotate" not in unannotated + [(_, annotated)], [(_, unannotated)] = on.calls, off.calls + assert annotated["annotate"] == {} and started["annotated"] is True + assert "annotate" not in unannotated and plain["annotated"] is False @pytest.mark.asyncio -async def test_annotating_on_a_driver_whose_card_lists_no_annotate_fails_the_start_best_effort_or_fatally(store): +async def test_a_driver_that_cant_draw_inputs_is_recorded_plain_with_a_warning_even_when_its_failures_are_fatal( + store, caplog): + """A computer composed before agent-env drew inputs lists no annotate on its card.""" computer = _Computer(store) - context = await _start(annotate={}).execute(_context(computer)) + with caplog.at_level("WARNING"): + context = await _start(fail_task_on_error=True).execute(_context(computer)) entry = context.metadata[RECORDINGS_KEY]["desk"] - assert entry["state"] == "failed" and "can't annotate a recording" in entry["error"] and computer.calls == [] - with pytest.raises(RuntimeError, match="lists no annotate"): - await _start(annotate={}, fail_task_on_error=True).execute(_context(_Computer(store))) + [(_, started)] = computer.calls + assert entry["state"] == "open" and entry["annotated"] is False and "annotate" not in started + assert "can't draw inputs" in caplog.text and "recording it plain" in caplog.text -def test_annotate_round_trips_only_when_set_and_is_an_object(): - annotated = _start(annotate={}) - assert StartComputerRecordingTaskStep.from_dict(annotated.to_dict()).annotate == {} - assert "annotate" not in _start().to_dict() - with pytest.raises(ValueError, match="annotate is an object of options"): - _start(annotate=True) +def test_annotation_settings_round_trip_only_when_set_and_take_a_boolean_is_enabled(): + off = _start(annotation_settings={"is_enabled": False}) + assert StartComputerRecordingTaskStep.from_dict(off.to_dict()).annotation_settings == {"is_enabled": False} + assert not off.annotates and _start().annotates and _start(annotation_settings={}).annotates + assert "annotation_settings" not in _start().to_dict() + for settings, problem in (({"is_enabled": "no"}, "is true or false"), ({"marks": []}, "takes is_enabled"), + (True, "is an object")): + with pytest.raises(ValueError, match=problem): + _start(annotation_settings=settings) From b5cfdb8e3c719cb5bbdb8a6afac1e2908ffe0c6a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 16:49:39 -0800 Subject: [PATCH 09/10] refactor(computers): one model for annotation options, marks named by their operations, pointer moves their own type The driver's annotate options are one Pydantic model, AnnotateOptions, which the card lists and the step's AnnotationSettings extends with is_enabled, so a later option is added once. A mark's kind is its operation's name, one vocabulary for the inputs drawn at a point, and a pointer move is a PointerMove with where it was and its path, not a Mark. A test drives every input tool on a page fake, moved to the shared fakes, through an annotated recording and checks the marks each makes, so a tool whose arguments change, or a new one, shows. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/annotate.py | 153 +++++++++++------- .../agentenv_protocol/computer/recording.py | 2 +- .../agentenv-protocol/tests/computer_fake.py | 59 +++++++ .../tests/test_computer_page.py | 59 +------ .../tests/test_computer_recording_annotate.py | 71 +++++++- .../task_steps/computer_recording.py | 35 ++-- tst/unit/task_step/computer_recording_test.py | 4 +- 7 files changed, 243 insertions(+), 140 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py index b8e4c8248..d0e0df402 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/annotate.py @@ -12,33 +12,33 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any, Optional +from typing import Any, Optional, Union import av from PIL import Image as PILImage from PIL import ImageDraw, ImageFont +from pydantic import BaseModel, ConfigDict, ValidationError from .errors import InvalidArgument -#: The options ``annotate`` takes: none yet, so ``{}`` draws every mark. -OPTIONS: tuple[str, ...] = () #: The role of a call that names none: the agent. DEFAULT_ROLE = "default" #: The pointer an input acts through when it names none: the computer's own, the only one the SDK drives yet. DEFAULT_POINTER = "default" #: Each role's colour: the agent's first, then each other role's in the order it first acts on the recording. COLOURS = ((255, 138, 0), (0, 184, 148), (175, 82, 222), (255, 59, 105), (52, 120, 246), (250, 204, 21)) -#: How long a mark shows from its input, in seconds; a drag's, after its moves. -SHOW_S = {"click": 0.8, "down": 0.8, "up": 0.8, "move": 0.5, "scroll": 0.9, "drag": 0.8, "chip": 1.6} +#: How long a mark shows from its input, in seconds; a drag's, after its moves. A mark's kind is its operation's name, +#: or chip for one drawn as a caption. +SHOW_S = {"click": 0.8, "pointer_down": 0.8, "pointer_up": 0.8, "pointer_move": 0.5, "scroll": 0.9, "drag": 0.8, + "chip": 1.6} #: The last part of a mark's time, in seconds, over which it fades out. FADE_S = 0.3 #: How long a mark no frame has drawn is kept, in seconds: past it, the recording's frames have stopped coming. KEEP_S = 30.0 -#: The marks of inputs that leave the pointer at their last point, a drag's after moving it along its path. -MOVES_POINTER = ("click", "drag", "scroll", "move", "down", "up") #: How long after the pointer jumps a line shows where it came from, in seconds. JUMP_S = 0.4 -#: The inputs whose marks are drawn at a point of the display. +#: The inputs whose marks are drawn at a point of the display, which leave the pointer at their last point, a drag's +#: after moving it along its path. POINTED = ("click", "drag", "scroll", "pointer_move", "pointer_down", "pointer_up") #: The most a chip of typed text shows, which grows with its length from the chip's own time. MOST_TEXT_S = 4.0 @@ -57,12 +57,44 @@ "ArrowDown": "Down", " ": "Space"} +class AnnotateOptions(BaseModel): + """What computer-recording/v1 ``start``'s ``annotate`` takes: no options yet, so ``{}`` draws every input. An + option is a field here, which the card lists, ``start`` checks, and a task step's annotation settings take too.""" + + model_config = ConfigDict(extra="forbid", strict=True) + + +def options_problem(error: ValidationError, model: type[BaseModel] = AnnotateOptions) -> str: + """What is wrong with annotation options ``model`` refused, in a sentence.""" + takes = ", ".join(model.model_fields) or "no options yet" + problems = [] + for e in error.errors(): + where = ".".join(str(part) for part in e["loc"]) + if e["type"] == "extra_forbidden": + problems.append(f"it takes {takes}, not {where!r}") + elif not where: + problems.append(f"it is an object of options, got {type(e['input']).__name__}") + else: + problems.append(f"{where}: {e['msg'].lower()}") + return "; ".join(problems) + + +def check_annotate(annotate: object) -> Optional[dict]: + """``start``'s ``annotate``: None, or an object of the options this driver takes, as given.""" + if annotate is None: + return None + try: + return AnnotateOptions.model_validate(annotate).model_dump(exclude_unset=True) + except ValidationError as e: + raise InvalidArgument(f"annotate: {options_problem(e)}; give {{}} to draw every input") from None + + @dataclass(frozen=True) class Mark: - """One input on a recording: ``kind`` is click, down, up, move, scroll, drag or chip, or pointer for where a - pointer was and then went; ``points`` are fractions of the display, so they land on any frame size; it shows from - ``at`` until ``until``, a drag moving for its first ``moving_s``. ``pointer`` is the pointer it acted through; - ``faded``, a pointer drawn fainter, as one that didn't move last.""" + """One input on a recording: ``kind`` is its operation (click, drag, scroll, pointer_move, pointer_down, + pointer_up) for one drawn at a point, or chip for one drawn as a caption; ``points`` are fractions of the display, + so they land on any frame size; it shows from ``at`` until ``until``, a drag moving for its first ``moving_s``. + ``pointer`` is the pointer it acted through.""" kind: str at: datetime @@ -75,18 +107,26 @@ class Mark: label: str = "" moving_s: float = 0.0 pointer: str = DEFAULT_POINTER + + +@dataclass(frozen=True) +class PointerMove: + """Where a role's pointer went at ``at``: from ``was`` along ``path``, a single point for a jump, a drag's path + while it moves for ``moving_s``. ``faded`` and ``label`` are how a frame draws it among others: fainter when it + didn't move last, named when there are several.""" + + at: datetime + role: str + pointer: str + was: tuple[float, float] + path: tuple[tuple[float, float], ...] + moving_s: float = 0.0 faded: bool = False + label: str = "" -def check_annotate(annotate: object) -> Optional[dict]: - """``start``'s ``annotate``: None, or an object of the options this driver takes.""" - if annotate is None: - return None - if not isinstance(annotate, dict): - raise InvalidArgument(f"annotate is an object of options, got {type(annotate).__name__}") - if unknown := sorted(set(annotate) - set(OPTIONS)): - raise InvalidArgument(f"annotate takes no options yet, got {unknown}; give {{}} to draw every input") - return dict(annotate) +#: What a frame draws: a mark or a pointer, in its colour, at its age in seconds. +Shown = tuple[Union[Mark, PointerMove], tuple[int, int, int], float] class Annotations: @@ -101,7 +141,7 @@ def __init__(self, pointer: bool = False) -> None: self.drawn: set[int] = set() self.colours: dict[str, tuple[int, int, int]] = {DEFAULT_ROLE: COLOURS[0]} self.pointer = pointer - self.moves: dict[tuple[str, str], list[Mark]] = {} # by role and pointer: each move, where it was and went + self.moves: dict[tuple[str, str], list[PointerMove]] = {} # each role's and pointer's, oldest first def add(self, marks: Sequence[Mark]) -> None: for mark in marks: @@ -109,14 +149,13 @@ def add(self, marks: Sequence[Mark]) -> None: stale = mark.at - timedelta(seconds=KEEP_S) self._keep([kept for kept in self.marks if kept.until > stale]) self.marks.append(mark) - if self.pointer and mark.kind in MOVES_POINTER and mark.points: + if self.pointer and mark.kind in POINTED: moves = self.moves.setdefault((mark.role, mark.pointer), []) path = mark.points if mark.kind == "drag" else mark.points[-1:] - was = moves[-1].points[-1] if moves else path[0] - moves.append(Mark("pointer", mark.at, mark.at, mark.role, (was, *path), moving_s=mark.moving_s, - pointer=mark.pointer)) + was = moves[-1].path[-1] if moves else path[0] + moves.append(PointerMove(mark.at, mark.role, mark.pointer, was, path, mark.moving_s)) - def _pointers(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: + def _pointers(self, at: datetime) -> list[Shown]: """The pointers a frame captured ``at`` shows, each as its last move before the frame at its age, the one that moved last drawn over the others; none for a role or pointer that hadn't acted by then.""" current = [] @@ -131,7 +170,7 @@ def _pointers(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], floa label=_pointer_label(move) if several else "") for move in current] return [(move, self.colours[move.role], (at - move.at).total_seconds()) for move in shown] - def showing(self, at: datetime) -> list[tuple[Mark, tuple[int, int, int], float]]: + def showing(self, at: datetime) -> list[Shown]: """The marks a frame captured ``at`` shows, each with its colour and the age, in seconds, it is drawn at: a mark over by then that no frame has drawn yet is drawn once, at the age it is last fully shown. A stream's frames come in the order they were captured, so a mark that is over and drawn is over for every later one.""" @@ -187,13 +226,12 @@ def chip(label: str, seconds: Optional[float] = None) -> Mark: x, y, direction, amount = args[:4] marks.append(Mark("scroll", at, shown("scroll"), role, (where(x, y),), direction=direction, count=amount)) elif operation == "pointer_move": - marks.append(Mark("move", at, shown("move"), role, (where(*args[:2]),))) + marks.append(Mark(operation, at, shown(operation), role, (where(*args[:2]),))) elif operation in ("pointer_down", "pointer_up"): - kind = "down" if operation == "pointer_down" else "up" if point is not None: - marks.append(Mark(kind, at, shown(kind), role, (where(*point[:2]),), button=args[0])) + marks.append(Mark(operation, at, shown(operation), role, (where(*point[:2]),), button=args[0])) else: - marks.append(chip(f"{'press' if kind == 'down' else 'release'} {args[0]}")) + marks.append(chip(f"{'press' if operation == 'pointer_down' else 'release'} {args[0]}")) elif operation == "type_text": text = args[0] marks.append(chip(f'type "{_shown_text(text)}"', min(MOST_TEXT_S, SHOW_S["chip"] + len(text) / 25))) @@ -239,21 +277,27 @@ def tool_mark(tool: str, arguments: Mapping[str, Any], *, role: Optional[str], a label=label if role == DEFAULT_ROLE else f"{role}: {label}") -def draw(image: Any, marks: Sequence[tuple[Mark, tuple[int, int, int], float]]) -> PILImage.Image: - """``image``, a Pillow image or a PyAV frame, with ``marks`` drawn, each in its colour at its age.""" +def draw(image: Any, shown: Sequence[Shown]) -> PILImage.Image: + """``image``, a Pillow image or a PyAV frame, with what a frame shows drawn, each in its colour at its age.""" base = (image.to_image() if isinstance(image, av.VideoFrame) else image).convert("RGBA") overlay = PILImage.new("RGBA", base.size, (0, 0, 0, 0)) pen = ImageDraw.Draw(overlay) unit = max(base.size) / UNIT_EDGE + + def pixels(point: tuple[float, float]) -> tuple[float, float]: + return (point[0] * base.width, point[1] * base.height) + chips = [] - for mark, colour, age in marks: - span = max((mark.until - mark.at).total_seconds(), 1e-3) - if mark.kind == "chip": - chips.append((mark, colour)) + for item, colour, age in shown: + if isinstance(item, PointerMove): + _draw_pointer(pen, item, pixels(item.was), [pixels(p) for p in item.path], colour, unit, age) + continue + if item.kind == "chip": + chips.append((item, colour)) continue - fade = 1.0 if mark.kind == "pointer" else 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) - points = [(fx * base.width, fy * base.height) for fx, fy in mark.points] - _DRAWERS[mark.kind](pen, mark, points, colour, unit, age, fade) + span = max((item.until - item.at).total_seconds(), 1e-3) + fade = 1.0 - max(0.0, min(1.0, (age - (span - FADE_S)) / FADE_S)) + _DRAWERS[item.kind](pen, item, [pixels(p) for p in item.points], colour, unit, age, fade) if chips: _draw_chips(pen, chips, base.size, unit) return PILImage.alpha_composite(base, overlay).convert("RGB") @@ -331,8 +375,9 @@ def _draw_click(pen, mark, points, colour, unit, age, fade) -> None: def _draw_press(pen, mark, points, colour, unit, age, fade) -> None: x, y = points[0] alpha = round(255 * fade) - r = unit * (STYLE.press_down if mark.kind == "down" else STYLE.press_up) - fill = (*colour, round(alpha * STYLE.press_fill)) if mark.kind == "down" else None + down = mark.kind == "pointer_down" + r = unit * (STYLE.press_down if down else STYLE.press_up) + fill = (*colour, round(alpha * STYLE.press_fill)) if down else None pen.ellipse((x - r, y - r, x + r, y + r), fill=fill, outline=(*WHITE, alpha), width=_width(STYLE.halo, unit)) pen.ellipse((x - r, y - r, x + r, y + r), outline=(*colour, alpha), width=_width(STYLE.stroke, unit)) @@ -376,25 +421,25 @@ def _draw_drag(pen, mark, points, colour, unit, age, fade) -> None: ARROW = ((0, 0), (0, 17), (4, 13), (7, 20), (10, 19), (7, 12), (12, 12)) -def _draw_pointer(pen, mark, points, colour, unit, age, fade) -> None: - """The agent's pointer: along a drag's path while it moves, else where its input left it, after a fading line - from where it jumped.""" - was, path = points[0], points[1:] - tip = _along(path, age / mark.moving_s)[-1] if len(path) > 1 and age < mark.moving_s else path[-1] +def _draw_pointer(pen, move: PointerMove, was, path, colour, unit, age) -> None: + """A pointer: along a drag's path while it moves, else where its input left it, after a fading line from where + it jumped; fainter when it didn't move last, and named when it is one of several.""" + tip = _along(path, age / move.moving_s)[-1] if len(path) > 1 and age < move.moving_s else path[-1] if age < JUMP_S and math.dist(was, path[0]) > unit * STYLE.cross: alpha = round(STYLE.jump_alpha * (1 - age / JUMP_S)) pen.line((*was, *path[0]), fill=(*colour, alpha), width=_width(STYLE.stroke, unit)) scale = unit * STYLE.pointer arrow = [(tip[0] + dx * scale, tip[1] + dy * scale) for dx, dy in ARROW] - alpha = STYLE.pointer_faded_alpha if mark.faded else 255 + alpha = STYLE.pointer_faded_alpha if move.faded else 255 pen.polygon(arrow, fill=(*colour, alpha), outline=(*WHITE, alpha), width=_width(STYLE.pointer_outline, unit)) - if mark.label: + if move.label: corner = max(dx for dx, _dy in ARROW), max(dy for _dx, dy in ARROW) - _label(pen, (tip[0] + corner[0] * scale, tip[1] + corner[1] * scale), mark.label, colour, unit, alpha) + _label(pen, (tip[0] + corner[0] * scale, tip[1] + corner[1] * scale), move.label, colour, unit, alpha) -_DRAWERS = {"click": _draw_click, "down": _draw_press, "up": _draw_press, "move": _draw_move, "scroll": _draw_scroll, - "drag": _draw_drag, "pointer": _draw_pointer} +#: How each kind of mark drawn at a point is drawn, by its operation. +_DRAWERS = {"click": _draw_click, "pointer_down": _draw_press, "pointer_up": _draw_press, + "pointer_move": _draw_move, "scroll": _draw_scroll, "drag": _draw_drag} def _draw_chips(pen, chips, size, unit) -> None: @@ -445,7 +490,7 @@ def _along(points: list[tuple[float, float]], fraction: float) -> list[tuple[flo return shown -def _pointer_label(move: Mark) -> str: +def _pointer_label(move: PointerMove) -> str: """A pointer's name beside it: its role's, the agent's as agent, and its own when it isn't the default.""" role = "agent" if move.role == DEFAULT_ROLE else move.role return role if move.pointer == DEFAULT_POINTER else f"{role} {move.pointer}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 2ff0ca085..28d960194 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -128,7 +128,7 @@ def recording_params(recording: Recording, streams: bool) -> dict: return {"formats": [MP4], "scopes": ["display"], "fps": {"default": min(DEFAULT_FPS, most), "max": most}, "cursor": True, "source": "stream" if streams else "capture", "max_segment_s": recording.max_segment_s, "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s, - "annotate": {"options": list(annotating.OPTIONS)}} + "annotate": {"options": list(annotating.AnnotateOptions.model_fields)}} class Recordings: diff --git a/packages/agentenv-protocol/tests/computer_fake.py b/packages/agentenv-protocol/tests/computer_fake.py index 6133104cb..7e526c98d 100644 --- a/packages/agentenv-protocol/tests/computer_fake.py +++ b/packages/agentenv-protocol/tests/computer_fake.py @@ -15,13 +15,22 @@ class with no display. A click paints its button's colour at its point; everythi ComputerSettings, Display, Effect, + FileChooser, + Image, + PageDialog, + PageState, Platform, + Tab, capture, click, + dialog, display, drag, + history, key_down, key_up, + navigate, + page_state, pointer_down, pointer_move, pointer_position, @@ -29,7 +38,9 @@ class with no display. A click paints its button's colour at its point; everythi press, scroll, set_display, + tabs, type_text, + upload, ) LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") @@ -162,6 +173,54 @@ async def keys_up(self, keys): self.screen.held = [k for k in self.screen.held if k not in keys] +class PageFake(FakeDriver): + """A fake whose screen is a browser's page, with two tabs, the first active, and a log of the page calls.""" + + def __init__(self) -> None: + super().__init__() + self.tabs = [Tab("a", "https://mail.test/", "Inbox", True), Tab("b", "https://docs.test/", "Docs")] + self.open_dialog: PageDialog | None = None + self.chooser: FileChooser | None = None + self.calls: list = [] + self.captured: PageState | None = None + + @capture + async def grab(self): + image = await super().grab() + if self.captured is None: + return image + return Image(image.width, image.height, image.tobytes(), "RGB", self.captured) + + @navigate + async def go_to(self, url): + self.calls.append(("navigate", url)) + + @history + async def step(self, direction): + self.calls.append(("history", direction)) + + @tabs + async def tab_action(self, action, tab, url): + self.calls.append(("tabs", action, tab, url)) + if action == "switch": + self.tabs = [Tab(t.id, t.url, t.title, t.id == tab) for t in self.tabs] + return self.tabs + + @dialog + async def answer(self, accept, text): + self.calls.append(("dialog", accept, text)) + self.open_dialog = None + + @upload + async def choose(self, paths): + self.calls.append(("upload", paths)) + + @page_state + async def page_now(self): + active = next(t for t in self.tabs if t.active) + return PageState(active.id, active.url, active.title, self.open_dialog, self.chooser) + + class MinimalDriver(ComputerDriverEnvironment): """Only the required operations, synchronous ones: no pointer or keyboard group, no display modes.""" diff --git a/packages/agentenv-protocol/tests/test_computer_page.py b/packages/agentenv-protocol/tests/test_computer_page.py index f417cbb30..862d86a9f 100644 --- a/packages/agentenv-protocol/tests/test_computer_page.py +++ b/packages/agentenv-protocol/tests/test_computer_page.py @@ -8,74 +8,17 @@ pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") -from computer_fake import FakeDriver, http_client, mcp_session # noqa: E402 +from computer_fake import FakeDriver, PageFake, http_client, mcp_session # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 FileChooser, - Image, PageDialog, PageState, - Tab, - capture, - dialog, - history, - navigate, - page_state, - tabs, - upload, ) PAGE_TOOLS = {"computer_navigate", "computer_history", "computer_tabs", "computer_dialog", "computer_upload"} -class PageFake(FakeDriver): - """A fake whose screen is a browser's page, with two tabs, the first active, and a log of the page calls.""" - - def __init__(self) -> None: - super().__init__() - self.tabs = [Tab("a", "https://mail.test/", "Inbox", True), Tab("b", "https://docs.test/", "Docs")] - self.open_dialog: PageDialog | None = None - self.chooser: FileChooser | None = None - self.calls: list = [] - self.captured: PageState | None = None - - @capture - async def grab(self): - image = await super().grab() - if self.captured is None: - return image - return Image(image.width, image.height, image.tobytes(), "RGB", self.captured) - - @navigate - async def go_to(self, url): - self.calls.append(("navigate", url)) - - @history - async def step(self, direction): - self.calls.append(("history", direction)) - - @tabs - async def tab_action(self, action, tab, url): - self.calls.append(("tabs", action, tab, url)) - if action == "switch": - self.tabs = [Tab(t.id, t.url, t.title, t.id == tab) for t in self.tabs] - return self.tabs - - @dialog - async def answer(self, accept, text): - self.calls.append(("dialog", accept, text)) - self.open_dialog = None - - @upload - async def choose(self, paths): - self.calls.append(("upload", paths)) - - @page_state - async def page_now(self): - active = next(t for t in self.tabs if t.active) - return PageState(active.id, active.url, active.title, self.open_dialog, self.chooser) - - @pytest.mark.asyncio async def test_a_page_driver_declares_computer_page_and_serves_its_tools_and_a_desktop_driver_neither(): async with http_client(PageFake()) as http: diff --git a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py index e4b8f6f13..b5f527582 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording_annotate.py +++ b/packages/agentenv-protocol/tests/test_computer_recording_annotate.py @@ -14,7 +14,7 @@ pytest.importorskip("PIL") av = pytest.importorskip("av") -from computer_fake import FakeDriver, MinimalDriver, http_client, mcp_session # noqa: E402 +from computer_fake import FakeDriver, MinimalDriver, PageFake, http_client, mcp_session # noqa: E402 from mcp.types import CallToolResult, TextContent # noqa: E402 from PIL import Image as PILImage # noqa: E402 @@ -22,6 +22,8 @@ from agentenv_protocol.computer import ( # noqa: E402 COMPUTER_RECORDING, DeviceUnavailable, + FileChooser, + PageDialog, RecordedSegment, Recording, record, @@ -33,11 +35,13 @@ STYLE, UNIT_EDGE, Annotations, + PointerMove, draw, input_marks, tool_mark, ) from agentenv_protocol.computer.conformance import checks # noqa: E402 +from agentenv_protocol.computer.tools import COMPUTER_TOOLS # noqa: E402 from agentenv_protocol.tool_context import ROLE_META_KEY # noqa: E402 START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" @@ -206,7 +210,7 @@ async def test_start_refuses_an_annotate_that_isnt_an_object_or_names_an_option_ assert option.status_code == scalar.status_code == 400 assert option.json()["error"]["code"] == scalar.json()["error"]["code"] == "invalid_argument" - assert "takes no options yet, got ['events']" in option.json()["error"]["message"] + assert "it takes no options yet, not 'events'" in option.json()["error"]["message"] @pytest.mark.asyncio @@ -227,7 +231,7 @@ def labels(operation, args, role=None, **kw): assert labels("press", (["Control", "s"],), repeat=3) == [("chip", "key Ctrl+S x3")] assert labels("key_down", (["Shift"],), hold_ms=500) == [("chip", "hold Shift 500 ms")] assert labels("pointer_down", ("left",)) == [("chip", "press left")] - assert labels("pointer_down", ("left",), point=(10, 20)) == [("down", "")] + assert labels("pointer_down", ("left",), point=(10, 20)) == [("pointer_down", "")] assert labels("navigate", ("https://example.com",), role="user") == [("chip", "user: open https://example.com")] assert labels("tabs", ("list", None, None)) == [] [click] = input_marks("click", (500, 250, "right", 1, []), role=None, at=T0, size=(1000, 500)) @@ -388,8 +392,8 @@ def test_the_pointer_follows_the_last_input_before_each_frame_from_the_first_inp annotations.add(input_marks("pointer_move", (50, 50), role=None, at=T0 + timedelta(seconds=1), size=(100, 100))) def pointer(seconds): - return [(mark.points[0], mark.points[-1]) for mark, _colour, _age in - annotations.showing(T0 + timedelta(seconds=seconds)) if mark.kind == "pointer"] + return [(move.was, move.path[-1]) for move, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if isinstance(move, PointerMove)] assert pointer(-1) == [] assert pointer(0.5) == [((0.1, 0.1), (0.1, 0.1))] @@ -402,8 +406,8 @@ def test_each_role_and_pointer_has_a_pointer_the_last_mover_on_top_each_named_wh annotations.add(input_marks("click", (10, 10, "left", 1, []), role=None, at=T0, size=(100, 100))) def pointers(seconds): - return [(mark.role, mark.pointer, mark.points[-1], mark.faded, mark.label) for mark, _colour, _age in - annotations.showing(T0 + timedelta(seconds=seconds)) if mark.kind == "pointer"] + return [(move.role, move.pointer, move.path[-1], move.faded, move.label) for move, _colour, _age in + annotations.showing(T0 + timedelta(seconds=seconds)) if isinstance(move, PointerMove)] assert pointers(0.5) == [("default", "default", (0.1, 0.1), False, "")] annotations.add(input_marks("click", (50, 50, "left", 1, []), role="user", at=T0 + timedelta(seconds=1), @@ -415,3 +419,56 @@ def pointers(seconds): assert pointers(2.5) == [("default", "default", (0.1, 0.1), True, "agent"), ("user", "default", (0.5, 0.5), True, "user"), ("default", "p2", (0.8, 0.2), False, "agent p2")] + + +#: Each input tool, a call to it, and the marks it makes, as (kind, label), in the order the calls go. +EVERY_INPUT = ( + ("computer_click", {"x": 10, "y": 10}, [("click", "")]), + ("computer_drag", {"path": [[10, 10], [60, 40]]}, [("drag", "")]), + ("computer_scroll", {"x": 10, "y": 10, "direction": "down"}, [("scroll", "")]), + ("computer_type", {"text": "hi"}, [("chip", 'type "hi"')]), + ("computer_key", {"keys": ["Control", "s"]}, [("chip", "key Ctrl+S")]), + ("computer_pointer_move", {"x": 20, "y": 20}, [("pointer_move", "")]), + ("computer_pointer_down", {}, [("pointer_down", "")]), + ("computer_pointer_up", {"x": 30, "y": 30}, [("pointer_up", "")]), + ("computer_keyboard_down", {"keys": ["Shift"]}, [("chip", "hold Shift")]), + ("computer_keyboard_up", {"keys": ["Shift"]}, [("chip", "release Shift")]), + ("computer_navigate", {"url": "https://x.test/"}, [("chip", "open https://x.test/")]), + ("computer_history", {"direction": "back"}, [("chip", "back")]), + ("computer_tabs", {"action": "switch", "tab": "b"}, [("chip", "tab switch b")]), + ("computer_dialog", {"accept": True}, [("chip", "dialog OK")]), + ("computer_upload", {"paths": ["a.pdf"]}, [("chip", "upload a.pdf")]), +) + + +@pytest.mark.asyncio +async def test_every_input_tool_is_marked_by_what_it_did_with_the_arguments_it_hands_its_operation(caplog): + """The marks are made from each operation's arguments as the tools build them, so a tool whose arguments change + shows here: by a wrong mark, or by the warning a mark that couldn't be made logs. A new input tool fails it until + it's listed.""" + driver = PageFake() + noted: list = [] + async with http_client(driver) as http, mcp_session(driver) as session: + served = {tool.name for tool in (await session.list_tools()).tools if tool.name in COMPUTER_TOOLS + and tool.annotations is not None and tool.annotations.readOnlyHint is False} + started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json() + recordings = driver._computer.recordings + note = recordings.note + recordings.note = lambda marks: (noted.append(list(marks)), note(marks)) + await session.call_tool("computer_screenshot", {}) + made = {} + with caplog.at_level("WARNING"): + for tool, args, _marks in EVERY_INPUT: + if tool == "computer_dialog": + driver.open_dialog = PageDialog("confirm", "Leave the page?") + if tool == "computer_upload": + driver.chooser = FileChooser() + noted.clear() + result = await session.call_tool(tool, args) + assert not result.isError, (tool, result.content[0].text) + made[tool] = [(mark.kind, mark.label) for marks in noted for mark in marks] + await http.post(END, json={"recording_id": started["recording_id"]}) + + assert served == {tool for tool, _args, _marks in EVERY_INPUT} + assert made == {tool: marks for tool, _args, marks in EVERY_INPUT} + assert "couldn't mark" not in caplog.text diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index c5ceb4de7..ea5932e0d 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -12,10 +12,12 @@ import uuid from typing import ClassVar, Optional +from agentenv_protocol.computer.annotate import AnnotateOptions, options_problem from agentenv_protocol.computer.card import COMPUTER_RECORDING from agentenv_protocol.computer.manifest import ManifestSegment, RecordingManifest from agentenv_protocol.computer.recording import MANIFEST from agentenv_protocol.client import find_extension +from pydantic import ValidationError from agent_env.artifact import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse @@ -47,6 +49,13 @@ MAX_RECORDING_BYTES = 64 << 30 +class AnnotationSettings(AnnotateOptions): + """``start_computer_recording``'s ``annotation_settings``: whether the recording draws its inputs, as it does + unless ``is_enabled`` is false, and the options its driver's ``annotate`` takes, which the step passes on.""" + + is_enabled: bool = True + + class StartComputerRecordingTaskStep(TaskStep): """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``, its driver drawing each input over the frames unless ``annotation_settings`` says ``is_enabled: false``; a driver that @@ -64,12 +73,17 @@ def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: raise ValueError("start_computer_recording needs an env_id and a name for the recording") self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor self.environment_name = environment_name - self.annotation_settings = _checked_annotation_settings(annotation_settings) + self.annotation_settings = annotation_settings + try: + self.annotation = AnnotationSettings.model_validate(annotation_settings or {}) + except ValidationError as e: + raise ValueError(f"start_computer_recording's annotation_settings: " + f"{options_problem(e, AnnotationSettings)}") from None @property def annotates(self) -> bool: """Whether the recording draws each input, as it does unless its settings turn it off.""" - return (self.annotation_settings or {}).get("is_enabled", True) + return self.annotation.is_enabled def to_dict(self) -> dict: data = {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, @@ -110,7 +124,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning("start_computer_recording %s on %s: its driver can't draw inputs, its card listing no " "annotate, as a computer composed before agent-env drew them; recording it plain", self.name, self.env_id) - options = {key: value for key, value in (self.annotation_settings or {}).items() if key != "is_enabled"} + options = self.annotation.model_dump(exclude={"is_enabled"}, exclude_unset=True) started = await deployed.invoke(COMPUTER_RECORDING, "start", { "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, "upload": grant, **({"annotate": options} if annotated else {})}, @@ -246,21 +260,6 @@ def _annotates(deployed, environment_name: Optional[str]) -> bool: return "annotate" in ((_recording_extension(deployed, environment_name) or {}).get("params") or {}) -def _checked_annotation_settings(settings: object) -> Optional[dict]: - """``annotation_settings``: None, or an object whose ``is_enabled``, true unless given, is a boolean.""" - if settings is None: - return None - if not isinstance(settings, dict): - raise ValueError(f"start_computer_recording's annotation_settings is an object, such as " - f'{{"is_enabled": false}}; got {settings!r}') - if unknown := sorted(set(settings) - {"is_enabled"}): - raise ValueError(f"start_computer_recording's annotation_settings takes is_enabled, got {unknown}") - if not isinstance(settings.get("is_enabled", True), bool): - raise ValueError(f"start_computer_recording's annotation_settings.is_enabled is true or false, got " - f"{settings['is_enabled']!r}") - return dict(settings) - - def _from_namespace(entry: dict) -> RecordingManifest: """What the recording's namespace holds, for a recording its driver didn't end for the run: the manifest the driver uploaded beside the segments when it ended it, or else one of the segments there.""" diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index c8910b5e5..909b3dd2b 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -379,7 +379,7 @@ def test_annotation_settings_round_trip_only_when_set_and_take_a_boolean_is_enab assert StartComputerRecordingTaskStep.from_dict(off.to_dict()).annotation_settings == {"is_enabled": False} assert not off.annotates and _start().annotates and _start(annotation_settings={}).annotates assert "annotation_settings" not in _start().to_dict() - for settings, problem in (({"is_enabled": "no"}, "is true or false"), ({"marks": []}, "takes is_enabled"), - (True, "is an object")): + for settings, problem in (({"is_enabled": "no"}, "is_enabled: input should be a valid boolean"), + ({"marks": []}, "it takes is_enabled, not 'marks'"), (True, "is an object of options")): with pytest.raises(ValueError, match=problem): _start(annotation_settings=settings) From 2649c22d205cfff297e26c63aa14b05abdd5deaa Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 16:53:41 -0800 Subject: [PATCH 10/10] fix(task_step): a recording's entry says whether its video was drawn on, and falsy annotation settings are refused Co-Authored-By: Claude Opus 5.5 --- .../task_steps/computer_recording.py | 5 ++++- tst/unit/task_step/computer_recording_test.py | 21 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index ea5932e0d..a6949f4df 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -75,7 +75,8 @@ def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: self.environment_name = environment_name self.annotation_settings = annotation_settings try: - self.annotation = AnnotationSettings.model_validate(annotation_settings or {}) + self.annotation = AnnotationSettings.model_validate({} if annotation_settings is None else + annotation_settings) except ValidationError as e: raise ValueError(f"start_computer_recording's annotation_settings: " f"{options_problem(e, AnnotationSettings)}") from None @@ -229,6 +230,8 @@ def _registered(context: TaskStepContext, name: str, entry: dict, manifest: Reco uploaded = [s for s in manifest.segments if s.uploaded] entry.update(state="ended", ended_by=manifest.ended_by, segments=len(uploaded), missing_segments=len(manifest.segments) - len(uploaded)) + if manifest.source is not None: # the driver's own says whether it drew; one rebuilt from segments can't + entry["annotated"] = manifest.annotate is not None logger.info("recording %s: %d segments, ended by %s", name, len(uploaded), manifest.ended_by) diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 909b3dd2b..72bd76b2a 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -39,6 +39,7 @@ def __init__(self, store, env_id: str = "desktop", forgets: bool = False, garble self.store, self.forgets, self.garbles, self.drop_last = store, forgets, garbles, drop_last self.calls: list[tuple[str, dict]] = [] self.started: set[str] = set() + self.manifest: dict = {} # what else its manifest says, as a driver's own does async def invoke(self, uri, method, params=None, **_): assert uri == COMPUTER_RECORDING @@ -60,7 +61,7 @@ async def invoke(self, uri, method, params=None, **_): {"index": 1, "path": "rec-1/0001.mp4", "uploaded": not self.drop_last}] if not self.drop_last: self.store.put(f"{root}/rec-1/0001.mp4", b"mp4-1", content_type="video/mp4") - manifest = {"recording_id": "rec-1", "ended_by": params["reason"], "segments": segments} + manifest = {"recording_id": "rec-1", "ended_by": params["reason"], "segments": segments, **self.manifest} self.store.put(f"{root}/rec-1/manifest.json", json.dumps(manifest).encode(), content_type="application/json") return {"manifests": [manifest]} if method == "end_all" else manifest @@ -380,6 +381,22 @@ def test_annotation_settings_round_trip_only_when_set_and_take_a_boolean_is_enab assert not off.annotates and _start().annotates and _start(annotation_settings={}).annotates assert "annotation_settings" not in _start().to_dict() for settings, problem in (({"is_enabled": "no"}, "is_enabled: input should be a valid boolean"), - ({"marks": []}, "it takes is_enabled, not 'marks'"), (True, "is an object of options")): + ({"marks": []}, "it takes is_enabled, not 'marks'"), (True, "is an object of options"), + (False, "is an object of options"), ("", "is an object of options")): with pytest.raises(ValueError, match=problem): _start(annotation_settings=settings) + + +@pytest.mark.parametrize("manifest, annotated", [({"source": "stream", "annotate": {}}, True), + ({"source": "stream"}, False), ({}, True)], + ids=["drawn", "a-native-recorders-segments", "rebuilt-from-segments"]) +@pytest.mark.asyncio +async def test_a_recordings_entry_says_whether_its_video_was_drawn_on_as_its_drivers_own_manifest_says( + store, manifest, annotated): + """A driver that annotates can still hand over finished segments, which aren't drawn on and whose manifest names + no annotate; a manifest rebuilt from the segments alone can't say, so the start's answer stands.""" + computer = _Computer(store) + computer.environment_card, computer.manifest = _ANNOTATING, manifest + context = await _end().execute(await _start().execute(_context(computer))) + + assert context.metadata[RECORDINGS_KEY]["desk"]["annotated"] is annotated