From ed737915dca0cf9a5cb0d90223bec0df9d227187 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 20:16:47 -0800 Subject: [PATCH 1/2] feat(task): tasks declare headline_verifiers A task names, by step id, the grades that are its reward: one is the reward and the run's pass or fail, several a multi-reward over them, none a multi-reward over every grade. Saved only when set, so no stored task or ledger digest changes; a bundle task file may be {"headline_verifiers": [...], "steps": [...]}, and a bare array still loads with none. Construction refuses a headline that names no step of the task, or one named twice. The plan, materialize, the ledger, bundle init, episodes, task create and task get all carry it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- src/agent_env/bundle/importer.py | 3 +- src/agent_env/bundle/ledger.py | 5 +++ src/agent_env/bundle/materialize.py | 5 ++- src/agent_env/bundle/parse.py | 31 +++++++++++--- src/agent_env/bundle/plan.py | 3 +- src/agent_env/cli/task/create.py | 25 ++++++----- src/agent_env/cli/task/get.py | 6 ++- src/agent_env/task/episode/spec.py | 3 +- src/agent_env/task/task.py | 22 ++++++++++ tst/integration/cli/run_bundle_test.py | 16 +++++++ tst/unit/bundle/importer_test.py | 14 ++++++- tst/unit/bundle/materialize_test.py | 36 ++++++++++++++++ tst/unit/bundle/parse_test.py | 35 +++++++++++++--- tst/unit/cli/test_task_create_preflight.py | 30 +++++++++++++ tst/unit/cli/test_task_get.py | 23 ++++++++++ tst/unit/task/episode/spec_test.py | 6 +++ tst/unit/task/task_document_test.py | 49 ++++++++++++++++++++++ 18 files changed, 285 insertions(+), 31 deletions(-) create mode 100644 tst/unit/cli/test_task_get.py diff --git a/AGENTS.md b/AGENTS.md index bc7731236..fbf61d7ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | @@ -133,7 +133,7 @@ and deserialized through a registry. | `openenv/` | `build_app(bundle)`: the [OpenEnv](https://github.com/openenv/OpenEnv) server for a bundle's tasks, each session playing one episode through the Episode API, with the envs' MCP tools, an added `end_turn` tool and the task's verifier score as the reward. It prepares bundles as `agent-env run` does (`prepare_bundle(played=True)`, without the agent the client plays) and keeps namespace routing on for the app's life. `openenv` comes with the `openenv` extra, a floor with no cap; this package and `training/` are the only ones that import it. Each task's description in the Task API names its `toolset` (the played agent's envs, role, and whether it gets `bash`). With `token`, every request and WebSocket but `/health` must carry `Authorization: Bearer `, for a server others reach. | | `training/` | One module per outside trainer, each behind its own extra, and the modules they share; [Training helpers](#training-helpers) lists them. | | `runner/` | The `[runner]` seam: `Runner.submit()` returns a `RunHandle` (`run_id`, `instance_id`) before the run starts; `LocalRunner` is built in. `runner/store.py` is the run index every runner writes and the explorer reads. | -| `bundle/` | A bundle is a folder of entities (`envs/`, `agents/`, `artifacts/`, `skills/`, `tasks/.json`, `evals/.toml`) that runs without a registry: `agent-env run` plans it, writes it into the `@local` namespace (reusing each version whose inputs haven't changed) and runs its evals or tasks; `agent-env run hf:///[@][/]` (`bundle/hub.py`) runs one from a Hugging Face Hub dataset, picked from the commit's file list (the path, else `bundles/`, else the root) and downloaded alone with the card into a folder per commit, under ids rooted at `@hf//`, which only that path produces, each run's context recording the commit (`metadata["bundle_source"]`); `agent-env bundle init` writes a stored task or eval as one, with every env, agent and artifact it uses in its own folder (each image as its build context, pruned to what its Dockerfile reads and pinned to the Python and npm versions of the stored image, or, given `--image-ref ID=REF`, as that registry image pinned by digest with its context beside it, `image = { ref, dockerfile }`, which a run pulls and builds only when it can't; `task_images` lists the images an import writes, downloading nothing, so an export can push them first; a computer env as its base, driver and apps, each with its files, never what it composed, which a run of the folder composes, and an env or artifact of an installed bundle, there or anywhere an entity names one, by its id) so it runs with no store; `agent-env bundle register` writes a folder into the configured stores under registry ids (``, and an installed bundle's agent, env or artifact a reference pulls in under its alias, an env only into a shared store), each task and eval naming the versions it reads and each image it builds pushed, through a ledger in the target store so registering again writes only what changed. | +| `bundle/` | A bundle is a folder of entities (`envs/`, `agents/`, `artifacts/`, `skills/`, `tasks/.json`, a JSON array of steps or `{"headline_verifiers": [...], "steps": [...]}`, `evals/.toml`) that runs without a registry: `agent-env run` plans it, writes it into the `@local` namespace (reusing each version whose inputs haven't changed) and runs its evals or tasks; `agent-env run hf:///[@][/]` (`bundle/hub.py`) runs one from a Hugging Face Hub dataset, picked from the commit's file list (the path, else `bundles/`, else the root) and downloaded alone with the card into a folder per commit, under ids rooted at `@hf//`, which only that path produces, each run's context recording the commit (`metadata["bundle_source"]`); `agent-env bundle init` writes a stored task or eval as one, with every env, agent and artifact it uses in its own folder (each image as its build context, pruned to what its Dockerfile reads and pinned to the Python and npm versions of the stored image, or, given `--image-ref ID=REF`, as that registry image pinned by digest with its context beside it, `image = { ref, dockerfile }`, which a run pulls and builds only when it can't; `task_images` lists the images an import writes, downloading nothing, so an export can push them first; a computer env as its base, driver and apps, each with its files, never what it composed, which a run of the folder composes, and an env or artifact of an installed bundle, there or anywhere an entity names one, by its id) so it runs with no store; `agent-env bundle register` writes a folder into the configured stores under registry ids (``, and an installed bundle's agent, env or artifact a reference pulls in under its alias, an env only into a shared store), each task and eval naming the versions it reads and each image it builds pushed, through a ledger in the target store so registering again writes only what changed. | | `eval/` | `Eval` (`eval`), a versioned list of task references, and its store. | | `plugins/` | Entry-point plugins (see Extension points): the group names, `inventory()`, which `agent-env plugin list/show/check` report, `load_failures` and `settings`. The top level is public; the underscored modules are internal. | | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 058aa45f4..d31f74646 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -278,7 +278,8 @@ def add_task(self, task: Task) -> str: self._ref(site, f"{where}: {site.path}", pins) self._default_agents(step, cls.entity_refs or (), where) sync_context_paths(stored, steps) - self.files[path] = json.dumps(steps, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + body = {"headline_verifiers": list(task.headline_verifiers), "steps": steps} if task.headline_verifiers else steps + self.files[path] = json.dumps(body, indent=2, ensure_ascii=False, allow_nan=False) + "\n" self.tasks.append(ImportedTask(path, task.id, task.version, len(steps))) self.declared += task.requires or () return name diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index 8091f1417..554d09967 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -154,6 +154,8 @@ def digest(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Digest | inputs["published"] = published.pinned if write.requires: inputs["requires"] = list(write.requires) + if write.kind is BundleKind.TASK and write.source.entry.headline_verifiers: + inputs["headline_verifiers"] = list(write.source.entry.headline_verifiers) if self.pins_references or write.kind not in (BundleKind.TASK, BundleKind.EVAL): # A document records the versions of what it references (an env its images, a universe its # environments), so one written anew means it must be written again. A task or eval a run writes names @@ -301,6 +303,9 @@ def _changes(row: Mapping[str, Any], digest: Digest) -> list[str]: reasons.append("config changed") if before.get("requires") != after.get("requires"): reasons.append(f"requires {', '.join(after['requires'])}" if after.get("requires") else "requires nothing") + if before.get("headline_verifiers") != after.get("headline_verifiers"): + reasons.append(f"headline_verifiers {', '.join(after['headline_verifiers'])}" if after.get("headline_verifiers") + else "no headline_verifiers") if before.get("published") != after.get("published"): reasons.append(f"published as {after['published'].partition('@')[0]}" if after.get("published") else "not found published") diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 35c6a96f6..16211a6fe 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -179,7 +179,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: for write in tasks: through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps, requires=write.requires, - entity_ids=built[write.id].entity_ids).version) + entity_ids=built[write.id].entity_ids, + headline_verifiers=built[write.id].headline_verifiers).version) for write in evals: through_ledger(write, lambda: _write_eval(plan, write, written)) return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left)) @@ -455,7 +456,7 @@ def _build(write: Write, plan: Plan, written: Mapping[tuple[str, str], Materiali naming the one it reads.""" steps = write.source.config if written is None else pinned_steps(plan, write, written) return Task(id=write.id, version=None, steps=[build_step(step) for step in steps], - entity_ids=write.source.entity_ids) + entity_ids=write.source.entity_ids, headline_verifiers=write.source.entry.headline_verifiers) def pinned_steps(plan: Plan, write: Write, written: Mapping[tuple[str, str], Materialized]) -> list[dict]: diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index fcca905d8..c1e34170b 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -16,7 +16,7 @@ import unicodedata from collections import Counter, defaultdict from collections.abc import Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum from pathlib import Path from typing import Any, NoReturn @@ -65,9 +65,26 @@ def store(self) -> str: CONFIG_FILES = {BundleKind.ENV: "env.toml", BundleKind.AGENT: "agent.toml", BundleKind.ARTIFACT: "artifact.toml"} _DEFAULT_TYPE = {BundleKind.ENV: "mcp_server", BundleKind.AGENT: "a2a_agent"} _EVAL_KEYS = ("tasks", "type", "id") +_TASK_KEYS = ("steps", "headline_verifiers") _FILE_SUFFIX = {BundleKind.TASK: ".json", BundleKind.EVAL: ".toml"} +def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]: + """A task file's steps and ``headline_verifiers``: a bare array of steps, or ``{"headline_verifiers": [...], + "steps": [...]}``. Raises ValueError saying what's wrong with any other shape.""" + steps, headlines = data, None + if isinstance(data, dict): + if unknown := sorted(set(data) - set(_TASK_KEYS)): + raise ValueError(f"unknown key{'s' * (len(unknown) > 1)} {', '.join(map(repr, unknown))}; a task file's " + f"object takes {' and '.join(_TASK_KEYS)}") + steps, headlines = data.get("steps"), data.get("headline_verifiers") + if not (isinstance(steps, list) and all(isinstance(step, dict) for step in steps)) or not ( + headlines is None or isinstance(headlines, list) and all(isinstance(h, str) for h in headlines)): + raise ValueError("a task file must be a JSON array of step objects, or an object whose steps is one and whose " + "headline_verifiers lists step ids") + return steps, headlines + + @dataclass(frozen=True) class BundleEntry: kind: BundleKind @@ -77,6 +94,8 @@ class BundleEntry: type: str # declared, else the kind's default; for artifacts, inferred from the directory # The toml as written ({} when absent), a task's steps, None for skills. config: Any = field(compare=False, repr=False) + # A task file's headline_verifiers, None when it names none. + headline_verifiers: tuple[str, ...] | None = field(default=None, compare=False) @dataclass(frozen=True) @@ -315,10 +334,12 @@ def _entry(self, kind: BundleKind, name: str, path: Path) -> BundleEntry: return read.get(kind, self._artifact)(name, path) def _task(self, name: str, path: Path) -> BundleEntry: - steps = self._read_json(path) - if not (isinstance(steps, list) and all(isinstance(step, dict) for step in steps)): - self._skip(path, "a task file must be a JSON array of step objects") - return self._make(BundleKind.TASK, name, path, "task", steps) + try: + steps, headlines = task_file_parts(self._read_json(path)) + except ValueError as e: + self._skip(path, str(e)) + entry = self._make(BundleKind.TASK, name, path, "task", steps) + return replace(entry, headline_verifiers=tuple(headlines) if headlines else None) def _eval(self, name: str, path: Path) -> BundleEntry: config = self._read_toml(path) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 990477072..d09056061 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -1122,7 +1122,8 @@ def _build_tasks(self, closure: set[_Key]) -> None: self.steps[key] = steps if not failed: try: - Task(id=source.entry.id, version=None, steps=steps) + Task(id=source.entry.id, version=None, steps=steps, + headline_verifiers=source.entry.headline_verifiers) except ValueError as e: self.problems.append(f"{self._path(source.entry)}: {e}") diff --git a/src/agent_env/cli/task/create.py b/src/agent_env/cli/task/create.py index 4475219da..455dc5312 100644 --- a/src/agent_env/cli/task/create.py +++ b/src/agent_env/cli/task/create.py @@ -20,27 +20,32 @@ def create(filepath: Path, task_id: str, skip_validation: bool): """Create a task from a JSON file containing step definitions. \b - The JSON file should contain a list of step dicts, e.g.: + The JSON file holds a list of step dicts, as a bundle's task file does, e.g.: [ {"id": "step-1", "type": "deploy_env", "env_id": "my-env"}, {"id": "step-2", "type": "prompt_agent", "prompt": "Hello"} ] + or an object with them as "steps" and the step ids of its reward as "headline_verifiers". """ + from agent_env.bundle.parse import task_file_parts from agent_env.task import Task click.echo(f"Reading steps from {filepath}...") with open(filepath) as f: - steps = json.load(f) - - if not isinstance(steps, list): - click.echo("Error: JSON file must contain a list of step dicts", err=True) + data = json.load(f) + try: + steps, headlines = task_file_parts(data) + except ValueError as e: + click.echo(f"Error: {e}", err=True) raise SystemExit(1) built = _build_steps(steps) - _report_preflight( - Task(id=task_id, version=None, steps=[s for _, s in built]), - skip_validation, - ) + try: + checked = Task(id=task_id, version=None, steps=[s for _, s in built], headline_verifiers=headlines) + except ValueError as e: + click.echo(f"Error: {e}", err=True) + raise SystemExit(1) + _report_preflight(checked, skip_validation) task_steps = [] for i, (kwargs, step) in enumerate(built): @@ -49,7 +54,7 @@ def create(filepath: Path, task_id: str, skip_validation: bool): task_steps.append(saved) click.echo(f"Creating task '{task_id}' with {len(task_steps)} steps...") - task = Task.put(id=task_id, steps=task_steps) + task = Task.put(id=task_id, steps=task_steps, headline_verifiers=headlines) click.echo(f"Created task: id={task.id} version={task.version} steps={len(task.steps)}") diff --git a/src/agent_env/cli/task/get.py b/src/agent_env/cli/task/get.py index 127851a2d..411860d27 100644 --- a/src/agent_env/cli/task/get.py +++ b/src/agent_env/cli/task/get.py @@ -9,9 +9,11 @@ @click.option("--id", "task_id", required=True, help="Task id") @click.option("--version", "task_version", default=None, type=int, help="Task version (defaults to latest)") def get(task_id: str, task_version: int | None): - """Get a task and output its step definitions as JSON.""" + """Get a task and output its step definitions as JSON, as task create takes them: an object with its steps and + headline_verifiers when it has them.""" from agent_env.task import Task task = Task.get(task_id, task_version) steps = task.to_dict()["steps"] - click.echo(json.dumps(steps, indent=2)) + body = {"headline_verifiers": task.headline_verifiers, "steps": steps} if task.headline_verifiers else steps + click.echo(json.dumps(body, indent=2)) diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 4de2b033d..082b09f69 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -38,7 +38,8 @@ def from_task( raise EpisodeError(f"task {task.id!r} can't run as an episode: " + "; ".join(found)) sandboxed = {name: works_in_sandbox(seeded, name) for name in outside} steps = [_in_episode(step, as_run, sandboxed, turn_timeout) for step, as_run in zip(task.steps, seeded)] - return cls(copy=Task(id=task.id, version=task.version, steps=steps), trainer_agents=tuple(outside), seed=seed) + copy = Task(id=task.id, version=task.version, steps=steps, headline_verifiers=task.headline_verifiers) + return cls(copy=copy, trainer_agents=tuple(outside), seed=seed) @property def prompts(self) -> tuple[tuple[str, str | None], ...]: diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 94379e45c..8b761d893 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -355,6 +355,8 @@ class Task: registration records (``agent-env bundle register``); None when nothing recorded it. ``entity_ids`` maps each id a bundle rewrote in the task's steps, as the task names it, to the id the bundle gave it; ``run`` puts it in ``context.metadata`` for ``run_entity_id``. None for a task stored as written. + ``headline_verifiers`` names, by step id, the grades that are the task's reward: one is the reward and the run's + pass or fail, several a multi-reward over them; None, or empty, a multi-reward over every grade. """ type: ClassVar[str] = "task" @@ -366,14 +368,31 @@ def __init__( steps: list[TaskStep] | None = None, requires: Sequence[str] | None = None, entity_ids: Mapping[str, str] | None = None, + headline_verifiers: Sequence[str] | None = None, ): self.id = id self.version = version self.steps: list[TaskStep] = steps or [] self.requires: list[str] | None = list(requires) if requires is not None else None self.entity_ids: dict[str, str] | None = dict(entity_ids) if entity_ids else None + self.headline_verifiers: list[str] | None = list(headline_verifiers) if headline_verifiers else None self._validate_dag(self.steps) self._validate_pairings(self.steps) + self._validate_headlines(self.headline_verifiers, self.steps) + + @staticmethod + def _validate_headlines(headlines: list[str] | None, steps: list[TaskStep]) -> None: + """Refuse a headline that isn't a step id of this task, or that is named twice. Whether the step it names is a + verifier is the task lint's to say.""" + if headlines is None: + return + ids = {step.id for step in steps} + unknown = [h for h in headlines if not isinstance(h, str) or h not in ids] + if unknown: + raise ValueError(f"headline_verifiers names no step in this task: {unknown!r} (its steps: {sorted(ids)})") + if len(set(headlines)) != len(headlines): + twice = sorted({h for h in headlines if headlines.count(h) > 1}) + raise ValueError(f"headline_verifiers names {twice!r} more than once") @staticmethod def _validate_pairings(steps: list[TaskStep]) -> None: @@ -446,6 +465,8 @@ def to_dict(self) -> dict: data["requires"] = list(self.requires) if self.entity_ids: data["entity_ids"] = dict(self.entity_ids) + if self.headline_verifiers: + data["headline_verifiers"] = list(self.headline_verifiers) return data @classmethod @@ -473,6 +494,7 @@ def from_dict(cls, data: dict) -> "Task": steps=steps, requires=data["requires"] if isinstance(data.get("requires"), list) else None, entity_ids=data["entity_ids"] if isinstance(data.get("entity_ids"), dict) else None, + headline_verifiers=data["headline_verifiers"] if isinstance(data.get("headline_verifiers"), list) else None, ) async def run( diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 6df3a8a36..22b58e4ed 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -163,6 +163,22 @@ def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bund assert [(result["criterion"], result["result"]) for result in results] == [("greets", False), ("check passes", False)] +def test_a_task_file_naming_its_headline_verifiers_runs_and_its_stored_task_keeps_them(bundle, quiet_logs): + steps = json.loads((bundle / "tasks/hello.json").read_text()) + (bundle / "tasks/hello.json").write_text(json.dumps({"headline_verifiers": ["hello"], "steps": steps})) + + assert "tasks/hello.json v1: passed (hello: 1)" in _run(bundle) + assert "tasks/hello.json: v1, unchanged" in _run(bundle) + with namespace_routing(): + assert Task.get("@local/~/my-hello/hello").headline_verifiers == ["hello"] + + (bundle / "tasks/hello.json").write_text(json.dumps(steps)) + assert "tasks/hello.json: v2 (no headline_verifiers)" in _run(bundle) + with namespace_routing(): + assert Task.get("@local/~/my-hello/hello").headline_verifiers is None + assert Task.get("@local/~/my-hello/hello", 1).headline_verifiers == ["hello"] + + @skip_without_gnu_stat() def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs): steps = json.loads((bundle / "tasks/hello.json").read_text()) diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index b46903ddc..6b153c2d6 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -14,7 +14,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.artifact import FileArtifact from agent_env.artifact.artifacts.skill import SkillArtifact -from agent_env.bundle import BundleError, parse_bundle +from agent_env.bundle import BundleError, BundleKind, parse_bundle from agent_env.bundle import importer as importer_module from agent_env.bundle.importer import ImportedEval, import_eval, import_task, task_images from agent_env.bundle.installed import InstalledBundle @@ -101,6 +101,18 @@ def test_a_task_imports_as_its_stored_steps_naming_each_entity_it_uses_by_the_fo assert imported.pinned == () +def test_a_task_naming_headline_verifiers_imports_as_the_object_form_its_folder_reads_back(store): + Task.put(id="triage", steps=Task.get("triage").steps, headline_verifiers=["judge"]) + + root = import_task("triage", store / "triage").root + + written = json.loads((root / "tasks/triage.json").read_text()) + assert (written["headline_verifiers"], [step["id"] for step in written["steps"]]) == ( + ["judge"], [step["id"] for step in STEPS]) + (task,) = [entry for entry in parse_bundle(root).entries if entry.kind is BundleKind.TASK] + assert task.headline_verifiers == ("judge",) + + def test_an_id_a_step_of_the_task_writes_stays_unpinned_and_unchanged(store): steps = _steps(import_task("triage", store / "triage").root, "triage") diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 3e61edd24..98d78f37d 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -377,6 +377,42 @@ def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_d assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"} +_GRADED = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, + {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}, + {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]}] + + +def _task_file(root, **fields): + (root / "tasks/t.json").write_text(json.dumps({**fields, "steps": _GRADED} if fields else _GRADED)) + + +def test_a_task_files_headline_verifiers_are_written_and_changing_them_writes_a_new_version(bundle_dir): + _task_file(bundle_dir) + bare = _summary(_run(bundle_dir))[f"{ROOT}/t"] + _task_file(bundle_dir, headline_verifiers=[]) + assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0], True, ()) + + _task_file(bundle_dir, headline_verifiers=["final"]) + assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0] + 1, False, ("headline_verifiers final",)) + assert Task.get(f"{ROOT}/t").headline_verifiers == ["final"] + assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0] + 1, True, ()) + + _task_file(bundle_dir, headline_verifiers=["tests", "final"]) + assert _summary(_run(bundle_dir))[f"{ROOT}/t"][1:] == (False, ("headline_verifiers tests, final",)) + assert Task.get(f"{ROOT}/t").headline_verifiers == ["tests", "final"] + + +@_RUN_OR_DRY_RUN +def test_a_headline_naming_no_step_is_refused_before_anything_is_written(bundle_dir, dry_run): + _task_file(bundle_dir, headline_verifiers=["finl"]) + + (problem,) = _problems(lambda: _run(bundle_dir, dry_run=dry_run)) + + assert problem.startswith("tasks/t.json: headline_verifiers names no step in this task: ['finl']") + with pytest.raises(NotFoundError): + Task.get(f"{ROOT}/t") + + @pytest.mark.parametrize(("toml", "problem"), [ ('image = "claude-image"\nowner = "me"\n', "agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, metadata, type and id"), diff --git a/tst/unit/bundle/parse_test.py b/tst/unit/bundle/parse_test.py index cd925e650..5fe85ab29 100644 --- a/tst/unit/bundle/parse_test.py +++ b/tst/unit/bundle/parse_test.py @@ -1,5 +1,6 @@ """Reading a bundle folder into its entities: ids, kinds, artifact types and every folder rule.""" +import json import os import sys import unicodedata @@ -12,6 +13,9 @@ from agent_env.bundle import _fs from agent_env.store.ids import LOCAL_PREFIX, MAX_AUTHORED_LOCAL_ID_BYTES +TASK_SHAPE = ("a task file must be a JSON array of step objects, or an object whose steps is one and whose " + "headline_verifiers lists step ids") + def build(root: Path, spec: dict[str, str | bytes | None]) -> Path: """Create ``root`` with files (str or bytes content) and folders (a key ending in ``/``).""" @@ -624,15 +628,34 @@ def test_an_entry_point_root_must_leave_room_for_a_name(self, tmp_path): class TestFiles: def test_task_must_be_an_array_of_objects(self, tmp_path): - root = build(tmp_path / "b", {"tasks/a.json": '{"steps": []}', "tasks/b.json": "[1]"}) - assert problems(root) == ( - "tasks/a.json: a task file must be a JSON array of step objects", - "tasks/b.json: a task file must be a JSON array of step objects", - ) + root = build(tmp_path / "b", {"tasks/a.json": '{"steps": [1]}', "tasks/b.json": "[1]"}) + assert problems(root) == (f"tasks/a.json: {TASK_SHAPE}", f"tasks/b.json: {TASK_SHAPE}") def test_null_is_not_a_task(self, tmp_path): root = build(tmp_path / "b", {"tasks/t.json": "null"}) - assert problems(root) == ("tasks/t.json: a task file must be a JSON array of step objects",) + assert problems(root) == (f"tasks/t.json: {TASK_SHAPE}",) + + def test_a_task_file_may_be_an_object_naming_its_headline_verifiers(self, tmp_path): + steps = [{"id": "final", "type": "aggregate_verifiers"}] + root = build(tmp_path / "b", { + "tasks/named.json": json.dumps({"headline_verifiers": ["final"], "steps": steps}), + "tasks/none.json": json.dumps({"headline_verifiers": [], "steps": steps}), + "tasks/bare.json": json.dumps(steps), + }) + + entries = {entry.name: entry for entry in parse_bundle(root).entries} + + assert (entries["named"].config, entries["named"].headline_verifiers) == (steps, ("final",)) + assert (entries["none"].config, entries["none"].headline_verifiers) == (steps, None) + assert (entries["bare"].config, entries["bare"].headline_verifiers) == (steps, None) + + def test_a_task_files_object_takes_only_steps_and_a_list_of_headline_verifiers(self, tmp_path): + root = build(tmp_path / "b", {"tasks/a.json": '{"steps": [], "headline": ["final"]}', + "tasks/b.json": '{"steps": [], "headline_verifiers": "final"}'}) + assert problems(root) == ( + "tasks/a.json: unknown key 'headline'; a task file's object takes steps and headline_verifiers", + f"tasks/b.json: {TASK_SHAPE}", + ) @pytest.mark.parametrize("rel, content", [ ("tasks/t.json", "[" * 100_000 + "]" * 100_000), diff --git a/tst/unit/cli/test_task_create_preflight.py b/tst/unit/cli/test_task_create_preflight.py index 3aca34f04..19d336790 100644 --- a/tst/unit/cli/test_task_create_preflight.py +++ b/tst/unit/cli/test_task_create_preflight.py @@ -39,6 +39,36 @@ def store(monkeypatch): yield put_step, put_task +def _object_file(tmp_path, **fields): + path = tmp_path / "task.json" + path.write_text(json.dumps({"steps": json.loads(_task_file(tmp_path).read_text()), **fields})) + return path + + +def test_an_object_file_saves_its_headline_verifiers_with_the_task(tmp_path, store): + _, put_task = store + with patch("agent_env.artifact.Artifact.get", side_effect=NotFoundError("nope")): + result = CliRunner().invoke(create, [str(_object_file(tmp_path, headline_verifiers=["c1"])), "--id", "t", + "--skip-validation"]) + + assert result.exit_code == 0, result.output + assert put_task.call_args.kwargs["headline_verifiers"] == ["c1"] + + +@pytest.mark.parametrize(("fields", "message"), [ + ({"headline": ["c1"]}, "unknown key 'headline'; a task file's object takes steps and headline_verifiers"), + ({"headline_verifiers": ["final"]}, "headline_verifiers names no step in this task: ['final']"), +], ids=["unknown-key", "unknown-headline"]) +def test_an_object_file_that_is_refused_persists_nothing(tmp_path, store, fields, message): + put_step, put_task = store + + result = CliRunner().invoke(create, [str(_object_file(tmp_path, **fields)), "--id", "t", "--skip-validation"]) + + assert result.exit_code == 1 and message in result.output + put_step.assert_not_called() + put_task.assert_not_called() + + def _run(tmp_path, *extra): return CliRunner().invoke(create, [ str(_task_file(tmp_path)), "--id", "t", *extra, diff --git a/tst/unit/cli/test_task_get.py b/tst/unit/cli/test_task_get.py new file mode 100644 index 000000000..624a4fc7c --- /dev/null +++ b/tst/unit/cli/test_task_get.py @@ -0,0 +1,23 @@ +"""`agent-env task get` prints a task as `task create` takes it.""" + +import json + +import pytest +from click.testing import CliRunner + +from agent_env.cli.task.get import get +from agent_env.task import Task + +STEPS = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, + {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}] + + +@pytest.mark.parametrize("headlines", [None, ["tests"]], ids=["bare", "headlines"]) +def test_a_task_prints_as_its_steps_or_with_its_headline_verifiers(monkeypatch, headlines): + task = Task.from_dict({"id": "t", "version": 1, "steps": STEPS, "headline_verifiers": headlines}) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, task_id, version=None: task)) + + printed = json.loads(CliRunner().invoke(get, ["--id", "t"]).output) + + steps = [step.to_dict() for step in task.steps] + assert printed == ({"headline_verifiers": headlines, "steps": steps} if headlines else steps) diff --git a/tst/unit/task/episode/spec_test.py b/tst/unit/task/episode/spec_test.py index b23aff047..c7c590562 100644 --- a/tst/unit/task/episode/spec_test.py +++ b/tst/unit/task/episode/spec_test.py @@ -89,6 +89,12 @@ def test_a_task_prompting_no_agent_has_nothing_to_play(): assert _refusal(_task(ENV, VERIFY)) == "task 'dock' prompts no agent" +def test_the_copy_keeps_the_tasks_headline_verifiers(): + task = Task.from_dict({**_portsim().to_dict(), "headline_verifiers": ["verify"]}) + + assert EpisodeSpec.from_task(task)._copy.headline_verifiers == ["verify"] + + def test_the_copy_stands_in_for_the_outside_agents_steps_and_keeps_everything_else(): task = _portsim(BOB, ASK_BOB) copy = EpisodeSpec.from_task(task, trainer_agents=["default-agent"])._copy diff --git a/tst/unit/task/task_document_test.py b/tst/unit/task/task_document_test.py index 605aaa770..ad8f5e422 100644 --- a/tst/unit/task/task_document_test.py +++ b/tst/unit/task/task_document_test.py @@ -64,3 +64,52 @@ def test_a_record_that_isnt_a_requirement_is_named_and_never_offered(monkeypatch def test_a_record_that_isnt_a_list_is_read_as_none(): assert Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [], "requires": "agentenv-acme"}).requires \ is None + + +_GRADED = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, + {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}, + {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]}] + + +def _graded(**fields) -> dict: + return {"id": "t", "type": "task", "version": 1, "steps": [dict(step) for step in _GRADED], **fields} + + +@pytest.mark.parametrize("headlines", [["final"], ["tests", "final"], ["final", "tests"]]) +def test_a_tasks_headline_verifiers_round_trip_in_the_order_given(headlines): + task = Task.from_dict(_graded(headline_verifiers=headlines)) + + assert task.headline_verifiers == headlines + assert Task.from_dict(task.to_dict()).to_dict()["headline_verifiers"] == headlines + + +@pytest.mark.parametrize("fields", [{}, {"headline_verifiers": []}, {"headline_verifiers": "final"}], + ids=["absent", "empty", "not-a-list"]) +def test_a_task_naming_no_headline_verifiers_records_none(fields): + task = Task.from_dict(_graded(**fields)) + + assert task.headline_verifiers is None + assert "headline_verifiers" not in task.to_dict() + + +@pytest.mark.parametrize(("headlines", "message"), [ + (["final", "verify"], "headline_verifiers names no step in this task: ['verify'] (its steps: ['box', 'final', " + "'tests'])"), + (["final", 3], "headline_verifiers names no step in this task: [3]"), + (["final", "tests", "final"], "headline_verifiers names ['final'] more than once"), +], ids=["unknown", "not-a-string", "twice"]) +def test_a_headline_naming_no_step_of_the_task_or_named_twice_is_refused(headlines, message): + with pytest.raises(ValueError) as refused: + Task.from_dict(_graded(headline_verifiers=headlines)) + + assert str(refused.value).startswith(message) + + +def test_each_version_of_a_task_keeps_its_own_headline_verifiers(local_stores): + steps = Task.from_dict(_graded()).steps + + first = Task.put(id="t", steps=steps, headline_verifiers=["final"]) + second = Task.put(id="t", steps=steps, headline_verifiers=["tests", "final"]) + + assert Task.get("t", first.version).headline_verifiers == ["final"] + assert (Task.get("t").version, Task.get("t").headline_verifiers) == (second.version, ["tests", "final"]) From 2cdd688182f1f2485150b6fbd61a5b1e0c2f36a1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 23:21:00 -0800 Subject: [PATCH 2/2] refactor(bundle): one writer for a task file's object form, beside its reader bundle init and task get write the object form through task_file_body, which task_file_parts reads back, and the headline tests share one graded task's steps. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/importer.py | 5 +++-- src/agent_env/bundle/parse.py | 8 +++++++- src/agent_env/cli/task/get.py | 5 ++--- tst/unit/bundle/_support.py | 6 ++++++ tst/unit/bundle/materialize_test.py | 11 ++++------- tst/unit/task/task_document_test.py | 8 ++------ 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index d31f74646..916dab877 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -50,6 +50,7 @@ parse_bundle, root_problems, segment_problem, + task_file_body, ) from agent_env.bundle.plan import STORE_GETTERS, check_bundle from agent_env.bundle.register import REGISTRATIONS @@ -278,8 +279,8 @@ def add_task(self, task: Task) -> str: self._ref(site, f"{where}: {site.path}", pins) self._default_agents(step, cls.entity_refs or (), where) sync_context_paths(stored, steps) - body = {"headline_verifiers": list(task.headline_verifiers), "steps": steps} if task.headline_verifiers else steps - self.files[path] = json.dumps(body, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + self.files[path] = json.dumps(task_file_body(steps, task.headline_verifiers), indent=2, ensure_ascii=False, + allow_nan=False) + "\n" self.tasks.append(ImportedTask(path, task.id, task.version, len(steps))) self.declared += task.requires or () return name diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index c1e34170b..e1e8d9695 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -19,7 +19,7 @@ from dataclasses import dataclass, field, replace from enum import Enum from pathlib import Path -from typing import Any, NoReturn +from typing import Any, NoReturn, Sequence from packaging.requirements import InvalidRequirement, Requirement @@ -85,6 +85,12 @@ def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]: return steps, headlines +def task_file_body(steps: list[dict], headlines: Sequence[str] | None) -> list[dict] | dict[str, Any]: + """A task file holding ``steps`` and ``headlines``, as ``task_file_parts`` reads it back: the bare array when there + are no headlines.""" + return {"headline_verifiers": list(headlines), "steps": steps} if headlines else steps + + @dataclass(frozen=True) class BundleEntry: kind: BundleKind diff --git a/src/agent_env/cli/task/get.py b/src/agent_env/cli/task/get.py index 411860d27..1a896fbad 100644 --- a/src/agent_env/cli/task/get.py +++ b/src/agent_env/cli/task/get.py @@ -11,9 +11,8 @@ def get(task_id: str, task_version: int | None): """Get a task and output its step definitions as JSON, as task create takes them: an object with its steps and headline_verifiers when it has them.""" + from agent_env.bundle.parse import task_file_body from agent_env.task import Task task = Task.get(task_id, task_version) - steps = task.to_dict()["steps"] - body = {"headline_verifiers": task.headline_verifiers, "steps": steps} if task.headline_verifiers else steps - click.echo(json.dumps(body, indent=2)) + click.echo(json.dumps(task_file_body(task.to_dict()["steps"], task.headline_verifiers), indent=2)) diff --git a/tst/unit/bundle/_support.py b/tst/unit/bundle/_support.py index c39c9896f..ae7e13a6c 100644 --- a/tst/unit/bundle/_support.py +++ b/tst/unit/bundle/_support.py @@ -22,6 +22,12 @@ CLAUDE_CODE_IMAGE = next(image for image in AGENT_IMAGES if image.name == "agentenv-claude-code") # The digest the registry fixture pins every tag to. DIGEST = "sha256:" + "a" * 64 +# A task's steps with two verifiers, the second an aggregate of the first, for headline_verifiers to name. +GRADED_STEPS = ( + {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, + {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}, + {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]}, +) def layout(root, files): diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 98d78f37d..6eac77532 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -38,7 +38,8 @@ from agent_env.task_step.task_step import TaskStep from agent_env.utils.docker.context import BuildContext from agent_env.env.published_images import EXAMPLE_IMAGES -from tst.unit.bundle._support import CLAUDE_CODE_IMAGE, DIGEST, RefusingStore, layout, local_builds, local_store, plan_of, shipped_agent +from tst.unit.bundle._support import (CLAUDE_CODE_IMAGE, DIGEST, GRADED_STEPS, RefusingStore, layout, local_builds, + local_store, plan_of, shipped_agent) ROOT = "@local/~/triage" LAYOUT = { @@ -377,13 +378,9 @@ def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_d assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"} -_GRADED = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, - {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}, - {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]}] - - def _task_file(root, **fields): - (root / "tasks/t.json").write_text(json.dumps({**fields, "steps": _GRADED} if fields else _GRADED)) + steps = list(GRADED_STEPS) + (root / "tasks/t.json").write_text(json.dumps({**fields, "steps": steps} if fields else steps)) def test_a_task_files_headline_verifiers_are_written_and_changing_them_writes_a_new_version(bundle_dir): diff --git a/tst/unit/task/task_document_test.py b/tst/unit/task/task_document_test.py index ad8f5e422..2e867d24d 100644 --- a/tst/unit/task/task_document_test.py +++ b/tst/unit/task/task_document_test.py @@ -3,6 +3,7 @@ from agent_env.plugins import _requirements from agent_env.task import Task from agent_env.task.task import UnknownStepTypeError +from tst.unit.bundle._support import GRADED_STEPS def test_a_stored_task_document_with_an_unread_project_id_still_loads(): @@ -66,13 +67,8 @@ def test_a_record_that_isnt_a_list_is_read_as_none(): is None -_GRADED = [{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}, - {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []}, - {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]}] - - def _graded(**fields) -> dict: - return {"id": "t", "type": "task", "version": 1, "steps": [dict(step) for step in _GRADED], **fields} + return {"id": "t", "type": "task", "version": 1, "steps": [dict(step) for step in GRADED_STEPS], **fields} @pytest.mark.parametrize("headlines", [["final"], ["tests", "final"], ["final", "tests"]])