From 64b74a3637b3734c84b994c274c1a4e0e77a82bc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 17 Nov 2026 19:46:58 -0800 Subject: [PATCH 01/10] feat(verifiers): a VerifierTaskStep base writes each verifier's named grade A verifier on the new base implements verify() and returns a Verification; the base scores it and writes context.metadata["grades"][] beside the legacy verifications entries, which stay byte for byte as before. An exception from verify() is a failed grade, raised again; raise_after keeps a recorded verdict graded and still fails the step; a skip carries its reason. agent_prompt_response_verifier moves onto the base as the pilot. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_plugin_api.py | 4 + AGENTS.md | 2 +- .../task_steps/verifiers/__init__.py | 11 +- .../agent_prompt_response_verifier.py | 63 ++--- .../task_steps/verifiers/verifier.py | 240 +++++++++++++++++ tst/unit/task/episode/run_test.py | 18 ++ .../test_agent_prompt_response_verifier.py | 113 ++++++++ .../task_step/verifiers/test_verifier_base.py | 247 ++++++++++++++++++ 8 files changed, 651 insertions(+), 47 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/verifiers/verifier.py create mode 100644 tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py create mode 100644 tst/unit/task_step/verifiers/test_verifier_base.py diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index 464f49c85..d83976956 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -27,6 +27,7 @@ BASES = ( "agent_env.env.env.Env", "agent_env.task_step.task_step.TaskStep", + "agent_env.task_step.task_steps.verifiers.verifier.VerifierTaskStep", "agent_env.artifact.artifact.Artifact", "agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider", "agent_env.providers.env_state.env_state_provider.EnvStateProvider", @@ -47,6 +48,9 @@ "agent_env.runner.store.ensure_indexes", "agent_env.runner.store.insert_run", "agent_env.runner.store.get_run", "agent_env.runner.store.list_runs", "agent_env.runner.store.count_runs", "agent_env.runner.store.active_runs", "agent_env.runner.store.set_running", "agent_env.runner.store.mark_terminal", + # What a verifier step returns and scores its rows with. + "agent_env.task_step.task_steps.verifiers.Verification", "agent_env.task_step.task_steps.verifiers.VerifierStatus", + "agent_env.task_step.task_steps.verifiers.ScoreAggregator", "agent_env.task_step.task_steps.verifiers.aggregate_score", # What a verify script is handed of the run it grades. "agent_env.task_step.task_steps.verifiers.Run", "agent_env.task_step.task_steps.verifiers.Turn", "agent_env.task_step.task_steps.verifiers.Message", "agent_env.task_step.task_steps.verifiers.Firing", diff --git a/AGENTS.md b/AGENTS.md index a2d1a8c1e..850794fb5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). A verifier on the `VerifierTaskStep` base (`verifiers/verifier.py`; `agent_prompt_response_verifier` so far, the other built-ins still write their entries themselves) implements `verify` and returns a `Verification`; the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings` with `scoring_rules`) beside the legacy `verifications` entries every reader still takes, and an exception from `verify` is a failed grade raised again. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index 3edb4d3f1..ba6f4cd82 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -1,9 +1,9 @@ -"""Verifier task steps: the steps that score a run and record a row under -``context.metadata["verifications"]``, plus the helpers only they use (score -aggregation, judge output formats, trajectory compaction and frame selection for -the LLM judge).""" +"""Verifier task steps: the steps that grade a run, the ``VerifierTaskStep`` base they move onto, which writes each one's +named grade under ``context.metadata["grades"]`` beside its ``verifications`` entry, and the helpers only they use +(score aggregation, judge output formats, trajectory compaction and frame selection for the LLM judge).""" from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierStatus, VerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_context import EnvClock, Firing, Message, Run, RunRecordUnavailable, Turn from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier @@ -25,6 +25,9 @@ "RunRecordUnavailable", "ScoreAggregator", "Turn", + "Verification", + "VerifierStatus", + "VerifierTaskStep", "VerifySandboxTaskStep", "aggregate_score", "is_rubrics_verifier", diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 7d868f330..d27084736 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -10,9 +10,10 @@ response_regex_present) raise ValueError. A verifier that can't actually verify anything is a misconfiguration, not a runtime decision. -Result rows merge `{**criterion, **outcome}` and aggregated score is written -to `context.metadata["verifications"][verifier_id]`, matching the shape used -by `VerifySandboxTaskStep` and `RubricsVerifierTaskStep`. +Result rows merge `{**criterion, **outcome}`. On the `VerifierTaskStep` base, which +aggregates them into the step's grade and writes the +`context.metadata["verifications"][verifier_id]` entry, in the shape used by +`VerifySandboxTaskStep` and `RubricsVerifierTaskStep`. """ from __future__ import annotations @@ -22,8 +23,9 @@ from typing import ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency -from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ } -class AgentPromptResponseVerifierTaskStep(TaskStep): +class AgentPromptResponseVerifierTaskStep(VerifierTaskStep): type: ClassVar[str] = "agent_prompt_response_verifier" entity_refs = () @@ -43,18 +45,17 @@ def __init__( version: Optional[int], prompt_id: str, criteria: Optional[list[dict]] = None, - score_aggregator: Optional[ScoreAggregator] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.prompt_id = prompt_id self.criteria = list(criteria or []) - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS - self.verifier_id = verifier_id or uuid.uuid4().hex def to_dict(self) -> dict: base = super().to_dict() @@ -66,16 +67,9 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> "AgentPromptResponseVerifierTaskStep": - raw_agg = data.get("score_aggregator") - return cls( - **cls._base_from_dict(data), - prompt_id=data["prompt_id"], - criteria=data.get("criteria"), - score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, - verifier_id=data.get("verifier_id"), - ) + return cls(**cls._base_from_dict(data), prompt_id=data["prompt_id"], criteria=data.get("criteria")) - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: prompt_response = next( (pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None, @@ -87,16 +81,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning( f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}" ) - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - "results": [{ - "id": "prompt_error", - "score": 0, - "result": False, - "message": f"Skipped: prompt had error_type={prompt_response.error_type}", - }], + return Verification(criteria=[{ + "id": "prompt_error", "score": 0, - } - return context + "result": False, + "message": f"Skipped: prompt had error_type={prompt_response.error_type}", + }], score=0) response = prompt_response.response or "" @@ -116,18 +106,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "result": bool(outcome["passed"]), "justification": outcome["justification"], }) - - score = aggregate_score(results, self.score_aggregator) - - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - "results": results, - "score": score, - } - logger.info( - f"Verification '{self.verifier_id}': " - f"{len(results)} criteria evaluated, score={score}" - ) - return context + return Verification(criteria=results) def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict: rtype = criterion["type"] diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py new file mode 100644 index 000000000..5ac06528b --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -0,0 +1,240 @@ +"""The base every verifier step subclasses. A verifier implements ``verify``; the base scores what it returns and writes +the step's named grade to ``context.metadata["grades"][]``, beside the ``verifications`` entries today's +readers take.""" + +from __future__ import annotations + +import functools +import hashlib +import json +import logging +import uuid +from abc import abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from importlib.metadata import packages_distributions +from typing import Any, ClassVar, Optional + +from agent_env.config import get_config +from agent_env.plugins import _requirements +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +logger = logging.getLogger(__name__) + +GRADES_KEY = "grades" +VERIFICATIONS_KEY = "verifications" +# The version of the rules the base applies to every verifier's rows and pass rule; a change that can move a grade +# bumps it, and every grade records it in its verifier_settings. +SCORING_RULES = 1 +DEFAULT_MIN_PASSING_SCORE = 1.0 +OUTPUT_INLINE_BYTES = 16 * 1024 +SETTING_INLINE_BYTES = 1024 +# The step's fields that name or schedule it rather than say how it grades, left out of its verifier_settings. +_NOT_SETTINGS = ("id", "type", "version", "depends_on", "fail_task_on_error", "retry_config", "verifier_id") +_FRAMEWORK = "agentenv-framework" + + +class VerifierStatus(str, Enum): + GRADED = "graded" + SKIPPED = "skipped" + FAILED = "failed" + + +@dataclass +class Verification: + """What one run of a verifier found. + + ``criteria``: a row per check, read for ``id``, ``score``, ``result``, ``weight`` and ``skipped``, any other keys + kept. ``score``: the verifier's own score when it isn't the step's aggregate of the rows, which it then replaces. + ``output``: anything else worth keeping, such as a judge's justification or a test's log. ``legacy_entries``: the + ``verifications`` entries to write, by key, when they aren't ``{verifier_id: {"results", "score"}}``. ``skipped``: + why the verifier judged nothing. ``raise_after``: an exception the step raises once both are written, for a verdict + that also fails its step.""" + + criteria: list[dict[str, Any]] = field(default_factory=list) + score: Optional[float] = None + output: Any = None + legacy_entries: Optional[dict[str, dict[str, Any]]] = None + skipped: Optional[str] = None + raise_after: Optional[BaseException] = None + + +class VerifierTaskStep(TaskStep): + """A step that grades a run. ``execute`` runs ``verify`` and writes the step's grade, and the legacy entries after + it; an exception from ``verify`` is a failed grade, raised again. Subclasses implement ``verify``, not ``execute``.""" + + DEFAULT_SCORE_AGGREGATOR: ClassVar[ScoreAggregator] = ScoreAggregator.ALL_PASS + + def __init__( + self, + id: str, + version: Optional[int], + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + retry_config: Optional[RetryConfig | dict] = None, + *, + verifier_id: Optional[str] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, + min_passing_score: Optional[float] = None, + ): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + retry_config=retry_config) + self.verifier_id = verifier_id or id + self.score_aggregator = ScoreAggregator(score_aggregator) if score_aggregator else self.DEFAULT_SCORE_AGGREGATOR + self.min_passing_score = DEFAULT_MIN_PASSING_SCORE if min_passing_score is None else float(min_passing_score) + + @abstractmethod + async def verify(self, context: TaskStepContext) -> Verification: + """Grade the run in ``context``.""" + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + try: + verification = await self.verify(context) + except Exception as exc: + self._write_grade(context, self._failed_grade(context, exc)) + raise + self._write_grade(context, self._grade(context, verification)) + entries = verification.legacy_entries + if entries is None: + entries = {self.verifier_id: self._legacy_entry(verification)} + context.metadata.setdefault(VERIFICATIONS_KEY, {}).update(entries) + if verification.raise_after is not None: + raise verification.raise_after + return context + + def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: + """What this step grades with in ``context``'s run: its own fields, one over ``SETTING_INLINE_BYTES`` kept as its + sha256 and size. A subclass adds what the run changes or decides, such as the ``step_params`` it applies or a + judge's model. Pure, since it is also read at run start.""" + return {k: _compact(v) for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} + + def config_problems(self) -> list[str]: + problems = super().config_problems() + if not 0.0 < self.min_passing_score <= 1.0: + problems.append(f"min_passing_score must be above 0 and at most 1, not {self.min_passing_score}") + return problems + + def to_dict(self) -> dict[str, Any]: + base = super().to_dict() + if self.verifier_id != self.id: + base["verifier_id"] = self.verifier_id + if self.score_aggregator is not self.DEFAULT_SCORE_AGGREGATOR: + base["score_aggregator"] = self.score_aggregator.value + if self.min_passing_score != DEFAULT_MIN_PASSING_SCORE: + base["min_passing_score"] = self.min_passing_score + return base + + @classmethod + def _base_from_dict(cls, data: dict) -> dict: + return { + **super()._base_from_dict(data), + "verifier_id": data.get("verifier_id"), + "score_aggregator": data.get("score_aggregator"), + "min_passing_score": data.get("min_passing_score"), + } + + def _grade(self, context: TaskStepContext, verification: Verification) -> dict[str, Any]: + if verification.skipped is not None: + status, score = VerifierStatus.SKIPPED, 0.0 + else: + status, score = VerifierStatus.GRADED, float(self._score(verification)) + grade = self._grade_fields(context, status, score, verification.criteria) + grade["verifier_skip_reasoning"] = verification.skipped + grade.update(self._output(context, verification.output)) + return grade + + def _failed_grade(self, context: TaskStepContext, exc: Exception) -> dict[str, Any]: + grade = self._grade_fields(context, VerifierStatus.FAILED, 0.0, []) + grade["verifier_error_reasoning"] = f"{type(exc).__name__}: {exc}" + return grade + + def _grade_fields(self, context: TaskStepContext, status: VerifierStatus, score: float, + criteria: list[dict[str, Any]]) -> dict[str, Any]: + passed = status is VerifierStatus.GRADED and score >= self.min_passing_score + rows = [_row(i, row) for i, row in enumerate(criteria)] or [{ + "id": self.id, "score": score, "result": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, + }] + return { + "name": self.id, + "score": score, + "min_passing_score": self.min_passing_score, + "passed": passed, + "verifier_status": status.value, + "verifier_error_reasoning": None, + "verifier_skip_reasoning": None, + "verifier_output": None, + "verifier_criteria": rows, + "verifier_type": self.type, + "verifier_version": _providing_version(type(self).__module__), + "verifier_settings": {"scoring_rules": SCORING_RULES, **self.verifier_settings(context)}, + } + + def _score(self, verification: Verification) -> float: + if verification.score is not None: + return verification.score + return aggregate_score(verification.criteria, self.score_aggregator) + + def _legacy_entry(self, verification: Verification) -> dict[str, Any]: + if verification.skipped is not None: + return {"results": verification.criteria, "score": None, "skipped": verification.skipped} + return {"results": verification.criteria, "score": self._score(verification)} + + def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: + """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the + object store; inline with ``verifier_output_offload_error`` when that write fails, which never fails the step.""" + if output is None: + return {} + body = json.dumps(output).encode() + if len(body) <= OUTPUT_INLINE_BYTES: + return {"verifier_output": output} + try: + url = _store_output(context, self.id, body) + except Exception as exc: + logger.warning("verifier %s: kept its %d-byte output inline, the object store refused it: %s", + self.id, len(body), exc) + return {"verifier_output": output, "verifier_output_offload_error": f"{type(exc).__name__}: {exc}"} + return {"verifier_output_ref": {"object_url": url, "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest()}} + + def _write_grade(self, context: TaskStepContext, grade: dict[str, Any]) -> None: + context.metadata.setdefault(GRADES_KEY, {})[self.id] = grade + logger.info("verifier %s: %s, score %s, %s", self.id, grade["verifier_status"], grade["score"], + "passed" if grade["passed"] else "not passed") + + +def _row(index: int, row: dict[str, Any]) -> dict[str, Any]: + """``row`` with the keys every grade's rows carry, each default chosen so ``aggregate_score`` reads the row as it + reads the original: ``result`` is never filled in, since ALL_PASS counts only an explicit ``True``.""" + return { + **row, + "id": row.get("id", str(index)), + "score": row.get("score", 1.0 if row.get("result") else 0.0), + "result": row.get("result"), + "weight": row.get("weight", 1.0), + "skipped": row.get("skipped", False), + } + + +def _compact(value: Any) -> Any: + body = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + if len(body) <= SETTING_INLINE_BYTES: + return value + return {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)} + + +def _store_output(context: TaskStepContext, step_id: str, body: bytes) -> str: + config = get_config() + key = (f"{config.get_artifact_key_prefix()}verifier-outputs/instance_id={context.instance_id}/" + f"{step_id}-{uuid.uuid4().hex}.json") + return config.get_object_store().put(key, body, content_type="application/json") + + +@functools.cache +def _providing_version(module: str) -> Optional[str]: + """The version of the installed package that holds ``module``: agentenv-framework's for agent_env's own, else that + of the one distribution installing its top-level package; None when none does, as for a module on the path.""" + top = module.partition(".")[0] + names = [_FRAMEWORK] if top == "agent_env" else list(dict.fromkeys(packages_distributions().get(top, ()))) + return _requirements.installed(names[0]) if len(names) == 1 else None diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index a51290470..6d004c028 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -573,6 +573,24 @@ async def test_a_tolerated_verifier_that_recorded_no_score_is_missing_from_the_s assert await episode.result() == {"replies": 1.0} +@pytest.mark.asyncio +async def test_the_stored_run_holds_each_verifiers_named_grade_a_failed_one_included(teardowns): + passing = {"id": "crates", "type": "agent_prompt_response_verifier", "prompt_id": "turn", + "verifier_id": "has-crate", "criteria": [{"type": "response_contains", "needles": ["crate"]}]} + broken = {"id": "graded", "type": "agent_prompt_response_verifier", "prompt_id": "missing", + "verifier_id": "graded", "criteria": [{"type": "response_contains", "needles": ["crate"]}], + "fail_task_on_error": False} + episode = await _spec(ITEMS, ALICE, TURN, passing, broken).start() + await (await episode.next_turn()).end("Added a crate.") + + assert await episode.result() == {"crates": 1.0} + grades = Task.get_instance(episode.id).context["metadata"]["grades"] + assert {step: (g["verifier_status"], g["score"], g["passed"]) for step, g in grades.items()} == { + "crates": ("graded", 1.0, True), "graded": ("failed", 0.0, False)} + assert grades["graded"]["verifier_error_reasoning"].startswith("StepContextError: PromptResponse") + assert grades["crates"]["verifier_criteria"][0]["result"] is True + + BREAKS_WHILE_THE_TURN_IS_OPEN = ( ITEMS, {**ALICE, "depends_on": ["items"]}, {**TURN, "depends_on": ["alice"]}, {"id": "later", "type": "episode_test_gate", "depends_on": ["items"]}, diff --git a/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py new file mode 100644 index 000000000..cbf223433 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py @@ -0,0 +1,113 @@ +"""agent_prompt_response_verifier on the VerifierTaskStep base: its legacy entry is byte for byte what it wrote before +the move (the strings below were recorded from main at 810e27760), and it now writes a named grade beside it.""" +import asyncio +import json + +import pytest + +from agent_env.plugins import _requirements +from agent_env.task.task import recorded_scores +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep + +CRITERIA = [ + {"id": "says-done", "type": "response_contains", "needles": ["done", "ok"]}, + {"type": "response_regex_present", "pattern": "^Total: \\d+$", "flags": ["MULTILINE"], "weight": 2}, + {"id": "names-file", "type": "response_contains", "needles": ["report.csv"], "weight": 0.5}, +] + +ROWS = (r'''[{"id": "says-done", "type": "response_contains", "needles": ["done", "ok"], "criterion_index": 0, ''' + r'''"score": 1.0, "result": true, "justification": "all needles present"}, {"type": "response_regex_present", ''' + r'''"pattern": "^Total: \\d+$", "flags": ["MULTILINE"], "weight": 2, "criterion_index": 1, "score": 1.0, ''' + r'''"result": true, "justification": "pattern matched"}, {"id": "names-file", "type": "response_contains", ''' + r'''"needles": ["report.csv"], "weight": 0.5, "criterion_index": 2, "score": 0.0, "result": false, ''' + r'''"justification": "missing: ['report.csv']"}]''') +BEFORE = { + "all_pass": '{"results": ' + ROWS + ', "score": 0.0}', + "weighted_average": '{"results": ' + ROWS + ', "score": 0.8571428571428571}', + "prompt_error": ('{"results": [{"id": "prompt_error", "score": 0, "result": false, "message": "Skipped: prompt ' + 'had error_type=agent_error"}], "score": 0}'), +} +RESPONSES = {"all_pass": "done, ok\nTotal: 12\n", "weighted_average": "done ok\nTotal: 3"} + + +def _step(**kw) -> AgentPromptResponseVerifierTaskStep: + return AgentPromptResponseVerifierTaskStep(id="check", version=None, prompt_id="p1", criteria=CRITERIA, + verifier_id="vid", **kw) + + +def _run(step, response="", error_type=None) -> TaskStepContext: + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response=response, error_type=error_type)]) + asyncio.run(step.execute(context)) + return context + + +@pytest.mark.parametrize("aggregator", ["all_pass", "weighted_average"]) +def test_the_legacy_entry_is_unchanged_and_the_grade_carries_its_score(aggregator): + step = _step(score_aggregator=aggregator) + context = _run(step, RESPONSES[aggregator]) + + assert json.dumps(context.metadata["verifications"]["vid"]) == BEFORE[aggregator] + grade = context.metadata["grades"]["check"] + assert grade["score"] == json.loads(BEFORE[aggregator])["score"] + assert (grade["verifier_status"], grade["verifier_type"]) == ("graded", "agent_prompt_response_verifier") + assert grade["verifier_version"] == _requirements.installed("agentenv-framework") + assert [row["id"] for row in grade["verifier_criteria"]] == ["says-done", "1", "names-file"] + assert recorded_scores([step], context) == {"check": json.loads(BEFORE[aggregator])["score"]} + + +def test_a_prompt_error_is_still_a_graded_zero(): + context = _run(_step(), error_type="agent_error") + + assert json.dumps(context.metadata["verifications"]["vid"]) == BEFORE["prompt_error"] + grade = context.metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("graded", 0.0, False) + + +def test_an_unknown_criterion_type_is_a_failed_grade_and_still_raises(): + step = AgentPromptResponseVerifierTaskStep(id="check", version=None, prompt_id="p1", + criteria=[{"type": "response_sentiment"}], verifier_id="vid") + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response="hi")]) + + with pytest.raises(ValueError, match="unknown type 'response_sentiment'"): + asyncio.run(step.execute(context)) + + assert "verifications" not in context.metadata + assert context.metadata["grades"]["check"]["verifier_status"] == "failed" + assert context.metadata["grades"]["check"]["verifier_error_reasoning"].startswith("ValueError: criterion #0") + + +def test_a_missing_prompt_response_is_a_failed_grade_and_still_raises(): + context = TaskStepContext() + + with pytest.raises(StepContextError): + asyncio.run(_step().execute(context)) + + assert context.metadata["grades"]["check"]["verifier_status"] == "failed" + + +def test_its_stored_form_is_unchanged(): + before = json.loads(r'''{"criteria": [{"id": "says-done", "needles": ["done", "ok"], "type": "response_contains"}, + {"flags": ["MULTILINE"], "pattern": "^Total: \\d+$", "type": "response_regex_present", "weight": 2}, + {"id": "names-file", "needles": ["report.csv"], "type": "response_contains", "weight": 0.5}], + "fail_task_on_error": true, "id": "check", "prompt_id": "p1", "score_aggregator": "all_pass", + "type": "agent_prompt_response_verifier", "verifier_id": "vid", "version": null}''') + + assert _step().to_dict() == before + assert AgentPromptResponseVerifierTaskStep.from_dict(before).to_dict() == before + + +def test_its_settings_are_its_own_fields(): + settings = _run(_step(), RESPONSES["all_pass"]).metadata["grades"]["check"]["verifier_settings"] + + assert settings == {"scoring_rules": 1, "prompt_id": "p1", "criteria": CRITERIA, "score_aggregator": "all_pass"} + + +def test_a_min_passing_score_round_trips_and_sets_the_pass_rule(): + step = AgentPromptResponseVerifierTaskStep.from_dict({**_step().to_dict(), "score_aggregator": "weighted_average", + "min_passing_score": 0.8}) + + grade = _run(step, RESPONSES["weighted_average"]).metadata["grades"]["check"] + + assert step.to_dict()["min_passing_score"] == 0.8 + assert (grade["min_passing_score"], grade["passed"]) == (0.8, True) diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py new file mode 100644 index 000000000..6691260a8 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -0,0 +1,247 @@ +"""VerifierTaskStep: the named grade it writes beside a verifier's legacy entries, for each way verify can end.""" +import asyncio +import copy +import hashlib +import json +from types import SimpleNamespace +from typing import ClassVar + +import pytest + +from agent_env.plugins import _registration +from agent_env.task.step_journal import _apply_forward +from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError +from agent_env.task_step.context_ops import build_context_update_ops +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.verifiers import ScoreAggregator, Verification, VerifierTaskStep, aggregate_score +from agent_env.task_step.task_steps.verifiers import verifier as verifier_module + + +class _Toy(VerifierTaskStep): + type: ClassVar[str] = "toy_verifier" + + def __init__(self, id="check", version=None, outcome=None, note="", **base): + super().__init__(id, version, **base) + self.outcome = outcome + self.note = note + + def to_dict(self): + return {**super().to_dict(), "note": self.note} + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data), note=data.get("note", "")) + + async def verify(self, context): + if isinstance(self.outcome, BaseException): + raise self.outcome + return self.outcome + + +def _run(step: VerifierTaskStep, context: TaskStepContext | None = None) -> TaskStepContext: + context = context or TaskStepContext() + asyncio.run(step.execute(context)) + return context + + +ROWS = [{"id": "a", "score": 1.0, "result": True, "why": "found it"}, {"id": "b", "score": 0.0, "result": False}] + + +def test_a_graded_verification_writes_its_named_grade_and_the_legacy_entry(): + metadata = _run(_Toy(outcome=Verification(criteria=ROWS), verifier_id="vid", note="n")).metadata + + assert metadata["verifications"] == {"vid": {"results": ROWS, "score": 0.0}} + grade = metadata["grades"]["check"] + assert grade == { + "name": "check", "score": 0.0, "min_passing_score": 1.0, "passed": False, "verifier_status": "graded", + "verifier_error_reasoning": None, "verifier_skip_reasoning": None, "verifier_output": None, + "verifier_criteria": [{**ROWS[0], "weight": 1.0, "skipped": False}, {**ROWS[1], "weight": 1.0, "skipped": False}], + "verifier_type": "toy_verifier", "verifier_version": None, + "verifier_settings": {"scoring_rules": 1, "note": "n"}, + } + + +def test_the_grade_passes_at_its_min_passing_score(): + rows = [{"id": "a", "score": 1.0}, {"id": "b", "score": 0.5}] + grade = _run(_Toy(outcome=Verification(criteria=rows), score_aggregator="weighted_average", + min_passing_score=0.75)).metadata["grades"]["check"] + + assert (grade["score"], grade["min_passing_score"], grade["passed"]) == (0.75, 0.75, True) + + +def test_a_direct_score_wins_and_keeps_the_rows_as_reported(): + metadata = _run(_Toy(outcome=Verification(criteria=ROWS, score=0.4))).metadata + + assert metadata["grades"]["check"]["score"] == 0.4 + assert [row["id"] for row in metadata["grades"]["check"]["verifier_criteria"]] == ["a", "b"] + assert metadata["verifications"]["check"]["score"] == 0.4 + + +def test_a_verification_with_no_rows_gets_one_for_the_grade_itself(): + grade = _run(_Toy(outcome=Verification(score=1))).metadata["grades"]["check"] + + assert grade["score"] == 1.0 and grade["passed"] is True + assert grade["verifier_criteria"] == [{"id": "check", "score": 1.0, "result": True, "weight": 1.0, "skipped": False}] + + +def test_a_raising_verifier_writes_a_failed_grade_no_legacy_entry_and_raises_the_same_exception(): + boom = ValueError("criterion #2 has unknown type 'x'") + context = TaskStepContext() + + with pytest.raises(ValueError) as raised: + _run(_Toy(outcome=boom), context) + + assert raised.value is boom + assert "verifications" not in context.metadata + grade = context.metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", 0.0, False) + assert grade["verifier_error_reasoning"] == "ValueError: criterion #2 has unknown type 'x'" + assert grade["verifier_criteria"] == [{"id": "check", "score": 0.0, "result": False, "weight": 1.0, "skipped": False}] + + +def test_raise_after_keeps_the_verdict_graded_then_fails_the_step(): + not_passing = VerifierNotPassingError("check: 1 criterion failed") + context = TaskStepContext() + + with pytest.raises(VerifierNotPassingError) as raised: + _run(_Toy(outcome=Verification(criteria=ROWS, raise_after=not_passing)), context) + + assert raised.value is not_passing + assert context.metadata["grades"]["check"]["verifier_status"] == "graded" + assert context.metadata["verifications"]["check"] == {"results": ROWS, "score": 0.0} + + +def test_a_skip_is_its_own_status_with_its_reason(): + metadata = _run(_Toy(outcome=Verification(skipped="no graded turns"))).metadata + + grade = metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("skipped", 0.0, False) + assert grade["verifier_skip_reasoning"] == "no graded turns" + assert grade["verifier_criteria"][0]["skipped"] is True + assert metadata["verifications"]["check"] == {"results": [], "score": None, "skipped": "no graded turns"} + + +def test_legacy_entries_are_written_exactly_as_the_verifier_gives_them(): + entries = {"vid": {"format": "f", "results": ROWS, "score": 0.0, "command": "pytest"}, + "vid-trajectory-mistakes": {"results": [], "score": 1.0}} + + metadata = _run(_Toy(outcome=Verification(criteria=ROWS, legacy_entries=entries), verifier_id="vid")).metadata + + assert json.dumps(metadata["verifications"]) == json.dumps(entries) + assert list(metadata["grades"]) == ["check"] + + +@pytest.mark.parametrize("rows", [ + [{"result": True}, {"result": False, "weight": 3}], + [{"score": 0.5}, {"score": 1.0, "weight": 2}], + [{"score": 1.0, "result": None}, {"score": 0.0, "result": True}], + [{"score": 1.0, "weight": 1}, {"score": 0.0, "weight": -1}], + [{"result": "yes"}, {}], +]) +def test_the_grade_rows_aggregate_as_the_rows_the_verifier_gave(rows): + graded = [verifier_module._row(i, row) for i, row in enumerate(rows)] + + for aggregator in ScoreAggregator: + assert aggregate_score(graded, aggregator) == aggregate_score(rows, aggregator) + assert [row["id"] for row in graded] == [str(i) for i in range(len(rows))] + + +def test_settings_are_the_steps_own_fields_with_large_ones_kept_by_digest(): + note = "x" * 2000 + step = _Toy(outcome=Verification(score=1), note=note, verifier_id="vid", min_passing_score=0.5) + + settings = _run(step).metadata["grades"]["check"]["verifier_settings"] + + body = json.dumps(note, sort_keys=True, separators=(",", ":")).encode() + assert settings == {"scoring_rules": 1, "min_passing_score": 0.5, + "note": {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}} + + +def test_base_fields_are_serialized_only_when_not_default_and_round_trip(): + assert _Toy().to_dict() == {"id": "check", "type": "toy_verifier", "version": None, "fail_task_on_error": True, + "note": ""} + step = _Toy(verifier_id="vid", score_aggregator="weighted_average", min_passing_score=0.5) + data = step.to_dict() + assert (data["verifier_id"], data["score_aggregator"], data["min_passing_score"]) == ("vid", "weighted_average", 0.5) + again = _Toy.from_dict(data) + assert (again.verifier_id, again.score_aggregator, again.min_passing_score) == ( + "vid", ScoreAggregator.WEIGHTED_AVERAGE, 0.5) + assert _Toy.from_dict(_Toy().to_dict()).verifier_id == "check" + + +@pytest.mark.parametrize("bad", [0, -0.5, 1.5, float("nan")]) +def test_a_min_passing_score_outside_zero_to_one_is_a_config_problem(bad): + assert _Toy(min_passing_score=bad).config_problems() == [ + f"min_passing_score must be above 0 and at most 1, not {float(bad)}"] + assert _Toy(min_passing_score=1).config_problems() == [] + + +class _Store: + def __init__(self, fail=False): + self.fail, self.puts = fail, {} + + def put(self, key, body, content_type=None): + if self.fail: + raise ConnectionError("store unreachable") + self.puts[key] = (body, content_type) + return f"file:///objects/{key}" + + +def _with_store(monkeypatch, store): + monkeypatch.setattr(verifier_module, "get_config", lambda: SimpleNamespace( + get_artifact_key_prefix=lambda: "pfx/", get_object_store=lambda: store)) + + +def test_a_small_output_stays_on_the_grade(): + grade = _run(_Toy(outcome=Verification(score=1, output={"log": "ok"}))).metadata["grades"]["check"] + + assert grade["verifier_output"] == {"log": "ok"} and "verifier_output_ref" not in grade + + +def test_a_large_output_goes_to_the_object_store(monkeypatch): + store = _Store() + _with_store(monkeypatch, store) + output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} + + grade = _run(_Toy(outcome=Verification(score=1, output=output)), TaskStepContext(instance_id="i1")).metadata[ + "grades"]["check"] + + [(key, (body, content_type))] = store.puts.items() + assert key.startswith("pfx/verifier-outputs/instance_id=i1/check-") and content_type == "application/json" + assert json.loads(body) == output + assert grade["verifier_output"] is None + assert grade["verifier_output_ref"] == {"object_url": f"file:///objects/{key}", "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest()} + + +def test_an_output_the_store_refuses_stays_inline_and_the_step_still_passes(monkeypatch): + _with_store(monkeypatch, _Store(fail=True)) + output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} + + grade = _run(_Toy(outcome=Verification(score=1, output=output))).metadata["grades"]["check"] + + assert grade["verifier_output"] == output and grade["passed"] is True + assert grade["verifier_output_offload_error"] == "ConnectionError: store unreachable" + + +def test_the_grade_and_legacy_entry_survive_the_stored_diff(): + pre = TaskStepContext(metadata={"verifications": {"earlier": {"results": [], "score": 1.0}}}) + post = copy.deepcopy(pre) + asyncio.run(_Toy(outcome=Verification(criteria=ROWS, output={"log": "ok"}), verifier_id="vid").execute(post)) + + doc = {"context": pre.to_safe_dict()} + _apply_forward(doc, build_context_update_ops(pre, post)) + + assert doc["context"]["metadata"]["grades"] == post.metadata["grades"] + assert doc["context"]["metadata"]["verifications"] == post.metadata["verifications"] + + +def test_a_verifier_must_implement_verify(): + class _NoVerify(VerifierTaskStep): + type: ClassVar[str] = "no_verify" + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + assert _registration.unimplemented(_NoVerify, TaskStep, ("from_dict",)).endswith("_NoVerify must implement verify") From 7c7492b47bbcb6dac4b9da17dc86c45959659c50 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 17 Nov 2026 19:52:37 -0800 Subject: [PATCH 02/10] fix(verifiers): a verifier's large output uploads on a worker thread Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/verifier.py | 19 +++++++++++-------- .../task_step/verifiers/test_verifier_base.py | 7 +++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 5ac06528b..b79b08f54 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -19,6 +19,7 @@ from agent_env.plugins import _requirements from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency +from agent_env.task_step.thread_work import finish_on_thread from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score logger = logging.getLogger(__name__) @@ -95,7 +96,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: except Exception as exc: self._write_grade(context, self._failed_grade(context, exc)) raise - self._write_grade(context, self._grade(context, verification)) + self._write_grade(context, await self._grade(context, verification)) entries = verification.legacy_entries if entries is None: entries = {self.verifier_id: self._legacy_entry(verification)} @@ -135,14 +136,14 @@ def _base_from_dict(cls, data: dict) -> dict: "min_passing_score": data.get("min_passing_score"), } - def _grade(self, context: TaskStepContext, verification: Verification) -> dict[str, Any]: + async def _grade(self, context: TaskStepContext, verification: Verification) -> dict[str, Any]: if verification.skipped is not None: status, score = VerifierStatus.SKIPPED, 0.0 else: status, score = VerifierStatus.GRADED, float(self._score(verification)) grade = self._grade_fields(context, status, score, verification.criteria) grade["verifier_skip_reasoning"] = verification.skipped - grade.update(self._output(context, verification.output)) + grade.update(await self._output(context, verification.output)) return grade def _failed_grade(self, context: TaskStepContext, exc: Exception) -> dict[str, Any]: @@ -181,16 +182,18 @@ def _legacy_entry(self, verification: Verification) -> dict[str, Any]: return {"results": verification.criteria, "score": None, "skipped": verification.skipped} return {"results": verification.criteria, "score": self._score(verification)} - def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: + async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the - object store; inline with ``verifier_output_offload_error`` when that write fails, which never fails the step.""" + object store, written on a worker thread; inline with ``verifier_output_offload_error`` when that write fails, + which never fails the step.""" if output is None: return {} body = json.dumps(output).encode() if len(body) <= OUTPUT_INLINE_BYTES: return {"verifier_output": output} try: - url = _store_output(context, self.id, body) + url = await finish_on_thread(functools.partial(_store_output, context.instance_id, self.id, body), + f"Uploading verifier {self.id}'s output") except Exception as exc: logger.warning("verifier %s: kept its %d-byte output inline, the object store refused it: %s", self.id, len(body), exc) @@ -224,9 +227,9 @@ def _compact(value: Any) -> Any: return {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)} -def _store_output(context: TaskStepContext, step_id: str, body: bytes) -> str: +def _store_output(instance_id: Optional[str], step_id: str, body: bytes) -> str: config = get_config() - key = (f"{config.get_artifact_key_prefix()}verifier-outputs/instance_id={context.instance_id}/" + key = (f"{config.get_artifact_key_prefix()}verifier-outputs/instance_id={instance_id}/" f"{step_id}-{uuid.uuid4().hex}.json") return config.get_object_store().put(key, body, content_type="application/json") diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 6691260a8..03fdea44f 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -3,6 +3,7 @@ import copy import hashlib import json +import threading from types import SimpleNamespace from typing import ClassVar @@ -178,9 +179,10 @@ def test_a_min_passing_score_outside_zero_to_one_is_a_config_problem(bad): class _Store: def __init__(self, fail=False): - self.fail, self.puts = fail, {} + self.fail, self.puts, self.threads = fail, {}, [] def put(self, key, body, content_type=None): + self.threads.append(threading.current_thread()) if self.fail: raise ConnectionError("store unreachable") self.puts[key] = (body, content_type) @@ -198,7 +200,7 @@ def test_a_small_output_stays_on_the_grade(): assert grade["verifier_output"] == {"log": "ok"} and "verifier_output_ref" not in grade -def test_a_large_output_goes_to_the_object_store(monkeypatch): +def test_a_large_output_goes_to_the_object_store_from_a_worker_thread(monkeypatch): store = _Store() _with_store(monkeypatch, store) output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} @@ -207,6 +209,7 @@ def test_a_large_output_goes_to_the_object_store(monkeypatch): "grades"]["check"] [(key, (body, content_type))] = store.puts.items() + assert store.threads != [threading.main_thread()] assert key.startswith("pfx/verifier-outputs/instance_id=i1/check-") and content_type == "application/json" assert json.loads(body) == output assert grade["verifier_output"] is None From 143cc0d80c78a3a355e666a3b716d1dd89461bc3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 17 Nov 2026 21:00:29 -0800 Subject: [PATCH 03/10] refactor(verifiers): a typed Grade, one score per verification, and shared helpers The base works out a verification's status and score once, for both its grade and its legacy entry, and builds the grade as a Grade TypedDict, which readers import. The package-version lookup and canonical JSON it needs move to plugins/_requirements.py and utils/canonical.py, which bundle/requires.py and the ledger now share. The base's tests go through execute and the step journal's replay rather than its private helpers. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_plugin_api.py | 4 +- src/agent_env/bundle/ledger.py | 4 +- src/agent_env/bundle/requires.py | 14 +- src/agent_env/plugins/_requirements.py | 12 +- .../task_steps/verifiers/__init__.py | 3 +- .../task_steps/verifiers/verifier.py | 126 ++++++++++-------- src/agent_env/utils/canonical.py | 12 ++ tst/unit/bundle/_support.py | 1 - .../task_step/verifiers/test_verifier_base.py | 43 +++--- 9 files changed, 134 insertions(+), 85 deletions(-) create mode 100644 src/agent_env/utils/canonical.py diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index d83976956..c68125112 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -48,8 +48,8 @@ "agent_env.runner.store.ensure_indexes", "agent_env.runner.store.insert_run", "agent_env.runner.store.get_run", "agent_env.runner.store.list_runs", "agent_env.runner.store.count_runs", "agent_env.runner.store.active_runs", "agent_env.runner.store.set_running", "agent_env.runner.store.mark_terminal", - # What a verifier step returns and scores its rows with. - "agent_env.task_step.task_steps.verifiers.Verification", "agent_env.task_step.task_steps.verifiers.VerifierStatus", + # What a verifier step returns, scores its rows with and writes. + "agent_env.task_step.task_steps.verifiers.Verification", "agent_env.task_step.task_steps.verifiers.Grade", "agent_env.task_step.task_steps.verifiers.VerifierStatus", "agent_env.task_step.task_steps.verifiers.ScoreAggregator", "agent_env.task_step.task_steps.verifiers.aggregate_score", # What a verify script is handed of the run it grades. "agent_env.task_step.task_steps.verifiers.Run", "agent_env.task_step.task_steps.verifiers.Turn", diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index 3a144e990..61df2f97a 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -10,7 +10,6 @@ from __future__ import annotations import hashlib -import json import stat from collections.abc import Callable, Iterable, Iterator, Mapping from contextlib import contextmanager, nullcontext @@ -31,6 +30,7 @@ from agent_env.store.ids import local_namespace from agent_env.store.local_state import holding_locks from agent_env.task.store import TASKS_COLLECTION +from agent_env.utils.canonical import canonical_json from .authoring import build_context_files from .parse import BundleKind @@ -321,7 +321,7 @@ def _diff(before: Mapping[str, str], after: Mapping[str, str]) -> tuple[list[str def _canonical(value: Any) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_scalar).encode() + return canonical_json(value, default=_scalar) def _scalar(value: Any) -> str: diff --git a/src/agent_env/bundle/requires.py b/src/agent_env/bundle/requires.py index f3b5e0fd4..9fd20d546 100644 --- a/src/agent_env/bundle/requires.py +++ b/src/agent_env/bundle/requires.py @@ -11,7 +11,7 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from enum import Enum -from importlib.metadata import distribution, packages_distributions +from importlib.metadata import packages_distributions from pathlib import Path from typing import TYPE_CHECKING @@ -136,15 +136,11 @@ def _shipping(self, cls: type | None) -> str | None: return None if self._packages is None: self._packages = packages_distributions() - module = cls.__module__ - names = list(dict.fromkeys(self._packages.get(module.partition(".")[0], ()))) - if len(names) > 1: # a namespace package: the one whose files hold the module - files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"} - names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}] - if len(names) != 1: + name = _requirements.distribution_holding(cls.__module__, self._packages) + if name is None: return None - version = _requirements.installed(canonicalize_name(names[0])) - return f"{names[0]} {version}" if version else names[0] + version = _requirements.installed(canonicalize_name(name)) + return f"{name} {version}" if version else name def uses(resolved: ResolvedBundle, providers: Providers | None = None) -> list[Use]: diff --git a/src/agent_env/plugins/_requirements.py b/src/agent_env/plugins/_requirements.py index d298340fd..084a435ae 100644 --- a/src/agent_env/plugins/_requirements.py +++ b/src/agent_env/plugins/_requirements.py @@ -9,7 +9,7 @@ import functools import shlex -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from importlib.metadata import PackageNotFoundError, distribution, version from typing import Any, Optional @@ -30,6 +30,16 @@ def installed(name: str) -> Optional[str]: return None +def distribution_holding(module: str, packages: Mapping[str, Sequence[str]]) -> Optional[str]: + """The one installed distribution whose files hold ``module``, given ``packages`` (``packages_distributions()``); + None when none or several do, as for a module on the path or an editable install.""" + names = list(dict.fromkeys(packages.get(module.partition(".")[0], ()))) + if len(names) > 1: # a namespace package: the one whose files hold the module + files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"} + names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}] + return names[0] if len(names) == 1 else None + + def incompatibility(dist: Any) -> Optional[str]: """Why ``dist``'s requirements on agent-env exclude the installed versions, or None. diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index ba6f4cd82..dd7450286 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -3,7 +3,7 @@ (score aggregation, judge output formats, trajectory compaction and frame selection for the LLM judge).""" from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score -from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierStatus, VerifierTaskStep +from agent_env.task_step.task_steps.verifiers.verifier import Grade, Verification, VerifierStatus, VerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_context import EnvClock, Firing, Message, Run, RunRecordUnavailable, Turn from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier @@ -18,6 +18,7 @@ "EnvClock", "EnvOutcomeVerifierTaskStep", "Firing", + "Grade", "Message", "RubricsVerifierTaskStep", "Run", diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index b79b08f54..4ce733ab0 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -13,14 +13,17 @@ from dataclasses import dataclass, field from enum import Enum from importlib.metadata import packages_distributions -from typing import Any, ClassVar, Optional +from typing import Any, ClassVar, Optional, TypedDict + +from packaging.utils import canonicalize_name from agent_env.config import get_config from agent_env.plugins import _requirements from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency -from agent_env.task_step.thread_work import finish_on_thread from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.thread_work import finish_on_thread +from agent_env.utils.canonical import canonical_json logger = logging.getLogger(__name__) @@ -43,6 +46,30 @@ class VerifierStatus(str, Enum): FAILED = "failed" +class _GradeFields(TypedDict): + name: str + score: float + min_passing_score: float + passed: bool + verifier_status: str + verifier_error_reasoning: Optional[str] + verifier_skip_reasoning: Optional[str] + verifier_output: Any + verifier_criteria: list[dict[str, Any]] + verifier_type: str + verifier_version: Optional[str] + verifier_settings: dict[str, Any] + + +class Grade(_GradeFields, total=False): + """A verifier step's named grade, as ``context.metadata["grades"][]`` holds it. ``verifier_status`` is a + ``VerifierStatus`` value; each ``verifier_criteria`` row has ``id``, ``score``, ``result``, ``weight`` and + ``skipped``, besides any keys the verifier gave it. The two fields here are written only for a large output.""" + + verifier_output_ref: dict[str, Any] + verifier_output_offload_error: str + + @dataclass class Verification: """What one run of a verifier found. @@ -64,7 +91,9 @@ class Verification: class VerifierTaskStep(TaskStep): """A step that grades a run. ``execute`` runs ``verify`` and writes the step's grade, and the legacy entries after - it; an exception from ``verify`` is a failed grade, raised again. Subclasses implement ``verify``, not ``execute``.""" + it; an exception from ``verify`` is a failed grade, raised again. Subclasses implement ``verify``, not ``execute``, + and their ``__init__`` takes and passes on ``verifier_id``, ``score_aggregator`` and ``min_passing_score``, which + ``_base_from_dict`` reads for them.""" DEFAULT_SCORE_AGGREGATOR: ClassVar[ScoreAggregator] = ScoreAggregator.ALL_PASS @@ -94,12 +123,25 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: try: verification = await self.verify(context) except Exception as exc: - self._write_grade(context, self._failed_grade(context, exc)) + self._write_grade(context, self._grade(context, VerifierStatus.FAILED, 0.0, [], + error=f"{type(exc).__name__}: {exc}")) raise - self._write_grade(context, await self._grade(context, verification)) + if verification.skipped is not None: + status, score = VerifierStatus.SKIPPED, None + elif verification.score is not None: + status, score = VerifierStatus.GRADED, verification.score + else: + status, score = VerifierStatus.GRADED, aggregate_score(verification.criteria, self.score_aggregator) + grade = self._grade(context, status, 0.0 if score is None else float(score), verification.criteria, + skip=verification.skipped) + grade.update(await self._output(context, verification.output)) + self._write_grade(context, grade) entries = verification.legacy_entries if entries is None: - entries = {self.verifier_id: self._legacy_entry(verification)} + entry = {"results": verification.criteria, "score": score} + if verification.skipped is not None: + entry["skipped"] = verification.skipped + entries = {self.verifier_id: entry} context.metadata.setdefault(VERIFICATIONS_KEY, {}).update(entries) if verification.raise_after is not None: raise verification.raise_after @@ -136,51 +178,26 @@ def _base_from_dict(cls, data: dict) -> dict: "min_passing_score": data.get("min_passing_score"), } - async def _grade(self, context: TaskStepContext, verification: Verification) -> dict[str, Any]: - if verification.skipped is not None: - status, score = VerifierStatus.SKIPPED, 0.0 - else: - status, score = VerifierStatus.GRADED, float(self._score(verification)) - grade = self._grade_fields(context, status, score, verification.criteria) - grade["verifier_skip_reasoning"] = verification.skipped - grade.update(await self._output(context, verification.output)) - return grade - - def _failed_grade(self, context: TaskStepContext, exc: Exception) -> dict[str, Any]: - grade = self._grade_fields(context, VerifierStatus.FAILED, 0.0, []) - grade["verifier_error_reasoning"] = f"{type(exc).__name__}: {exc}" - return grade - - def _grade_fields(self, context: TaskStepContext, status: VerifierStatus, score: float, - criteria: list[dict[str, Any]]) -> dict[str, Any]: + def _grade(self, context: TaskStepContext, status: VerifierStatus, score: float, criteria: list[dict[str, Any]], + *, error: Optional[str] = None, skip: Optional[str] = None) -> Grade: passed = status is VerifierStatus.GRADED and score >= self.min_passing_score rows = [_row(i, row) for i, row in enumerate(criteria)] or [{ "id": self.id, "score": score, "result": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, }] - return { - "name": self.id, - "score": score, - "min_passing_score": self.min_passing_score, - "passed": passed, - "verifier_status": status.value, - "verifier_error_reasoning": None, - "verifier_skip_reasoning": None, - "verifier_output": None, - "verifier_criteria": rows, - "verifier_type": self.type, - "verifier_version": _providing_version(type(self).__module__), - "verifier_settings": {"scoring_rules": SCORING_RULES, **self.verifier_settings(context)}, - } - - def _score(self, verification: Verification) -> float: - if verification.score is not None: - return verification.score - return aggregate_score(verification.criteria, self.score_aggregator) - - def _legacy_entry(self, verification: Verification) -> dict[str, Any]: - if verification.skipped is not None: - return {"results": verification.criteria, "score": None, "skipped": verification.skipped} - return {"results": verification.criteria, "score": self._score(verification)} + return Grade( + name=self.id, + score=score, + min_passing_score=self.min_passing_score, + passed=passed, + verifier_status=status.value, + verifier_error_reasoning=error, + verifier_skip_reasoning=skip, + verifier_output=None, + verifier_criteria=rows, + verifier_type=self.type, + verifier_version=_providing_version(type(self).__module__), + verifier_settings={"scoring_rules": SCORING_RULES, **self.verifier_settings(context)}, + ) async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the @@ -201,7 +218,7 @@ async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any] return {"verifier_output_ref": {"object_url": url, "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest()}} - def _write_grade(self, context: TaskStepContext, grade: dict[str, Any]) -> None: + def _write_grade(self, context: TaskStepContext, grade: Grade) -> None: context.metadata.setdefault(GRADES_KEY, {})[self.id] = grade logger.info("verifier %s: %s, score %s, %s", self.id, grade["verifier_status"], grade["score"], "passed" if grade["passed"] else "not passed") @@ -221,7 +238,7 @@ def _row(index: int, row: dict[str, Any]) -> dict[str, Any]: def _compact(value: Any) -> Any: - body = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + body = canonical_json(value) if len(body) <= SETTING_INLINE_BYTES: return value return {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)} @@ -236,8 +253,9 @@ def _store_output(instance_id: Optional[str], step_id: str, body: bytes) -> str: @functools.cache def _providing_version(module: str) -> Optional[str]: - """The version of the installed package that holds ``module``: agentenv-framework's for agent_env's own, else that - of the one distribution installing its top-level package; None when none does, as for a module on the path.""" - top = module.partition(".")[0] - names = [_FRAMEWORK] if top == "agent_env" else list(dict.fromkeys(packages_distributions().get(top, ()))) - return _requirements.installed(names[0]) if len(names) == 1 else None + """The version of the installed package that holds ``module``: agentenv-framework's for agent_env's own; None when + no one package does, as for a module on the path or an editable plugin.""" + if module.partition(".")[0] == "agent_env": + return _requirements.installed(_FRAMEWORK) + name = _requirements.distribution_holding(module, packages_distributions()) + return _requirements.installed(canonicalize_name(name)) if name else None diff --git a/src/agent_env/utils/canonical.py b/src/agent_env/utils/canonical.py new file mode 100644 index 000000000..1503bbc00 --- /dev/null +++ b/src/agent_env/utils/canonical.py @@ -0,0 +1,12 @@ +"""JSON written so that equal values give equal bytes, for hashing them.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any, Optional + + +def canonical_json(value: Any, default: Optional[Callable[[Any], Any]] = None) -> bytes: + """``value`` as JSON with its keys sorted and no spaces; ``default`` as for ``json.dumps``.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=default).encode() diff --git a/tst/unit/bundle/_support.py b/tst/unit/bundle/_support.py index 88f8ec95a..c39c9896f 100644 --- a/tst/unit/bundle/_support.py +++ b/tst/unit/bundle/_support.py @@ -69,7 +69,6 @@ def read(name): loaded=load)) monkeypatch.setattr(requires_module, "load_failures", lambda: failures or {}) monkeypatch.setattr(requires_module, "packages_distributions", lambda: packages or {}) - monkeypatch.setattr(requires_module, "distribution", read) monkeypatch.setattr(_requirements, "installed", (versions or {}).get) monkeypatch.setattr(_requirements, "distribution", read) diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 03fdea44f..75587d558 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -1,6 +1,5 @@ """VerifierTaskStep: the named grade it writes beside a verifier's legacy entries, for each way verify can end.""" import asyncio -import copy import hashlib import json import threading @@ -10,11 +9,13 @@ import pytest from agent_env.plugins import _registration -from agent_env.task.step_journal import _apply_forward +from agent_env.task.step_journal import replay_context from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError from agent_env.task_step.context_ops import build_context_update_ops from agent_env.task_step.task_step import TaskStep -from agent_env.task_step.task_steps.verifiers import ScoreAggregator, Verification, VerifierTaskStep, aggregate_score +from agent_env.task_step.task_steps.verifiers import ( + Grade, ScoreAggregator, Verification, VerifierTaskStep, aggregate_score, +) from agent_env.task_step.task_steps.verifiers import verifier as verifier_module @@ -140,11 +141,25 @@ def test_legacy_entries_are_written_exactly_as_the_verifier_gives_them(): [{"result": "yes"}, {}], ]) def test_the_grade_rows_aggregate_as_the_rows_the_verifier_gave(rows): - graded = [verifier_module._row(i, row) for i, row in enumerate(rows)] - for aggregator in ScoreAggregator: - assert aggregate_score(graded, aggregator) == aggregate_score(rows, aggregator) - assert [row["id"] for row in graded] == [str(i) for i in range(len(rows))] + metadata = _run(_Toy(outcome=Verification(criteria=rows), score_aggregator=aggregator)).metadata + + graded = metadata["grades"]["check"]["verifier_criteria"] + assert aggregate_score(graded, aggregator) == metadata["verifications"]["check"]["score"] + assert [row["id"] for row in graded] == [str(i) for i in range(len(rows))] + + +@pytest.mark.parametrize("outcome", [Verification(criteria=ROWS), Verification(skipped="nothing to judge"), + ValueError("broken"), Verification(score=1, output={"log": "ok"})]) +def test_every_grade_holds_the_fields_its_type_names(outcome): + context = TaskStepContext() + try: + _run(_Toy(outcome=outcome), context) + except ValueError: + pass + + keys = set(context.metadata["grades"]["check"]) + assert Grade.__required_keys__ <= keys <= Grade.__required_keys__ | Grade.__optional_keys__ def test_settings_are_the_steps_own_fields_with_large_ones_kept_by_digest(): @@ -227,16 +242,14 @@ def test_an_output_the_store_refuses_stays_inline_and_the_step_still_passes(monk assert grade["verifier_output_offload_error"] == "ConnectionError: store unreachable" -def test_the_grade_and_legacy_entry_survive_the_stored_diff(): - pre = TaskStepContext(metadata={"verifications": {"earlier": {"results": [], "score": 1.0}}}) - post = copy.deepcopy(pre) - asyncio.run(_Toy(outcome=Verification(criteria=ROWS, output={"log": "ok"}), verifier_id="vid").execute(post)) +def test_the_grade_and_legacy_entry_survive_the_step_journal(): + post = _run(_Toy(outcome=Verification(criteria=ROWS, output={"log": "ok"}), verifier_id="vid")) - doc = {"context": pre.to_safe_dict()} - _apply_forward(doc, build_context_update_ops(pre, post)) + ops = build_context_update_ops(None, post).to_journal_dict() + stored = replay_context([{"step_id": "check", "seq": 1, "ops": ops}], []) - assert doc["context"]["metadata"]["grades"] == post.metadata["grades"] - assert doc["context"]["metadata"]["verifications"] == post.metadata["verifications"] + assert stored["metadata"]["grades"] == post.metadata["grades"] + assert stored["metadata"]["verifications"] == post.metadata["verifications"] def test_a_verifier_must_implement_verify(): From e3d202503d6fe4bf8581534f2fbebcf99eac6413 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 08:21:38 -0800 Subject: [PATCH 04/10] refactor(verifiers): a dict output's keys join the legacy entry, in place of legacy_entries A verifier whose legacy verifications entry holds more than its results and score (a test's command and exit code, a judge's format fields) returns those as its output, which the grade keeps too, rather than spelling out whole entries. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/verifier.py | 24 +++++++++---------- .../task_step/verifiers/test_verifier_base.py | 23 +++++++++++------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 4ce733ab0..c254de423 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -76,21 +76,20 @@ class Verification: ``criteria``: a row per check, read for ``id``, ``score``, ``result``, ``weight`` and ``skipped``, any other keys kept. ``score``: the verifier's own score when it isn't the step's aggregate of the rows, which it then replaces. - ``output``: anything else worth keeping, such as a judge's justification or a test's log. ``legacy_entries``: the - ``verifications`` entries to write, by key, when they aren't ``{verifier_id: {"results", "score"}}``. ``skipped``: - why the verifier judged nothing. ``raise_after``: an exception the step raises once both are written, for a verdict - that also fails its step.""" + ``output``: anything else worth keeping, such as a judge's justification or a test's log; a dict's keys also join + the step's legacy ``verifications`` entry, after its ``results`` and ``score``. ``skipped``: why the verifier judged + nothing. ``raise_after``: an exception the step raises once both are written, for a verdict that also fails its + step.""" criteria: list[dict[str, Any]] = field(default_factory=list) score: Optional[float] = None output: Any = None - legacy_entries: Optional[dict[str, dict[str, Any]]] = None skipped: Optional[str] = None raise_after: Optional[BaseException] = None class VerifierTaskStep(TaskStep): - """A step that grades a run. ``execute`` runs ``verify`` and writes the step's grade, and the legacy entries after + """A step that grades a run. ``execute`` runs ``verify`` and writes the step's grade, and its legacy entry after it; an exception from ``verify`` is a failed grade, raised again. Subclasses implement ``verify``, not ``execute``, and their ``__init__`` takes and passes on ``verifier_id``, ``score_aggregator`` and ``min_passing_score``, which ``_base_from_dict`` reads for them.""" @@ -136,13 +135,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: skip=verification.skipped) grade.update(await self._output(context, verification.output)) self._write_grade(context, grade) - entries = verification.legacy_entries - if entries is None: - entry = {"results": verification.criteria, "score": score} - if verification.skipped is not None: - entry["skipped"] = verification.skipped - entries = {self.verifier_id: entry} - context.metadata.setdefault(VERIFICATIONS_KEY, {}).update(entries) + entry = {"results": verification.criteria, "score": score} + if verification.skipped is not None: + entry["skipped"] = verification.skipped + if isinstance(verification.output, dict): + entry.update(verification.output) + context.metadata.setdefault(VERIFICATIONS_KEY, {})[self.verifier_id] = entry if verification.raise_after is not None: raise verification.raise_after return context diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 75587d558..458c77a54 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -123,14 +123,20 @@ def test_a_skip_is_its_own_status_with_its_reason(): assert metadata["verifications"]["check"] == {"results": [], "score": None, "skipped": "no graded turns"} -def test_legacy_entries_are_written_exactly_as_the_verifier_gives_them(): - entries = {"vid": {"format": "f", "results": ROWS, "score": 0.0, "command": "pytest"}, - "vid-trajectory-mistakes": {"results": [], "score": 1.0}} +def test_a_dict_outputs_keys_join_the_legacy_entry_after_its_results_and_score(): + output = {"command": "pytest", "exit_code": 1, "stdout_head": "1 failed"} - metadata = _run(_Toy(outcome=Verification(criteria=ROWS, legacy_entries=entries), verifier_id="vid")).metadata + metadata = _run(_Toy(outcome=Verification(criteria=ROWS, output=output), verifier_id="vid")).metadata - assert json.dumps(metadata["verifications"]) == json.dumps(entries) - assert list(metadata["grades"]) == ["check"] + assert json.dumps(metadata["verifications"]) == json.dumps({"vid": {"results": ROWS, "score": 0.0, **output}}) + assert metadata["grades"]["check"]["verifier_output"] == output + + +def test_an_output_that_isnt_a_dict_stays_off_the_legacy_entry(): + metadata = _run(_Toy(outcome=Verification(score=1, output="3 passed"))).metadata + + assert metadata["verifications"]["check"] == {"results": [], "score": 1} + assert metadata["grades"]["check"]["verifier_output"] == "3 passed" @pytest.mark.parametrize("rows", [ @@ -220,8 +226,8 @@ def test_a_large_output_goes_to_the_object_store_from_a_worker_thread(monkeypatc _with_store(monkeypatch, store) output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} - grade = _run(_Toy(outcome=Verification(score=1, output=output)), TaskStepContext(instance_id="i1")).metadata[ - "grades"]["check"] + context = _run(_Toy(outcome=Verification(score=1, output=output)), TaskStepContext(instance_id="i1")) + grade = context.metadata["grades"]["check"] [(key, (body, content_type))] = store.puts.items() assert store.threads != [threading.main_thread()] @@ -230,6 +236,7 @@ def test_a_large_output_goes_to_the_object_store_from_a_worker_thread(monkeypatc assert grade["verifier_output"] is None assert grade["verifier_output_ref"] == {"object_url": f"file:///objects/{key}", "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest()} + assert context.metadata["verifications"]["check"]["log"] == output["log"] def test_an_output_the_store_refuses_stays_inline_and_the_step_still_passes(monkeypatch): From a5b39f0260277f37f215848d6c717a14abfe9767 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 09:14:14 -0800 Subject: [PATCH 05/10] refactor(verifiers): a grade's rows say whether each passed as passed, not result The grade's copy of each criterion row carries passed, whether the row passes under scoring rules 1 (an explicit True result, as ALL_PASS and ANY_PASS count it), as the grade itself does one level up. The legacy verifications rows keep the verifier's result unchanged for the readers that take it. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/verifier.py | 17 +++++++++------- tst/unit/task/episode/run_test.py | 2 +- .../test_agent_prompt_response_verifier.py | 3 ++- .../task_step/verifiers/test_verifier_base.py | 20 +++++++++++++------ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index c254de423..f93d92173 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -63,8 +63,9 @@ class _GradeFields(TypedDict): class Grade(_GradeFields, total=False): """A verifier step's named grade, as ``context.metadata["grades"][]`` holds it. ``verifier_status`` is a - ``VerifierStatus`` value; each ``verifier_criteria`` row has ``id``, ``score``, ``result``, ``weight`` and - ``skipped``, besides any keys the verifier gave it. The two fields here are written only for a large output.""" + ``VerifierStatus`` value; each ``verifier_criteria`` row has ``id``, ``score``, ``passed``, ``weight`` and + ``skipped``, besides any other keys the verifier gave it. The two fields here are written only for a large + output.""" verifier_output_ref: dict[str, Any] verifier_output_offload_error: str @@ -180,7 +181,7 @@ def _grade(self, context: TaskStepContext, status: VerifierStatus, score: float, *, error: Optional[str] = None, skip: Optional[str] = None) -> Grade: passed = status is VerifierStatus.GRADED and score >= self.min_passing_score rows = [_row(i, row) for i, row in enumerate(criteria)] or [{ - "id": self.id, "score": score, "result": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, + "id": self.id, "score": score, "passed": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, }] return Grade( name=self.id, @@ -223,13 +224,15 @@ def _write_grade(self, context: TaskStepContext, grade: Grade) -> None: def _row(index: int, row: dict[str, Any]) -> dict[str, Any]: - """``row`` with the keys every grade's rows carry, each default chosen so ``aggregate_score`` reads the row as it - reads the original: ``result`` is never filled in, since ALL_PASS counts only an explicit ``True``.""" + """The grade's copy of a verifier's ``row``: its ``passed`` whether the row passes under scoring rules 1, an + explicit ``True`` ``result`` (as ALL_PASS and ANY_PASS count it), and each other key's default the one + ``aggregate_score`` reads the row with.""" + copied = {k: v for k, v in row.items() if k != "result"} return { - **row, + **copied, "id": row.get("id", str(index)), "score": row.get("score", 1.0 if row.get("result") else 0.0), - "result": row.get("result"), + "passed": row.get("result") is True, "weight": row.get("weight", 1.0), "skipped": row.get("skipped", False), } diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 6d004c028..4f252515d 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -588,7 +588,7 @@ async def test_the_stored_run_holds_each_verifiers_named_grade_a_failed_one_incl assert {step: (g["verifier_status"], g["score"], g["passed"]) for step, g in grades.items()} == { "crates": ("graded", 1.0, True), "graded": ("failed", 0.0, False)} assert grades["graded"]["verifier_error_reasoning"].startswith("StepContextError: PromptResponse") - assert grades["crates"]["verifier_criteria"][0]["result"] is True + assert grades["crates"]["verifier_criteria"][0]["passed"] is True BREAKS_WHILE_THE_TURN_IS_OPEN = ( diff --git a/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py index cbf223433..22914b37f 100644 --- a/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py +++ b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py @@ -52,7 +52,8 @@ def test_the_legacy_entry_is_unchanged_and_the_grade_carries_its_score(aggregato assert grade["score"] == json.loads(BEFORE[aggregator])["score"] assert (grade["verifier_status"], grade["verifier_type"]) == ("graded", "agent_prompt_response_verifier") assert grade["verifier_version"] == _requirements.installed("agentenv-framework") - assert [row["id"] for row in grade["verifier_criteria"]] == ["says-done", "1", "names-file"] + assert [(row["id"], row["passed"]) for row in grade["verifier_criteria"]] == [ + ("says-done", True), ("1", True), ("names-file", False)] assert recorded_scores([step], context) == {"check": json.loads(BEFORE[aggregator])["score"]} diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 458c77a54..c44ce83f6 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -57,7 +57,10 @@ def test_a_graded_verification_writes_its_named_grade_and_the_legacy_entry(): assert grade == { "name": "check", "score": 0.0, "min_passing_score": 1.0, "passed": False, "verifier_status": "graded", "verifier_error_reasoning": None, "verifier_skip_reasoning": None, "verifier_output": None, - "verifier_criteria": [{**ROWS[0], "weight": 1.0, "skipped": False}, {**ROWS[1], "weight": 1.0, "skipped": False}], + "verifier_criteria": [ + {"id": "a", "score": 1.0, "why": "found it", "passed": True, "weight": 1.0, "skipped": False}, + {"id": "b", "score": 0.0, "passed": False, "weight": 1.0, "skipped": False}, + ], "verifier_type": "toy_verifier", "verifier_version": None, "verifier_settings": {"scoring_rules": 1, "note": "n"}, } @@ -83,7 +86,7 @@ def test_a_verification_with_no_rows_gets_one_for_the_grade_itself(): grade = _run(_Toy(outcome=Verification(score=1))).metadata["grades"]["check"] assert grade["score"] == 1.0 and grade["passed"] is True - assert grade["verifier_criteria"] == [{"id": "check", "score": 1.0, "result": True, "weight": 1.0, "skipped": False}] + assert grade["verifier_criteria"] == [{"id": "check", "score": 1.0, "passed": True, "weight": 1.0, "skipped": False}] def test_a_raising_verifier_writes_a_failed_grade_no_legacy_entry_and_raises_the_same_exception(): @@ -98,7 +101,7 @@ def test_a_raising_verifier_writes_a_failed_grade_no_legacy_entry_and_raises_the grade = context.metadata["grades"]["check"] assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", 0.0, False) assert grade["verifier_error_reasoning"] == "ValueError: criterion #2 has unknown type 'x'" - assert grade["verifier_criteria"] == [{"id": "check", "score": 0.0, "result": False, "weight": 1.0, "skipped": False}] + assert grade["verifier_criteria"] == [{"id": "check", "score": 0.0, "passed": False, "weight": 1.0, "skipped": False}] def test_raise_after_keeps_the_verdict_graded_then_fails_the_step(): @@ -146,13 +149,18 @@ def test_an_output_that_isnt_a_dict_stays_off_the_legacy_entry(): [{"score": 1.0, "weight": 1}, {"score": 0.0, "weight": -1}], [{"result": "yes"}, {}], ]) -def test_the_grade_rows_aggregate_as_the_rows_the_verifier_gave(rows): +def test_the_grade_rows_pass_and_score_as_scoring_read_the_rows_the_verifier_gave(rows): + by_rule = {ScoreAggregator.ALL_PASS: all, ScoreAggregator.ANY_PASS: any} for aggregator in ScoreAggregator: metadata = _run(_Toy(outcome=Verification(criteria=rows), score_aggregator=aggregator)).metadata - graded = metadata["grades"]["check"]["verifier_criteria"] - assert aggregate_score(graded, aggregator) == metadata["verifications"]["check"]["score"] + graded, legacy = metadata["grades"]["check"]["verifier_criteria"], metadata["verifications"]["check"] + if aggregator in by_rule: + assert legacy["score"] == (1.0 if by_rule[aggregator](row["passed"] for row in graded) else 0.0) + else: + assert aggregate_score(graded, aggregator) == legacy["score"] assert [row["id"] for row in graded] == [str(i) for i in range(len(rows))] + assert "result" not in graded[0] and legacy["results"] == rows @pytest.mark.parametrize("outcome", [Verification(criteria=ROWS), Verification(skipped="nothing to judge"), From bca01e0f2121a9e77fa6aaeac23ea2a9d54f9835 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 08:37:16 -0800 Subject: [PATCH 06/10] test(protocol): the agent clock test's earlier status predates the config it's checked against The test wrote the follower's earlier status and then had sync_time write the new config, which a status must postdate (by st_mtime_ns) to count. On a kernel whose file timestamps advance in coarse ticks, the two writes can share an mtime, and the earlier status of the same gateway and name then passed for the new config: once in CodeBuild's last 200 runs. The earlier status is now a second older, as a follower's earlier status is. Co-Authored-By: Claude Opus 5.5 --- .../agentenv-protocol/tests/test_a2a_agent_clock.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py index b23e5192f..f59285e2b 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py @@ -45,6 +45,14 @@ def _status(image, **status) -> None: (image / "status.json").write_text(json.dumps(status)) +def _status_from_before(image, **status) -> None: + """A status the follower wrote for an earlier config, a second before this one: a file's mtime on Linux moves in + ticks of a few milliseconds, so a config written right after a status can share its mtime.""" + _status(image, **status) + before = time.time_ns() - 1_000_000_000 + os.utime(image / "status.json", ns=(before, before)) + + def _follows_the_next_config(image, **status) -> threading.Thread: """A follower that, as the real one, writes its status once it reads a config other than the one it had.""" config = image / "follower.json" @@ -140,7 +148,7 @@ def test_a_zone_that_isnt_one_is_refused(image): {"url": "http://gw/clock/watch", "name": "agent-solver"}, # this one, under the token the config replaced ]) def test_what_the_follower_showed_of_its_config_before_never_passes_for_the_new_one(image, before): - _status(image, connected=True, version="b:3", armed=True, **before) + _status_from_before(image, connected=True, version="b:3", armed=True, **before) with _app() as client: answer = client.post("/ext/clock/sync-time", json=REQUEST) From b7f605fd677783974a45622e781287b0561e1f49 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 09:45:10 -0800 Subject: [PATCH 07/10] test(computer): the display-only reuse test checks what was reused, not how long the put took The test also bounded ComputerEnv.put at 60 s, which reuses the image but still boots and probes it to mark it ready; on a busy runner that took 85 s (one slow shard on #761). Reuse itself is what its other checks assert: the same image, the same installed packages, reused and not emulated. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/computer_env_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tst/integration/computer/computer_env_test.py b/tst/integration/computer/computer_env_test.py index 0da7874b3..69de4eea5 100644 --- a/tst/integration/computer/computer_env_test.py +++ b/tst/integration/computer/computer_env_test.py @@ -15,7 +15,6 @@ import json import platform import subprocess -import time import uuid import httpx @@ -101,17 +100,14 @@ async def test_the_image_of_this_machines_arch_boots_as_the_session_user_and_pas def test_an_env_that_changes_only_its_display_and_name_reuses_the_image(desktop): with namespace_routing(): - started = time.monotonic() other = ComputerEnv.put(id=f"{desktop.id}-other", computer=desktop.computer, driver=desktop.driver, environment_name="other", display={"width": 1024, "height": 768}) - took = time.monotonic() - started results = Env.get(other.id).metadata["composed"]["container"] first = desktop.metadata["composed"]["container"] assert results[ARCH]["reused"] is True and results[ARCH]["image"] == first[ARCH]["image"] assert results[ARCH]["status"] == "ready" and results[ARCH]["card"]["name"] == "other" assert results[ARCH]["installed"] == first[ARCH]["installed"] and results[ARCH]["emulated"] is False - assert took < 60, took #: A request to an address on the internet, around the egress proxy, and through it, printing the status and the body From 96590d29610e70dc81789c9580314265efd060a7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 10:15:27 -0800 Subject: [PATCH 08/10] refactor(verifiers): a grade that isn't graded has no score, and settings name the aggregator A skipped or failed grade's score is None rather than 0, as its legacy entry's already is, so a reader averaging scores can't count a skip or a broken verifier as a zero; how a skip counts in a run's result is the result rules' call. verifier_settings always names the score_aggregator the step scores with, default or not, so a changed default shows as drift, and leaves out min_passing_score, which the grade holds itself. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/verifier.py | 30 +++++++++++-------- tst/unit/task/episode/run_test.py | 2 +- .../task_step/verifiers/test_verifier_base.py | 23 +++++++++----- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index f93d92173..8933af89a 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -35,8 +35,10 @@ DEFAULT_MIN_PASSING_SCORE = 1.0 OUTPUT_INLINE_BYTES = 16 * 1024 SETTING_INLINE_BYTES = 1024 -# The step's fields that name or schedule it rather than say how it grades, left out of its verifier_settings. -_NOT_SETTINGS = ("id", "type", "version", "depends_on", "fail_task_on_error", "retry_config", "verifier_id") +# The step's fields left out of its verifier_settings: those that name or schedule it rather than say how it grades, and +# min_passing_score, which the grade holds itself. +_NOT_SETTINGS = ("id", "type", "version", "depends_on", "fail_task_on_error", "retry_config", "verifier_id", + "min_passing_score") _FRAMEWORK = "agentenv-framework" @@ -48,7 +50,7 @@ class VerifierStatus(str, Enum): class _GradeFields(TypedDict): name: str - score: float + score: Optional[float] min_passing_score: float passed: bool verifier_status: str @@ -63,9 +65,9 @@ class _GradeFields(TypedDict): class Grade(_GradeFields, total=False): """A verifier step's named grade, as ``context.metadata["grades"][]`` holds it. ``verifier_status`` is a - ``VerifierStatus`` value; each ``verifier_criteria`` row has ``id``, ``score``, ``passed``, ``weight`` and - ``skipped``, besides any other keys the verifier gave it. The two fields here are written only for a large - output.""" + ``VerifierStatus`` value, and ``score`` is None unless it is ``graded``; each ``verifier_criteria`` row has ``id``, + ``score``, ``passed``, ``weight`` and ``skipped``, besides any other keys the verifier gave it. The two fields here + are written only for a large output.""" verifier_output_ref: dict[str, Any] verifier_output_offload_error: str @@ -123,7 +125,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: try: verification = await self.verify(context) except Exception as exc: - self._write_grade(context, self._grade(context, VerifierStatus.FAILED, 0.0, [], + self._write_grade(context, self._grade(context, VerifierStatus.FAILED, None, [], error=f"{type(exc).__name__}: {exc}")) raise if verification.skipped is not None: @@ -132,7 +134,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: status, score = VerifierStatus.GRADED, verification.score else: status, score = VerifierStatus.GRADED, aggregate_score(verification.criteria, self.score_aggregator) - grade = self._grade(context, status, 0.0 if score is None else float(score), verification.criteria, + grade = self._grade(context, status, None if score is None else float(score), verification.criteria, skip=verification.skipped) grade.update(await self._output(context, verification.output)) self._write_grade(context, grade) @@ -148,9 +150,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: """What this step grades with in ``context``'s run: its own fields, one over ``SETTING_INLINE_BYTES`` kept as its - sha256 and size. A subclass adds what the run changes or decides, such as the ``step_params`` it applies or a - judge's model. Pure, since it is also read at run start.""" - return {k: _compact(v) for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} + sha256 and size, and its ``score_aggregator`` whether or not it is its default. A subclass adds what the run + changes or decides, such as the ``step_params`` it applies or a judge's model. Pure, since it is also read at + run start.""" + fields = {k: _compact(v) for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} + return {**fields, "score_aggregator": self.score_aggregator.value} def config_problems(self) -> list[str]: problems = super().config_problems() @@ -177,8 +181,8 @@ def _base_from_dict(cls, data: dict) -> dict: "min_passing_score": data.get("min_passing_score"), } - def _grade(self, context: TaskStepContext, status: VerifierStatus, score: float, criteria: list[dict[str, Any]], - *, error: Optional[str] = None, skip: Optional[str] = None) -> Grade: + def _grade(self, context: TaskStepContext, status: VerifierStatus, score: Optional[float], + criteria: list[dict[str, Any]], *, error: Optional[str] = None, skip: Optional[str] = None) -> Grade: passed = status is VerifierStatus.GRADED and score >= self.min_passing_score rows = [_row(i, row) for i, row in enumerate(criteria)] or [{ "id": self.id, "score": score, "passed": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 7bdf66832..367af76dd 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -588,7 +588,7 @@ async def test_the_stored_run_holds_each_verifiers_named_grade_a_failed_one_incl assert await episode.result() == {"crates": 1.0} grades = Task.get_instance(episode.id).context["metadata"]["grades"] assert {step: (g["verifier_status"], g["score"], g["passed"]) for step, g in grades.items()} == { - "crates": ("graded", 1.0, True), "graded": ("failed", 0.0, False)} + "crates": ("graded", 1.0, True), "graded": ("failed", None, False)} assert grades["graded"]["verifier_error_reasoning"].startswith("StepContextError: PromptResponse") assert grades["crates"]["verifier_criteria"][0]["passed"] is True diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index c44ce83f6..b5d799733 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -62,7 +62,7 @@ def test_a_graded_verification_writes_its_named_grade_and_the_legacy_entry(): {"id": "b", "score": 0.0, "passed": False, "weight": 1.0, "skipped": False}, ], "verifier_type": "toy_verifier", "verifier_version": None, - "verifier_settings": {"scoring_rules": 1, "note": "n"}, + "verifier_settings": {"scoring_rules": 1, "note": "n", "score_aggregator": "all_pass"}, } @@ -99,9 +99,9 @@ def test_a_raising_verifier_writes_a_failed_grade_no_legacy_entry_and_raises_the assert raised.value is boom assert "verifications" not in context.metadata grade = context.metadata["grades"]["check"] - assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", 0.0, False) + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", None, False) assert grade["verifier_error_reasoning"] == "ValueError: criterion #2 has unknown type 'x'" - assert grade["verifier_criteria"] == [{"id": "check", "score": 0.0, "passed": False, "weight": 1.0, "skipped": False}] + assert grade["verifier_criteria"] == [{"id": "check", "score": None, "passed": False, "weight": 1.0, "skipped": False}] def test_raise_after_keeps_the_verdict_graded_then_fails_the_step(): @@ -120,7 +120,7 @@ def test_a_skip_is_its_own_status_with_its_reason(): metadata = _run(_Toy(outcome=Verification(skipped="no graded turns"))).metadata grade = metadata["grades"]["check"] - assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("skipped", 0.0, False) + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("skipped", None, False) assert grade["verifier_skip_reasoning"] == "no graded turns" assert grade["verifier_criteria"][0]["skipped"] is True assert metadata["verifications"]["check"] == {"results": [], "score": None, "skipped": "no graded turns"} @@ -180,11 +180,20 @@ def test_settings_are_the_steps_own_fields_with_large_ones_kept_by_digest(): note = "x" * 2000 step = _Toy(outcome=Verification(score=1), note=note, verifier_id="vid", min_passing_score=0.5) - settings = _run(step).metadata["grades"]["check"]["verifier_settings"] + grade = _run(step).metadata["grades"]["check"] body = json.dumps(note, sort_keys=True, separators=(",", ":")).encode() - assert settings == {"scoring_rules": 1, "min_passing_score": 0.5, - "note": {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}} + assert grade["verifier_settings"] == {"scoring_rules": 1, "score_aggregator": "all_pass", + "note": {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}} + assert grade["min_passing_score"] == 0.5 + + +def test_settings_name_the_aggregator_the_step_scores_with_even_at_its_default(): + for aggregator, written in ((None, "all_pass"), ("weighted_average", "weighted_average")): + settings = _run(_Toy(outcome=Verification(score=1), score_aggregator=aggregator)).metadata["grades"][ + "check"]["verifier_settings"] + + assert settings["score_aggregator"] == written def test_base_fields_are_serialized_only_when_not_default_and_round_trip(): From 01b77c8afe536c498d8858bac247b946da60a34f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 10:19:14 -0800 Subject: [PATCH 09/10] test(protocol): name the earlier status's age in the agent clock test Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/tests/test_a2a_agent_clock.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py index f59285e2b..a24bb8bae 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py @@ -45,11 +45,14 @@ def _status(image, **status) -> None: (image / "status.json").write_text(json.dumps(status)) +EARLIER_STATUS_AGE_NS = 1_000_000_000 + + def _status_from_before(image, **status) -> None: - """A status the follower wrote for an earlier config, a second before this one: a file's mtime on Linux moves in - ticks of a few milliseconds, so a config written right after a status can share its mtime.""" + """A status the follower wrote for an earlier config, ``EARLIER_STATUS_AGE_NS`` before this one: a file's mtime on + Linux moves in ticks of a few milliseconds, so a config written right after a status can share its mtime.""" _status(image, **status) - before = time.time_ns() - 1_000_000_000 + before = time.time_ns() - EARLIER_STATUS_AGE_NS os.utime(image / "status.json", ns=(before, before)) From 27fde75986c11137de87b16e1bb1dbc98c588da0 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 11:03:19 -0800 Subject: [PATCH 10/10] test(computer): the disk tests' root filesystem builds from mirror.gcr.io's busybox The fast integration tier pulls its bases straight from their registries, with no mirror step, and ECR Public answered 429 for busybox on #761's run, failing the three disk tests and the stopped-container test. mirror.gcr.io serves Docker Hub's busybox under the same digest for amd64 and arm64. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/disk/rootfs/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tst/integration/computer/disk/rootfs/Dockerfile b/tst/integration/computer/disk/rootfs/Dockerfile index dba0242df..583eea178 100644 --- a/tst/integration/computer/disk/rootfs/Dockerfile +++ b/tst/integration/computer/disk/rootfs/Dockerfile @@ -1,5 +1,7 @@ -# A small root filesystem, for either arch, that the disk tests make disks of. -FROM public.ecr.aws/docker/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec +# A small root filesystem, for either arch, that the disk tests make disks of. From mirror.gcr.io, which serves Docker +# Hub's busybox under the same digest for both arches, since the fast tier pulls its bases straight from the registry +# and ECR Public rate-limits the runners' shared IPs. +FROM mirror.gcr.io/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec ENV GREETING="hello disk" USER nobody WORKDIR /tmp