diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index 464f49c85..c68125112 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -27,6 +27,7 @@ BASES = ( "agent_env.env.env.Env", "agent_env.task_step.task_step.TaskStep", + "agent_env.task_step.task_steps.verifiers.verifier.VerifierTaskStep", "agent_env.artifact.artifact.Artifact", "agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider", "agent_env.providers.env_state.env_state_provider.EnvStateProvider", @@ -47,6 +48,9 @@ "agent_env.runner.store.ensure_indexes", "agent_env.runner.store.insert_run", "agent_env.runner.store.get_run", "agent_env.runner.store.list_runs", "agent_env.runner.store.count_runs", "agent_env.runner.store.active_runs", "agent_env.runner.store.set_running", "agent_env.runner.store.mark_terminal", + # What a verifier step returns, scores its rows with and writes. + "agent_env.task_step.task_steps.verifiers.Verification", "agent_env.task_step.task_steps.verifiers.Grade", "agent_env.task_step.task_steps.verifiers.VerifierStatus", + "agent_env.task_step.task_steps.verifiers.ScoreAggregator", "agent_env.task_step.task_steps.verifiers.aggregate_score", # What a verify script is handed of the run it grades. "agent_env.task_step.task_steps.verifiers.Run", "agent_env.task_step.task_steps.verifiers.Turn", "agent_env.task_step.task_steps.verifiers.Message", "agent_env.task_step.task_steps.verifiers.Firing", diff --git a/AGENTS.md b/AGENTS.md index bda4f4396..02995ff96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). A verifier on the `VerifierTaskStep` base (`verifiers/verifier.py`; `agent_prompt_response_verifier` so far, the other built-ins still write their entries themselves) implements `verify` and returns a `Verification`; the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings` with `scoring_rules`) beside the legacy `verifications` entries every reader still takes, and an exception from `verify` is a failed grade raised again. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py index b23e5192f..a24bb8bae 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent_clock.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent_clock.py @@ -45,6 +45,17 @@ def _status(image, **status) -> None: (image / "status.json").write_text(json.dumps(status)) +EARLIER_STATUS_AGE_NS = 1_000_000_000 + + +def _status_from_before(image, **status) -> None: + """A status the follower wrote for an earlier config, ``EARLIER_STATUS_AGE_NS`` before this one: a file's mtime on + Linux moves in ticks of a few milliseconds, so a config written right after a status can share its mtime.""" + _status(image, **status) + before = time.time_ns() - EARLIER_STATUS_AGE_NS + os.utime(image / "status.json", ns=(before, before)) + + def _follows_the_next_config(image, **status) -> threading.Thread: """A follower that, as the real one, writes its status once it reads a config other than the one it had.""" config = image / "follower.json" @@ -140,7 +151,7 @@ def test_a_zone_that_isnt_one_is_refused(image): {"url": "http://gw/clock/watch", "name": "agent-solver"}, # this one, under the token the config replaced ]) def test_what_the_follower_showed_of_its_config_before_never_passes_for_the_new_one(image, before): - _status(image, connected=True, version="b:3", armed=True, **before) + _status_from_before(image, connected=True, version="b:3", armed=True, **before) with _app() as client: answer = client.post("/ext/clock/sync-time", json=REQUEST) diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index f519124d3..8091f1417 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -10,7 +10,6 @@ from __future__ import annotations import hashlib -import json import stat from collections.abc import Callable, Iterable, Iterator, Mapping from contextlib import contextmanager, nullcontext @@ -31,6 +30,7 @@ from agent_env.store.ids import local_namespace from agent_env.store.local_state import holding_locks from agent_env.task.store import TASKS_COLLECTION +from agent_env.utils.canonical import canonical_json from agent_env.utils.docker.context import shared_files from .authoring import build_context_files @@ -326,7 +326,7 @@ def _diff(before: Mapping[str, str], after: Mapping[str, str]) -> tuple[list[str def _canonical(value: Any) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_scalar).encode() + return canonical_json(value, default=_scalar) def _scalar(value: Any) -> str: diff --git a/src/agent_env/bundle/requires.py b/src/agent_env/bundle/requires.py index f3b5e0fd4..9fd20d546 100644 --- a/src/agent_env/bundle/requires.py +++ b/src/agent_env/bundle/requires.py @@ -11,7 +11,7 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from enum import Enum -from importlib.metadata import distribution, packages_distributions +from importlib.metadata import packages_distributions from pathlib import Path from typing import TYPE_CHECKING @@ -136,15 +136,11 @@ def _shipping(self, cls: type | None) -> str | None: return None if self._packages is None: self._packages = packages_distributions() - module = cls.__module__ - names = list(dict.fromkeys(self._packages.get(module.partition(".")[0], ()))) - if len(names) > 1: # a namespace package: the one whose files hold the module - files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"} - names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}] - if len(names) != 1: + name = _requirements.distribution_holding(cls.__module__, self._packages) + if name is None: return None - version = _requirements.installed(canonicalize_name(names[0])) - return f"{names[0]} {version}" if version else names[0] + version = _requirements.installed(canonicalize_name(name)) + return f"{name} {version}" if version else name def uses(resolved: ResolvedBundle, providers: Providers | None = None) -> list[Use]: diff --git a/src/agent_env/plugins/_requirements.py b/src/agent_env/plugins/_requirements.py index d298340fd..084a435ae 100644 --- a/src/agent_env/plugins/_requirements.py +++ b/src/agent_env/plugins/_requirements.py @@ -9,7 +9,7 @@ import functools import shlex -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from importlib.metadata import PackageNotFoundError, distribution, version from typing import Any, Optional @@ -30,6 +30,16 @@ def installed(name: str) -> Optional[str]: return None +def distribution_holding(module: str, packages: Mapping[str, Sequence[str]]) -> Optional[str]: + """The one installed distribution whose files hold ``module``, given ``packages`` (``packages_distributions()``); + None when none or several do, as for a module on the path or an editable install.""" + names = list(dict.fromkeys(packages.get(module.partition(".")[0], ()))) + if len(names) > 1: # a namespace package: the one whose files hold the module + files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"} + names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}] + return names[0] if len(names) == 1 else None + + def incompatibility(dist: Any) -> Optional[str]: """Why ``dist``'s requirements on agent-env exclude the installed versions, or None. diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index 3edb4d3f1..dd7450286 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -1,9 +1,9 @@ -"""Verifier task steps: the steps that score a run and record a row under -``context.metadata["verifications"]``, plus the helpers only they use (score -aggregation, judge output formats, trajectory compaction and frame selection for -the LLM judge).""" +"""Verifier task steps: the steps that grade a run, the ``VerifierTaskStep`` base they move onto, which writes each one's +named grade under ``context.metadata["grades"]`` beside its ``verifications`` entry, and the helpers only they use +(score aggregation, judge output formats, trajectory compaction and frame selection for the LLM judge).""" from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import Grade, Verification, VerifierStatus, VerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_context import EnvClock, Firing, Message, Run, RunRecordUnavailable, Turn from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier @@ -18,6 +18,7 @@ "EnvClock", "EnvOutcomeVerifierTaskStep", "Firing", + "Grade", "Message", "RubricsVerifierTaskStep", "Run", @@ -25,6 +26,9 @@ "RunRecordUnavailable", "ScoreAggregator", "Turn", + "Verification", + "VerifierStatus", + "VerifierTaskStep", "VerifySandboxTaskStep", "aggregate_score", "is_rubrics_verifier", diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 0c2c78672..d27084736 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -10,9 +10,10 @@ response_regex_present) raise ValueError. A verifier that can't actually verify anything is a misconfiguration, not a runtime decision. -Result rows merge `{**criterion, **outcome}` and aggregated score is written -to `context.metadata["verifications"][verifier_id]`, matching the shape used -by `VerifySandboxTaskStep` and `RubricsVerifierTaskStep`. +Result rows merge `{**criterion, **outcome}`. On the `VerifierTaskStep` base, which +aggregates them into the step's grade and writes the +`context.metadata["verifications"][verifier_id]` entry, in the shape used by +`VerifySandboxTaskStep` and `RubricsVerifierTaskStep`. """ from __future__ import annotations @@ -22,8 +23,9 @@ from typing import ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency -from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep logger = logging.getLogger(__name__) @@ -33,9 +35,8 @@ } -class AgentPromptResponseVerifierTaskStep(TaskStep): +class AgentPromptResponseVerifierTaskStep(VerifierTaskStep): type: ClassVar[str] = "agent_prompt_response_verifier" - step_role = StepRole.GRADER entity_refs = () def __init__( @@ -44,18 +45,17 @@ def __init__( version: Optional[int], prompt_id: str, criteria: Optional[list[dict]] = None, - score_aggregator: Optional[ScoreAggregator] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.prompt_id = prompt_id self.criteria = list(criteria or []) - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS - self.verifier_id = verifier_id or uuid.uuid4().hex def to_dict(self) -> dict: base = super().to_dict() @@ -67,16 +67,9 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> "AgentPromptResponseVerifierTaskStep": - raw_agg = data.get("score_aggregator") - return cls( - **cls._base_from_dict(data), - prompt_id=data["prompt_id"], - criteria=data.get("criteria"), - score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, - verifier_id=data.get("verifier_id"), - ) + return cls(**cls._base_from_dict(data), prompt_id=data["prompt_id"], criteria=data.get("criteria")) - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: prompt_response = next( (pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None, @@ -88,16 +81,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning( f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}" ) - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - "results": [{ - "id": "prompt_error", - "score": 0, - "result": False, - "message": f"Skipped: prompt had error_type={prompt_response.error_type}", - }], + return Verification(criteria=[{ + "id": "prompt_error", "score": 0, - } - return context + "result": False, + "message": f"Skipped: prompt had error_type={prompt_response.error_type}", + }], score=0) response = prompt_response.response or "" @@ -117,18 +106,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "result": bool(outcome["passed"]), "justification": outcome["justification"], }) - - score = aggregate_score(results, self.score_aggregator) - - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - "results": results, - "score": score, - } - logger.info( - f"Verification '{self.verifier_id}': " - f"{len(results)} criteria evaluated, score={score}" - ) - return context + return Verification(criteria=results) def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict: rtype = criterion["type"] diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py new file mode 100644 index 000000000..40cae0593 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -0,0 +1,267 @@ +"""The base every verifier step subclasses. A verifier implements ``verify``; the base scores what it returns and writes +the step's named grade to ``context.metadata["grades"][]``, beside the ``verifications`` entries today's +readers take.""" + +from __future__ import annotations + +import functools +import hashlib +import json +import logging +import uuid +from abc import abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from importlib.metadata import packages_distributions +from typing import Any, ClassVar, Optional, TypedDict + +from packaging.utils import canonicalize_name + +from agent_env.config import get_config +from agent_env.plugins import _requirements +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.thread_work import finish_on_thread +from agent_env.utils.canonical import canonical_json + +logger = logging.getLogger(__name__) + +GRADES_KEY = "grades" +VERIFICATIONS_KEY = "verifications" +# The version of the rules the base applies to every verifier's rows and pass rule; a change that can move a grade +# bumps it, and every grade records it in its verifier_settings. +SCORING_RULES = 1 +DEFAULT_MIN_PASSING_SCORE = 1.0 +OUTPUT_INLINE_BYTES = 16 * 1024 +SETTING_INLINE_BYTES = 1024 +# The step's fields left out of its verifier_settings: those that name or schedule it rather than say how it grades, and +# min_passing_score, which the grade holds itself. +_NOT_SETTINGS = ("id", "type", "version", "depends_on", "fail_task_on_error", "retry_config", "verifier_id", + "min_passing_score") +_FRAMEWORK = "agentenv-framework" + + +class VerifierStatus(str, Enum): + GRADED = "graded" + SKIPPED = "skipped" + FAILED = "failed" + + +class _GradeFields(TypedDict): + name: str + score: Optional[float] + min_passing_score: float + passed: bool + verifier_status: str + verifier_error_reasoning: Optional[str] + verifier_skip_reasoning: Optional[str] + verifier_output: Any + verifier_criteria: list[dict[str, Any]] + verifier_type: str + verifier_version: Optional[str] + verifier_settings: dict[str, Any] + + +class Grade(_GradeFields, total=False): + """A verifier step's named grade, as ``context.metadata["grades"][]`` holds it. ``verifier_status`` is a + ``VerifierStatus`` value, and ``score`` is None unless it is ``graded``; each ``verifier_criteria`` row has ``id``, + ``score``, ``passed``, ``weight`` and ``skipped``, besides any other keys the verifier gave it. The two fields here + are written only for a large output.""" + + verifier_output_ref: dict[str, Any] + verifier_output_offload_error: str + + +@dataclass +class Verification: + """What one run of a verifier found. + + ``criteria``: a row per check, read for ``id``, ``score``, ``result``, ``weight`` and ``skipped``, any other keys + kept. ``score``: the verifier's own score when it isn't the step's aggregate of the rows, which it then replaces. + ``output``: anything else worth keeping, such as a judge's justification or a test's log; a dict's keys also join + the step's legacy ``verifications`` entry, after its ``results`` and ``score``. ``skipped``: why the verifier judged + nothing. ``raise_after``: an exception the step raises once both are written, for a verdict that also fails its + step.""" + + criteria: list[dict[str, Any]] = field(default_factory=list) + score: Optional[float] = None + output: Any = None + skipped: Optional[str] = None + raise_after: Optional[BaseException] = None + + +class VerifierTaskStep(TaskStep): + """A step that grades a run. ``execute`` runs ``verify`` and writes the step's grade, and its legacy entry after + it; an exception from ``verify`` is a failed grade, raised again. Subclasses implement ``verify``, not ``execute``, + and their ``__init__`` takes and passes on ``verifier_id``, ``score_aggregator`` and ``min_passing_score``, which + ``_base_from_dict`` reads for them.""" + + step_role = StepRole.GRADER + DEFAULT_SCORE_AGGREGATOR: ClassVar[ScoreAggregator] = ScoreAggregator.ALL_PASS + + def __init__( + self, + id: str, + version: Optional[int], + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + retry_config: Optional[RetryConfig | dict] = None, + *, + verifier_id: Optional[str] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, + min_passing_score: Optional[float] = None, + ): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + retry_config=retry_config) + self.verifier_id = verifier_id or id + self.score_aggregator = ScoreAggregator(score_aggregator) if score_aggregator else self.DEFAULT_SCORE_AGGREGATOR + self.min_passing_score = DEFAULT_MIN_PASSING_SCORE if min_passing_score is None else float(min_passing_score) + + @abstractmethod + async def verify(self, context: TaskStepContext) -> Verification: + """Grade the run in ``context``.""" + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + try: + verification = await self.verify(context) + except Exception as exc: + self._write_grade(context, self._grade(context, VerifierStatus.FAILED, None, [], + error=f"{type(exc).__name__}: {exc}")) + raise + if verification.skipped is not None: + status, score = VerifierStatus.SKIPPED, None + elif verification.score is not None: + status, score = VerifierStatus.GRADED, verification.score + else: + status, score = VerifierStatus.GRADED, aggregate_score(verification.criteria, self.score_aggregator) + grade = self._grade(context, status, None if score is None else float(score), verification.criteria, + skip=verification.skipped) + grade.update(await self._output(context, verification.output)) + self._write_grade(context, grade) + entry = {"results": verification.criteria, "score": score} + if verification.skipped is not None: + entry["skipped"] = verification.skipped + if isinstance(verification.output, dict): + entry.update(verification.output) + context.metadata.setdefault(VERIFICATIONS_KEY, {})[self.verifier_id] = entry + if verification.raise_after is not None: + raise verification.raise_after + return context + + def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: + """What this step grades with in ``context``'s run: its own fields, one over ``SETTING_INLINE_BYTES`` kept as its + sha256 and size, and its ``score_aggregator`` whether or not it is its default. A subclass adds what the run + changes or decides, such as the ``step_params`` it applies or a judge's model. Pure, since it is also read at + run start.""" + fields = {k: _compact(v) for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} + return {**fields, "score_aggregator": self.score_aggregator.value} + + def config_problems(self) -> list[str]: + problems = super().config_problems() + if not 0.0 < self.min_passing_score <= 1.0: + problems.append(f"min_passing_score must be above 0 and at most 1, not {self.min_passing_score}") + return problems + + def to_dict(self) -> dict[str, Any]: + base = super().to_dict() + if self.verifier_id != self.id: + base["verifier_id"] = self.verifier_id + if self.score_aggregator is not self.DEFAULT_SCORE_AGGREGATOR: + base["score_aggregator"] = self.score_aggregator.value + if self.min_passing_score != DEFAULT_MIN_PASSING_SCORE: + base["min_passing_score"] = self.min_passing_score + return base + + @classmethod + def _base_from_dict(cls, data: dict) -> dict: + return { + **super()._base_from_dict(data), + "verifier_id": data.get("verifier_id"), + "score_aggregator": data.get("score_aggregator"), + "min_passing_score": data.get("min_passing_score"), + } + + def _grade(self, context: TaskStepContext, status: VerifierStatus, score: Optional[float], + criteria: list[dict[str, Any]], *, error: Optional[str] = None, skip: Optional[str] = None) -> Grade: + passed = status is VerifierStatus.GRADED and score >= self.min_passing_score + rows = [_row(i, row) for i, row in enumerate(criteria)] or [{ + "id": self.id, "score": score, "passed": passed, "weight": 1.0, "skipped": status is VerifierStatus.SKIPPED, + }] + return Grade( + name=self.id, + score=score, + min_passing_score=self.min_passing_score, + passed=passed, + verifier_status=status.value, + verifier_error_reasoning=error, + verifier_skip_reasoning=skip, + verifier_output=None, + verifier_criteria=rows, + verifier_type=self.type, + verifier_version=_providing_version(type(self).__module__), + verifier_settings={"scoring_rules": SCORING_RULES, **self.verifier_settings(context)}, + ) + + async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: + """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the + object store, written on a worker thread; inline with ``verifier_output_offload_error`` when that write fails, + which never fails the step.""" + if output is None: + return {} + body = json.dumps(output).encode() + if len(body) <= OUTPUT_INLINE_BYTES: + return {"verifier_output": output} + try: + url = await finish_on_thread(functools.partial(_store_output, context.instance_id, self.id, body), + f"Uploading verifier {self.id}'s output") + except Exception as exc: + logger.warning("verifier %s: kept its %d-byte output inline, the object store refused it: %s", + self.id, len(body), exc) + return {"verifier_output": output, "verifier_output_offload_error": f"{type(exc).__name__}: {exc}"} + return {"verifier_output_ref": {"object_url": url, "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest()}} + + def _write_grade(self, context: TaskStepContext, grade: Grade) -> None: + context.metadata.setdefault(GRADES_KEY, {})[self.id] = grade + logger.info("verifier %s: %s, score %s, %s", self.id, grade["verifier_status"], grade["score"], + "passed" if grade["passed"] else "not passed") + + +def _row(index: int, row: dict[str, Any]) -> dict[str, Any]: + """The grade's copy of a verifier's ``row``: its ``passed`` whether the row passes under scoring rules 1, an + explicit ``True`` ``result`` (as ALL_PASS and ANY_PASS count it), and each other key's default the one + ``aggregate_score`` reads the row with.""" + copied = {k: v for k, v in row.items() if k != "result"} + return { + **copied, + "id": row.get("id", str(index)), + "score": row.get("score", 1.0 if row.get("result") else 0.0), + "passed": row.get("result") is True, + "weight": row.get("weight", 1.0), + "skipped": row.get("skipped", False), + } + + +def _compact(value: Any) -> Any: + body = canonical_json(value) + if len(body) <= SETTING_INLINE_BYTES: + return value + return {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)} + + +def _store_output(instance_id: Optional[str], step_id: str, body: bytes) -> str: + config = get_config() + key = (f"{config.get_artifact_key_prefix()}verifier-outputs/instance_id={instance_id}/" + f"{step_id}-{uuid.uuid4().hex}.json") + return config.get_object_store().put(key, body, content_type="application/json") + + +@functools.cache +def _providing_version(module: str) -> Optional[str]: + """The version of the installed package that holds ``module``: agentenv-framework's for agent_env's own; None when + no one package does, as for a module on the path or an editable plugin.""" + if module.partition(".")[0] == "agent_env": + return _requirements.installed(_FRAMEWORK) + name = _requirements.distribution_holding(module, packages_distributions()) + return _requirements.installed(canonicalize_name(name)) if name else None diff --git a/src/agent_env/utils/canonical.py b/src/agent_env/utils/canonical.py new file mode 100644 index 000000000..1503bbc00 --- /dev/null +++ b/src/agent_env/utils/canonical.py @@ -0,0 +1,12 @@ +"""JSON written so that equal values give equal bytes, for hashing them.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any, Optional + + +def canonical_json(value: Any, default: Optional[Callable[[Any], Any]] = None) -> bytes: + """``value`` as JSON with its keys sorted and no spaces; ``default`` as for ``json.dumps``.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=default).encode() diff --git a/tst/integration/computer/computer_env_test.py b/tst/integration/computer/computer_env_test.py index 0da7874b3..69de4eea5 100644 --- a/tst/integration/computer/computer_env_test.py +++ b/tst/integration/computer/computer_env_test.py @@ -15,7 +15,6 @@ import json import platform import subprocess -import time import uuid import httpx @@ -101,17 +100,14 @@ async def test_the_image_of_this_machines_arch_boots_as_the_session_user_and_pas def test_an_env_that_changes_only_its_display_and_name_reuses_the_image(desktop): with namespace_routing(): - started = time.monotonic() other = ComputerEnv.put(id=f"{desktop.id}-other", computer=desktop.computer, driver=desktop.driver, environment_name="other", display={"width": 1024, "height": 768}) - took = time.monotonic() - started results = Env.get(other.id).metadata["composed"]["container"] first = desktop.metadata["composed"]["container"] assert results[ARCH]["reused"] is True and results[ARCH]["image"] == first[ARCH]["image"] assert results[ARCH]["status"] == "ready" and results[ARCH]["card"]["name"] == "other" assert results[ARCH]["installed"] == first[ARCH]["installed"] and results[ARCH]["emulated"] is False - assert took < 60, took #: A request to an address on the internet, around the egress proxy, and through it, printing the status and the body diff --git a/tst/integration/computer/disk/rootfs/Dockerfile b/tst/integration/computer/disk/rootfs/Dockerfile index dba0242df..583eea178 100644 --- a/tst/integration/computer/disk/rootfs/Dockerfile +++ b/tst/integration/computer/disk/rootfs/Dockerfile @@ -1,5 +1,7 @@ -# A small root filesystem, for either arch, that the disk tests make disks of. -FROM public.ecr.aws/docker/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec +# A small root filesystem, for either arch, that the disk tests make disks of. From mirror.gcr.io, which serves Docker +# Hub's busybox under the same digest for both arches, since the fast tier pulls its bases straight from the registry +# and ECR Public rate-limits the runners' shared IPs. +FROM mirror.gcr.io/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec ENV GREETING="hello disk" USER nobody WORKDIR /tmp diff --git a/tst/unit/bundle/_support.py b/tst/unit/bundle/_support.py index 88f8ec95a..c39c9896f 100644 --- a/tst/unit/bundle/_support.py +++ b/tst/unit/bundle/_support.py @@ -69,7 +69,6 @@ def read(name): loaded=load)) monkeypatch.setattr(requires_module, "load_failures", lambda: failures or {}) monkeypatch.setattr(requires_module, "packages_distributions", lambda: packages or {}) - monkeypatch.setattr(requires_module, "distribution", read) monkeypatch.setattr(_requirements, "installed", (versions or {}).get) monkeypatch.setattr(_requirements, "distribution", read) diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 21238c068..367af76dd 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -575,6 +575,24 @@ async def test_a_tolerated_verifier_that_recorded_no_score_is_missing_from_the_s assert await episode.result() == {"replies": 1.0} +@pytest.mark.asyncio +async def test_the_stored_run_holds_each_verifiers_named_grade_a_failed_one_included(teardowns): + passing = {"id": "crates", "type": "agent_prompt_response_verifier", "prompt_id": "turn", + "verifier_id": "has-crate", "criteria": [{"type": "response_contains", "needles": ["crate"]}]} + broken = {"id": "graded", "type": "agent_prompt_response_verifier", "prompt_id": "missing", + "verifier_id": "graded", "criteria": [{"type": "response_contains", "needles": ["crate"]}], + "fail_task_on_error": False} + episode = await _spec(ITEMS, ALICE, TURN, passing, broken).start() + await (await episode.next_turn()).end("Added a crate.") + + assert await episode.result() == {"crates": 1.0} + grades = Task.get_instance(episode.id).context["metadata"]["grades"] + assert {step: (g["verifier_status"], g["score"], g["passed"]) for step, g in grades.items()} == { + "crates": ("graded", 1.0, True), "graded": ("failed", None, False)} + assert grades["graded"]["verifier_error_reasoning"].startswith("StepContextError: PromptResponse") + assert grades["crates"]["verifier_criteria"][0]["passed"] is True + + BREAKS_WHILE_THE_TURN_IS_OPEN = ( ITEMS, {**ALICE, "depends_on": ["items"]}, {**TURN, "depends_on": ["alice"]}, {"id": "later", "type": "episode_test_gate", "depends_on": ["items"]}, diff --git a/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py new file mode 100644 index 000000000..22914b37f --- /dev/null +++ b/tst/unit/task_step/verifiers/test_agent_prompt_response_verifier.py @@ -0,0 +1,114 @@ +"""agent_prompt_response_verifier on the VerifierTaskStep base: its legacy entry is byte for byte what it wrote before +the move (the strings below were recorded from main at 810e27760), and it now writes a named grade beside it.""" +import asyncio +import json + +import pytest + +from agent_env.plugins import _requirements +from agent_env.task.task import recorded_scores +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep + +CRITERIA = [ + {"id": "says-done", "type": "response_contains", "needles": ["done", "ok"]}, + {"type": "response_regex_present", "pattern": "^Total: \\d+$", "flags": ["MULTILINE"], "weight": 2}, + {"id": "names-file", "type": "response_contains", "needles": ["report.csv"], "weight": 0.5}, +] + +ROWS = (r'''[{"id": "says-done", "type": "response_contains", "needles": ["done", "ok"], "criterion_index": 0, ''' + r'''"score": 1.0, "result": true, "justification": "all needles present"}, {"type": "response_regex_present", ''' + r'''"pattern": "^Total: \\d+$", "flags": ["MULTILINE"], "weight": 2, "criterion_index": 1, "score": 1.0, ''' + r'''"result": true, "justification": "pattern matched"}, {"id": "names-file", "type": "response_contains", ''' + r'''"needles": ["report.csv"], "weight": 0.5, "criterion_index": 2, "score": 0.0, "result": false, ''' + r'''"justification": "missing: ['report.csv']"}]''') +BEFORE = { + "all_pass": '{"results": ' + ROWS + ', "score": 0.0}', + "weighted_average": '{"results": ' + ROWS + ', "score": 0.8571428571428571}', + "prompt_error": ('{"results": [{"id": "prompt_error", "score": 0, "result": false, "message": "Skipped: prompt ' + 'had error_type=agent_error"}], "score": 0}'), +} +RESPONSES = {"all_pass": "done, ok\nTotal: 12\n", "weighted_average": "done ok\nTotal: 3"} + + +def _step(**kw) -> AgentPromptResponseVerifierTaskStep: + return AgentPromptResponseVerifierTaskStep(id="check", version=None, prompt_id="p1", criteria=CRITERIA, + verifier_id="vid", **kw) + + +def _run(step, response="", error_type=None) -> TaskStepContext: + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response=response, error_type=error_type)]) + asyncio.run(step.execute(context)) + return context + + +@pytest.mark.parametrize("aggregator", ["all_pass", "weighted_average"]) +def test_the_legacy_entry_is_unchanged_and_the_grade_carries_its_score(aggregator): + step = _step(score_aggregator=aggregator) + context = _run(step, RESPONSES[aggregator]) + + assert json.dumps(context.metadata["verifications"]["vid"]) == BEFORE[aggregator] + grade = context.metadata["grades"]["check"] + assert grade["score"] == json.loads(BEFORE[aggregator])["score"] + assert (grade["verifier_status"], grade["verifier_type"]) == ("graded", "agent_prompt_response_verifier") + assert grade["verifier_version"] == _requirements.installed("agentenv-framework") + assert [(row["id"], row["passed"]) for row in grade["verifier_criteria"]] == [ + ("says-done", True), ("1", True), ("names-file", False)] + assert recorded_scores([step], context) == {"check": json.loads(BEFORE[aggregator])["score"]} + + +def test_a_prompt_error_is_still_a_graded_zero(): + context = _run(_step(), error_type="agent_error") + + assert json.dumps(context.metadata["verifications"]["vid"]) == BEFORE["prompt_error"] + grade = context.metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("graded", 0.0, False) + + +def test_an_unknown_criterion_type_is_a_failed_grade_and_still_raises(): + step = AgentPromptResponseVerifierTaskStep(id="check", version=None, prompt_id="p1", + criteria=[{"type": "response_sentiment"}], verifier_id="vid") + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response="hi")]) + + with pytest.raises(ValueError, match="unknown type 'response_sentiment'"): + asyncio.run(step.execute(context)) + + assert "verifications" not in context.metadata + assert context.metadata["grades"]["check"]["verifier_status"] == "failed" + assert context.metadata["grades"]["check"]["verifier_error_reasoning"].startswith("ValueError: criterion #0") + + +def test_a_missing_prompt_response_is_a_failed_grade_and_still_raises(): + context = TaskStepContext() + + with pytest.raises(StepContextError): + asyncio.run(_step().execute(context)) + + assert context.metadata["grades"]["check"]["verifier_status"] == "failed" + + +def test_its_stored_form_is_unchanged(): + before = json.loads(r'''{"criteria": [{"id": "says-done", "needles": ["done", "ok"], "type": "response_contains"}, + {"flags": ["MULTILINE"], "pattern": "^Total: \\d+$", "type": "response_regex_present", "weight": 2}, + {"id": "names-file", "needles": ["report.csv"], "type": "response_contains", "weight": 0.5}], + "fail_task_on_error": true, "id": "check", "prompt_id": "p1", "score_aggregator": "all_pass", + "type": "agent_prompt_response_verifier", "verifier_id": "vid", "version": null}''') + + assert _step().to_dict() == before + assert AgentPromptResponseVerifierTaskStep.from_dict(before).to_dict() == before + + +def test_its_settings_are_its_own_fields(): + settings = _run(_step(), RESPONSES["all_pass"]).metadata["grades"]["check"]["verifier_settings"] + + assert settings == {"scoring_rules": 1, "prompt_id": "p1", "criteria": CRITERIA, "score_aggregator": "all_pass"} + + +def test_a_min_passing_score_round_trips_and_sets_the_pass_rule(): + step = AgentPromptResponseVerifierTaskStep.from_dict({**_step().to_dict(), "score_aggregator": "weighted_average", + "min_passing_score": 0.8}) + + grade = _run(step, RESPONSES["weighted_average"]).metadata["grades"]["check"] + + assert step.to_dict()["min_passing_score"] == 0.8 + assert (grade["min_passing_score"], grade["passed"]) == (0.8, True) diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py new file mode 100644 index 000000000..7bc7e7300 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -0,0 +1,291 @@ +"""VerifierTaskStep: the named grade it writes beside a verifier's legacy entries, for each way verify can end.""" +import asyncio +import hashlib +import json +import threading +from types import SimpleNamespace +from typing import ClassVar + +import pytest + +from agent_env.plugins import _registration +from agent_env.task.step_journal import replay_context +from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError +from agent_env.task_step.context_ops import build_context_update_ops +from agent_env.task_step.task_step import StepRole, TaskStep +from agent_env.task_step.task_steps.verifiers import ( + Grade, ScoreAggregator, Verification, VerifierTaskStep, aggregate_score, +) +from agent_env.task_step.task_steps.verifiers import verifier as verifier_module + + +class _Toy(VerifierTaskStep): + type: ClassVar[str] = "toy_verifier" + + def __init__(self, id="check", version=None, outcome=None, note="", **base): + super().__init__(id, version, **base) + self.outcome = outcome + self.note = note + + def to_dict(self): + return {**super().to_dict(), "note": self.note} + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data), note=data.get("note", "")) + + async def verify(self, context): + if isinstance(self.outcome, BaseException): + raise self.outcome + return self.outcome + + +def _run(step: VerifierTaskStep, context: TaskStepContext | None = None) -> TaskStepContext: + context = context or TaskStepContext() + asyncio.run(step.execute(context)) + return context + + +ROWS = [{"id": "a", "score": 1.0, "result": True, "why": "found it"}, {"id": "b", "score": 0.0, "result": False}] + + +def test_a_graded_verification_writes_its_named_grade_and_the_legacy_entry(): + metadata = _run(_Toy(outcome=Verification(criteria=ROWS), verifier_id="vid", note="n")).metadata + + assert metadata["verifications"] == {"vid": {"results": ROWS, "score": 0.0}} + grade = metadata["grades"]["check"] + assert grade == { + "name": "check", "score": 0.0, "min_passing_score": 1.0, "passed": False, "verifier_status": "graded", + "verifier_error_reasoning": None, "verifier_skip_reasoning": None, "verifier_output": None, + "verifier_criteria": [ + {"id": "a", "score": 1.0, "why": "found it", "passed": True, "weight": 1.0, "skipped": False}, + {"id": "b", "score": 0.0, "passed": False, "weight": 1.0, "skipped": False}, + ], + "verifier_type": "toy_verifier", "verifier_version": None, + "verifier_settings": {"scoring_rules": 1, "note": "n", "score_aggregator": "all_pass"}, + } + + +def test_the_grade_passes_at_its_min_passing_score(): + rows = [{"id": "a", "score": 1.0}, {"id": "b", "score": 0.5}] + grade = _run(_Toy(outcome=Verification(criteria=rows), score_aggregator="weighted_average", + min_passing_score=0.75)).metadata["grades"]["check"] + + assert (grade["score"], grade["min_passing_score"], grade["passed"]) == (0.75, 0.75, True) + + +def test_a_direct_score_wins_and_keeps_the_rows_as_reported(): + metadata = _run(_Toy(outcome=Verification(criteria=ROWS, score=0.4))).metadata + + assert metadata["grades"]["check"]["score"] == 0.4 + assert [row["id"] for row in metadata["grades"]["check"]["verifier_criteria"]] == ["a", "b"] + assert metadata["verifications"]["check"]["score"] == 0.4 + + +def test_a_verification_with_no_rows_gets_one_for_the_grade_itself(): + grade = _run(_Toy(outcome=Verification(score=1))).metadata["grades"]["check"] + + assert grade["score"] == 1.0 and grade["passed"] is True + assert grade["verifier_criteria"] == [{"id": "check", "score": 1.0, "passed": True, "weight": 1.0, "skipped": False}] + + +def test_a_raising_verifier_writes_a_failed_grade_no_legacy_entry_and_raises_the_same_exception(): + boom = ValueError("criterion #2 has unknown type 'x'") + context = TaskStepContext() + + with pytest.raises(ValueError) as raised: + _run(_Toy(outcome=boom), context) + + assert raised.value is boom + assert "verifications" not in context.metadata + grade = context.metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", None, False) + assert grade["verifier_error_reasoning"] == "ValueError: criterion #2 has unknown type 'x'" + assert grade["verifier_criteria"] == [{"id": "check", "score": None, "passed": False, "weight": 1.0, "skipped": False}] + + +def test_raise_after_keeps_the_verdict_graded_then_fails_the_step(): + not_passing = VerifierNotPassingError("check: 1 criterion failed") + context = TaskStepContext() + + with pytest.raises(VerifierNotPassingError) as raised: + _run(_Toy(outcome=Verification(criteria=ROWS, raise_after=not_passing)), context) + + assert raised.value is not_passing + assert context.metadata["grades"]["check"]["verifier_status"] == "graded" + assert context.metadata["verifications"]["check"] == {"results": ROWS, "score": 0.0} + + +def test_a_skip_is_its_own_status_with_its_reason(): + metadata = _run(_Toy(outcome=Verification(skipped="no graded turns"))).metadata + + grade = metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("skipped", None, False) + assert grade["verifier_skip_reasoning"] == "no graded turns" + assert grade["verifier_criteria"][0]["skipped"] is True + assert metadata["verifications"]["check"] == {"results": [], "score": None, "skipped": "no graded turns"} + + +def test_a_dict_outputs_keys_join_the_legacy_entry_after_its_results_and_score(): + output = {"command": "pytest", "exit_code": 1, "stdout_head": "1 failed"} + + metadata = _run(_Toy(outcome=Verification(criteria=ROWS, output=output), verifier_id="vid")).metadata + + assert json.dumps(metadata["verifications"]) == json.dumps({"vid": {"results": ROWS, "score": 0.0, **output}}) + assert metadata["grades"]["check"]["verifier_output"] == output + + +def test_an_output_that_isnt_a_dict_stays_off_the_legacy_entry(): + metadata = _run(_Toy(outcome=Verification(score=1, output="3 passed"))).metadata + + assert metadata["verifications"]["check"] == {"results": [], "score": 1} + assert metadata["grades"]["check"]["verifier_output"] == "3 passed" + + +@pytest.mark.parametrize("rows", [ + [{"result": True}, {"result": False, "weight": 3}], + [{"score": 0.5}, {"score": 1.0, "weight": 2}], + [{"score": 1.0, "result": None}, {"score": 0.0, "result": True}], + [{"score": 1.0, "weight": 1}, {"score": 0.0, "weight": -1}], + [{"result": "yes"}, {}], +]) +def test_the_grade_rows_pass_and_score_as_scoring_read_the_rows_the_verifier_gave(rows): + by_rule = {ScoreAggregator.ALL_PASS: all, ScoreAggregator.ANY_PASS: any} + for aggregator in ScoreAggregator: + metadata = _run(_Toy(outcome=Verification(criteria=rows), score_aggregator=aggregator)).metadata + + graded, legacy = metadata["grades"]["check"]["verifier_criteria"], metadata["verifications"]["check"] + if aggregator in by_rule: + assert legacy["score"] == (1.0 if by_rule[aggregator](row["passed"] for row in graded) else 0.0) + else: + assert aggregate_score(graded, aggregator) == legacy["score"] + assert [row["id"] for row in graded] == [str(i) for i in range(len(rows))] + assert "result" not in graded[0] and legacy["results"] == rows + + +@pytest.mark.parametrize("outcome", [Verification(criteria=ROWS), Verification(skipped="nothing to judge"), + ValueError("broken"), Verification(score=1, output={"log": "ok"})]) +def test_every_grade_holds_the_fields_its_type_names(outcome): + context = TaskStepContext() + try: + _run(_Toy(outcome=outcome), context) + except ValueError: + pass + + keys = set(context.metadata["grades"]["check"]) + assert Grade.__required_keys__ <= keys <= Grade.__required_keys__ | Grade.__optional_keys__ + + +def test_settings_are_the_steps_own_fields_with_large_ones_kept_by_digest(): + note = "x" * 2000 + step = _Toy(outcome=Verification(score=1), note=note, verifier_id="vid", min_passing_score=0.5) + + grade = _run(step).metadata["grades"]["check"] + + body = json.dumps(note, sort_keys=True, separators=(",", ":")).encode() + assert grade["verifier_settings"] == {"scoring_rules": 1, "score_aggregator": "all_pass", + "note": {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}} + assert grade["min_passing_score"] == 0.5 + + +def test_settings_name_the_aggregator_the_step_scores_with_even_at_its_default(): + for aggregator, written in ((None, "all_pass"), ("weighted_average", "weighted_average")): + settings = _run(_Toy(outcome=Verification(score=1), score_aggregator=aggregator)).metadata["grades"][ + "check"]["verifier_settings"] + + assert settings["score_aggregator"] == written + + +def test_base_fields_are_serialized_only_when_not_default_and_round_trip(): + assert _Toy().to_dict() == {"id": "check", "type": "toy_verifier", "version": None, "fail_task_on_error": True, + "note": ""} + step = _Toy(verifier_id="vid", score_aggregator="weighted_average", min_passing_score=0.5) + data = step.to_dict() + assert (data["verifier_id"], data["score_aggregator"], data["min_passing_score"]) == ("vid", "weighted_average", 0.5) + again = _Toy.from_dict(data) + assert (again.verifier_id, again.score_aggregator, again.min_passing_score) == ( + "vid", ScoreAggregator.WEIGHTED_AVERAGE, 0.5) + assert _Toy.from_dict(_Toy().to_dict()).verifier_id == "check" + + +@pytest.mark.parametrize("bad", [0, -0.5, 1.5, float("nan")]) +def test_a_min_passing_score_outside_zero_to_one_is_a_config_problem(bad): + assert _Toy(min_passing_score=bad).config_problems() == [ + f"min_passing_score must be above 0 and at most 1, not {float(bad)}"] + assert _Toy(min_passing_score=1).config_problems() == [] + + +class _Store: + def __init__(self, fail=False): + self.fail, self.puts, self.threads = fail, {}, [] + + def put(self, key, body, content_type=None): + self.threads.append(threading.current_thread()) + if self.fail: + raise ConnectionError("store unreachable") + self.puts[key] = (body, content_type) + return f"file:///objects/{key}" + + +def _with_store(monkeypatch, store): + monkeypatch.setattr(verifier_module, "get_config", lambda: SimpleNamespace( + get_artifact_key_prefix=lambda: "pfx/", get_object_store=lambda: store)) + + +def test_a_small_output_stays_on_the_grade(): + grade = _run(_Toy(outcome=Verification(score=1, output={"log": "ok"}))).metadata["grades"]["check"] + + assert grade["verifier_output"] == {"log": "ok"} and "verifier_output_ref" not in grade + + +def test_a_large_output_goes_to_the_object_store_from_a_worker_thread(monkeypatch): + store = _Store() + _with_store(monkeypatch, store) + output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} + + context = _run(_Toy(outcome=Verification(score=1, output=output)), TaskStepContext(instance_id="i1")) + grade = context.metadata["grades"]["check"] + + [(key, (body, content_type))] = store.puts.items() + assert store.threads != [threading.main_thread()] + assert key.startswith("pfx/verifier-outputs/instance_id=i1/check-") and content_type == "application/json" + assert json.loads(body) == output + assert grade["verifier_output"] is None + assert grade["verifier_output_ref"] == {"object_url": f"file:///objects/{key}", "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest()} + assert context.metadata["verifications"]["check"]["log"] == output["log"] + + +def test_an_output_the_store_refuses_stays_inline_and_the_step_still_passes(monkeypatch): + _with_store(monkeypatch, _Store(fail=True)) + output = {"log": "x" * (verifier_module.OUTPUT_INLINE_BYTES + 1)} + + grade = _run(_Toy(outcome=Verification(score=1, output=output))).metadata["grades"]["check"] + + assert grade["verifier_output"] == output and grade["passed"] is True + assert grade["verifier_output_offload_error"] == "ConnectionError: store unreachable" + + +def test_the_grade_and_legacy_entry_survive_the_step_journal(): + post = _run(_Toy(outcome=Verification(criteria=ROWS, output={"log": "ok"}), verifier_id="vid")) + + ops = build_context_update_ops(None, post).to_journal_dict() + stored = replay_context([{"step_id": "check", "seq": 1, "ops": ops}], []) + + assert stored["metadata"]["grades"] == post.metadata["grades"] + assert stored["metadata"]["verifications"] == post.metadata["verifications"] + + +def test_a_verifier_on_the_base_is_a_grader(): + assert _Toy.step_role is StepRole.GRADER and "step_role" not in _Toy().to_dict() + + +def test_a_verifier_must_implement_verify(): + class _NoVerify(VerifierTaskStep): + type: ClassVar[str] = "no_verify" + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + assert _registration.unimplemented(_NoVerify, TaskStep, ("from_dict",)).endswith("_NoVerify must implement verify")