From c7ffd81edf1a864147600658e3a89e55a8edc60d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 09:59:57 -0800 Subject: [PATCH 01/10] feat(task): agent-env task lint A task that can't run or grade correctly is refused before any agent runs: a field its step doesn't read, a step's own config problems, a headline that grades nothing, and an aggregate source no verifier writes. What may be wrong is a warning that blocks nothing: a step's config_warnings, a store entity read with no version, a task with no verifier, or several verifiers and no headline. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/bundle/plan.py | 25 +- src/agent_env/bundle/register.py | 7 +- src/agent_env/bundle/run.py | 6 +- src/agent_env/cli/eval/run.py | 15 + src/agent_env/cli/task/__init__.py | 2 + src/agent_env/cli/task/create.py | 18 +- src/agent_env/cli/task/lint.py | 96 +++++++ src/agent_env/cli/task/validate.py | 13 +- src/agent_env/task/lint.py | 211 ++++++++++++++ src/agent_env/task_step/task_step.py | 5 + .../agent_prompt_response_verifier.py | 30 ++ .../run_container_unit_tests_verifier.py | 17 ++ .../task_steps/verifiers/verify_sandbox.py | 32 +++ tst/integration/cli/run_bundle_test.py | 24 +- tst/unit/bundle/preflight_test.py | 7 +- tst/unit/bundle/run_test.py | 7 +- tst/unit/cli/eval_run_teardown_test.py | 2 +- tst/unit/cli/run_role_models_test.py | 2 +- tst/unit/cli/submit_runs_test.py | 2 +- tst/unit/cli/test_task_create_preflight.py | 25 ++ tst/unit/task/lint_test.py | 272 ++++++++++++++++++ 22 files changed, 795 insertions(+), 25 deletions(-) create mode 100644 src/agent_env/cli/task/lint.py create mode 100644 src/agent_env/task/lint.py create mode 100644 tst/unit/task/lint_test.py diff --git a/AGENTS.md b/AGENTS.md index d6bbf6649..968203bad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. `task/lint.py` is `agent-env task lint` (a bundle folder, a task file or a stored task): an error is what a task gets wrong before it runs (a field its step doesn't read, a step's own `config_problems`, a headline that grades nothing, an aggregate source no verifier writes) and refuses `agent-env run`, `bundle register`, `task create`, `task validate` and `eval run`; a warning is what may be wrong (a step's `config_warnings`, a store entity read with no version, no verifier, several and no headline) and is only printed. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index f8accd6e3..1ec64c843 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -57,6 +57,7 @@ from agent_env.store.base import NotFoundError from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.routing import namespace_routing_enabled +from agent_env.task.lint import Severity, lint from agent_env.task.task import Task from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep @@ -198,6 +199,7 @@ class Plan: alias_uses: Mapping[EntityKind, Mapping[str, AliasUse]] = field(default_factory=dict) agent: AgentOverride | None = None # the agent --agent names for the solver variant: Varied | None = None # what --set changes, written under its variant's ids + lint_warnings: tuple[str, ...] = () # each selected task's task lint warnings, as ": warning: ..." def bundle_of(self, entry: BundleEntry) -> Bundle: """The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's.""" @@ -248,6 +250,7 @@ def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, self.resolved = resolved self.default_agents = default_agents self.problems: list[str] = [] + self.lint_warnings: list[str] = [] self.nodes: dict[_Key, ResolvedEntry | BundleImage] = {} self.rank: dict[_Key, int] = {} self.edges: dict[_Key, list[_Key]] = {} @@ -575,7 +578,7 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr store_refs, store_latest = self._check_store_refs(closure) if override is not None: override = replace(override, version=store_latest.get((EntityKind.AGENT, override.agent_id))) - self._build_tasks(closure) + self._build_tasks(closure, linted=True) self._check_files(closure) self._check_cli_loads(closure) self._check_env_types(closure) @@ -587,7 +590,8 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr writes = self._ordered(closure) return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest, dict(self.origins), self._published(writes), - {kind: dict(uses) for kind, uses in self.alias_uses.items()}, override) + {kind: dict(uses) for kind, uses in self.alias_uses.items()}, override, + lint_warnings=tuple(self.lint_warnings)) def check(self) -> None: """The checks a run makes before it reads a store. A bundle with nothing to run passes only when its @@ -1122,8 +1126,11 @@ def _store_entity(self, ref: Reference, store_latest: dict[tuple[EntityKind, str except (NotFoundError, ValueError, KeyError, TypeError): return None # reported by the store ref check - def _build_tasks(self, closure: set[_Key]) -> None: - """Build each task's steps the way a write will, so a bad step fails before anything is written.""" + def _build_tasks(self, closure: set[_Key], *, linted: bool = False) -> None: + """Build each task's steps the way a write will, so a bad step fails before anything is written. ``linted``, + for a run or registration, adds task lint: its errors are problems and its warnings ``lint_warnings``. The + store-free check leaves it out, since a folder ``bundle init`` writes leaves user-scoped fields for its runner + to fill in.""" registry = get_task_step_registry() for key in sorted(closure, key=self.rank.__getitem__): source = self.nodes[key] @@ -1140,10 +1147,16 @@ def _build_tasks(self, closure: set[_Key]) -> None: self.steps[key] = steps if not failed: try: - Task(id=source.entry.id, version=None, steps=steps, - headline_verifiers=source.entry.headline_verifiers) + task = Task(id=source.entry.id, version=None, steps=steps, + headline_verifiers=source.entry.headline_verifiers) except ValueError as e: self.problems.append(f"{self._path(source.entry)}: {e}") + continue + for finding in lint(task, source.config, unpinned=False) if linted else (): + if finding.severity is Severity.ERROR: + self.problems.append(f"{self._path(source.entry)}: {finding.text}") + else: + self.lint_warnings.append(f"{self._path(source.entry)}: {finding}") # Order diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index bb8b75551..19bac4eaf 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -115,6 +115,8 @@ def register_bundle( raise BundleError(problems) requires = tuple(declare(resolved, declaring or declaring_for(bundle.root, installed=False, dry_run=dry_run))) say = on_progress or (lambda line: None) + for warning in plan.lint_warnings: + say(f" {warning}") materialization = materialize( registered, dry_run=dry_run, ledger=ledger, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), @@ -241,8 +243,9 @@ def _slug(name: str) -> str: # The fields of a Plan that hold no ``@local`` id, so a translated plan keeps them: the store's latest versions, by store -# id, and the agent ``--agent`` names, which a registration plans without. -_KEPT = frozenset({"store_latest", "agent", "variant"}) +# id, the agent ``--agent`` names, which a registration plans without, and task lint's warnings, which name the bundle's +# files. +_KEPT = frozenset({"store_latest", "agent", "variant", "lint_warnings"}) def translated(plan: Plan, ids: Mapping[str, str]) -> Plan: diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 15707cbe6..7521d1e00 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -484,8 +484,8 @@ def refuse_store_tasks(plan: Plan) -> None: def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> None: """Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, the - ids what --set changes is written under, and what plugin packages the bundle uses that its bundle.toml doesn't - require, as ``declaring`` says (``declare``).""" + ids what --set changes is written under, what plugin packages the bundle uses that its bundle.toml doesn't + require, as ``declaring`` says (``declare``), and each task's lint warnings.""" for use in (use for uses in plan.alias_uses.values() for use in uses.values()): say(_aliased(use)) if plan.agent is not None: @@ -498,6 +498,8 @@ def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> N say(f"{variant.typed}: {_changed(plan)} written under ids ending in ({variant.label})") for line in declare(plan.bundle, declaring): say(line) + for warning in plan.lint_warnings: + say(warning) def _changed(plan: Plan) -> str: diff --git a/src/agent_env/cli/eval/run.py b/src/agent_env/cli/eval/run.py index cbba1698f..a34f260c0 100644 --- a/src/agent_env/cli/eval/run.py +++ b/src/agent_env/cli/eval/run.py @@ -44,6 +44,7 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, from agent_env.eval import Eval from agent_env.task import Task + from agent_env.task.lint import Severity, lint click.echo(f"Fetching eval: id={eval_id} version={eval_version or 'latest'}...") eval_obj = Eval.get(eval_id, version=eval_version) @@ -56,6 +57,20 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, click.echo(f" Loaded task: {task.id} v{task.version} ({len(task.steps)} steps){version_note}") tasks.append(task) + refused = [] + for task in tasks: + for finding in lint(task): + line = f"{task.id} v{task.version}: {finding.text}" + if finding.severity is Severity.ERROR: + refused.append(line) + else: + click.echo(f" Warning: {line}") + if refused: + click.echo(f"Task lint found {len(refused)} error(s), so no run started:", err=True) + for line in refused: + click.echo(f" - {line}", err=True) + raise SystemExit(1) + print_banner() total_runs = len(tasks) * k diff --git a/src/agent_env/cli/task/__init__.py b/src/agent_env/cli/task/__init__.py index 41541af66..e99bd7b44 100644 --- a/src/agent_env/cli/task/__init__.py +++ b/src/agent_env/cli/task/__init__.py @@ -3,6 +3,7 @@ from .create import create from .get import get from .get_instance import get_instance +from .lint import lint from .run import run, run_batch from .teardown import teardown from .validate import validate @@ -17,6 +18,7 @@ def task(): task.add_command(create) task.add_command(get) task.add_command(get_instance) +task.add_command(lint) task.add_command(run) task.add_command(run_batch) task.add_command(teardown) diff --git a/src/agent_env/cli/task/create.py b/src/agent_env/cli/task/create.py index 455dc5312..3b033e43b 100644 --- a/src/agent_env/cli/task/create.py +++ b/src/agent_env/cli/task/create.py @@ -29,6 +29,7 @@ def create(filepath: Path, task_id: str, skip_validation: bool): """ from agent_env.bundle.parse import task_file_parts from agent_env.task import Task + from agent_env.task.lint import lint, unknown_fields click.echo(f"Reading steps from {filepath}...") with open(filepath) as f: @@ -39,13 +40,17 @@ def create(filepath: Path, task_id: str, skip_validation: bool): click.echo(f"Error: {e}", err=True) raise SystemExit(1) + if unknown := list(unknown_fields(steps)): + for finding in unknown: + click.echo(f"Error: {finding.text}", err=True) + raise SystemExit(1) built = _build_steps(steps) try: checked = Task(id=task_id, version=None, steps=[s for _, s in built], headline_verifiers=headlines) except ValueError as e: click.echo(f"Error: {e}", err=True) raise SystemExit(1) - _report_preflight(checked, skip_validation) + _report_preflight(checked, skip_validation, lint(checked, config=False)) task_steps = [] for i, (kwargs, step) in enumerate(built): @@ -90,8 +95,15 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]: return built -def _report_preflight(task, skip_validation: bool) -> None: - problems = task.preflight() +def _report_preflight(task, skip_validation: bool, findings) -> None: + """Refuse ``task`` for its lint errors and preflight problems, unless ``skip_validation``; print its lint warnings + either way. ``findings`` leave out ``config_problems``, which the preflight reports.""" + from agent_env.task.lint import Severity + + for finding in findings: + if finding.severity is Severity.WARNING: + click.echo(f"Warning: {finding.text}", err=True) + problems = [finding.text for finding in findings if finding.severity is Severity.ERROR] + task.preflight() if not problems: return click.echo(f"Preflight found {len(problems)} problem(s):", err=True) diff --git a/src/agent_env/cli/task/lint.py b/src/agent_env/cli/task/lint.py new file mode 100644 index 000000000..f2f613661 --- /dev/null +++ b/src/agent_env/cli/task/lint.py @@ -0,0 +1,96 @@ +"""Check a task for what would make it run or grade wrong, before any agent runs.""" + +import json +from pathlib import Path + +import click + + +@click.command() +@click.argument("path", required=False, type=click.Path(exists=True, path_type=Path)) +@click.option("--id", "task_id", default=None, help="A stored task's id, in place of PATH") +@click.option("--version", "task_version", default=None, type=int, help="Its version (defaults to latest)") +def lint(path: Path | None, task_id: str | None, task_version: int | None): + """Lint a task: a bundle folder's tasks, one task file, or a stored task (--id). + + \b + Errors are configs that can't run or grade correctly: an unknown field, a step's + own config problems, a headline that isn't a verifier, an aggregate source no + verifier writes. They also refuse agent-env run, bundle register, task create and + eval run. Warnings block nothing. Exits non-zero when there are errors. Reads no + store, but a bundle folder is planned as a run plans it. + """ + if (path is None) == (task_id is None): + raise click.UsageError("give a bundle folder or task file, or --id, but not both") + if task_id is not None: + _report(*_stored(task_id, task_version)) + elif path.is_dir(): + _report(*_bundle(path)) + elif path.parent.name == "tasks" and path.suffix == ".json": + _report(*_bundle(path.parent.parent, path.stem)) + else: + _report(*_file(path)) + + +def _stored(task_id: str, task_version: int | None) -> tuple[list[str], list[str]]: + from agent_env.task import Task + from agent_env.task.lint import Severity, lint as lint_task + + task = Task.get(task_id, task_version) + findings = lint_task(task) + where = f"{task.id} v{task.version}" + return ([f"{where}: {f.text}" for f in findings if f.severity is Severity.ERROR], + [f"{where}: {f}" for f in findings if f.severity is Severity.WARNING]) + + +def _bundle(root: Path, task: str | None = None) -> tuple[list[str], list[str]]: + """A bundle's tasks as ``agent-env run`` plans them, so its references resolve as a run resolves them.""" + from agent_env.bundle import BundleError, BundleKind, parse_bundle + from agent_env.bundle.installed import installed_bundles + from agent_env.bundle.plan import plan_bundle + from agent_env.bundle.resolve import resolve_bundle + from agent_env.store.routing import namespace_routing + + with namespace_routing(): + bundle = parse_bundle(root) + names = [task] if task else [entry.name for entry in bundle.entries if entry.kind is BundleKind.TASK] + try: + plan = plan_bundle(resolve_bundle(bundle), tasks=names, installed=installed_bundles) + except BundleError as e: + return list(e.problems), [] + return [], list(plan.lint_warnings) + + +def _file(path: Path) -> tuple[list[str], list[str]]: + """A task file on its own: its references name store ids, as ``task create`` reads them.""" + from agent_env.bundle.parse import task_file_parts + from agent_env.bundle.resolve import build_step + from agent_env.task import Task + from agent_env.task.lint import Severity, lint as lint_task, unknown_fields + + try: + steps, headlines = task_file_parts(json.loads(path.read_text())) + except (ValueError, json.JSONDecodeError) as e: + return [f"{path}: {e}"], [] + if unknown := list(unknown_fields(steps)): + return [f"{path}: {f.text}" for f in unknown], [] + try: + task = Task(id=path.stem, version=None, steps=[build_step(step) for step in steps], + headline_verifiers=headlines) + except Exception as e: + return [f"{path}: {e}"], [] + findings = lint_task(task, steps) + return ([f"{path}: {f.text}" for f in findings if f.severity is Severity.ERROR], + [f"{path}: {f}" for f in findings if f.severity is Severity.WARNING]) + + +def _report(errors: list[str], warnings: list[str]) -> None: + for line in warnings: + click.echo(line) + if errors: + click.echo(f"{len(errors)} error{'s' * (len(errors) != 1)}:", err=True) + for line in errors: + click.echo(f" - {line}", err=True) + raise SystemExit(1) + count = f"{len(warnings)} warning{'s' * (len(warnings) != 1)}" if warnings else "no problems" + click.echo(f"task lint: {count}") diff --git a/src/agent_env/cli/task/validate.py b/src/agent_env/cli/task/validate.py index 75ba10121..5737ec758 100644 --- a/src/agent_env/cli/task/validate.py +++ b/src/agent_env/cli/task/validate.py @@ -9,15 +9,20 @@ def validate(task_id: str, task_version: int | None): """Validate a stored task's step config without deploying anything. - Reports steps whose script artifact does not resolve, or resolves to the wrong - type, and deploy_env steps that couldn't deploy their env. Exits non-zero - when there are problems, so it can gate CI. + Reports what task lint finds, steps whose script artifact does not resolve, or + resolves to the wrong type, and deploy_env steps that couldn't deploy their env. + Exits non-zero when there are problems (lint's warnings aren't), so it can gate CI. """ from agent_env.task import Task + from agent_env.task.lint import Severity, lint from agent_env.task_step.task_step import TaskStep task = Task.get(task_id, task_version) - problems = task.preflight() + findings = lint(task, config=False) + for finding in findings: + if finding.severity is Severity.WARNING: + click.echo(f"{task.id} v{task.version}: {finding}", err=True) + problems = [finding.text for finding in findings if finding.severity is Severity.ERROR] + task.preflight() # Most step types have no preflight, so report coverage rather than implying a full check. checked = sum(1 for s in task.steps if type(s).preflight is not TaskStep.preflight) if problems: diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py new file mode 100644 index 000000000..f1245a1ab --- /dev/null +++ b/src/agent_env/task/lint.py @@ -0,0 +1,211 @@ +"""What a task gets wrong before any agent runs (``agent-env task lint``). An error is a config that can't run or grade +correctly, and refuses the run; everything else is a warning, which blocks nothing.""" + +from __future__ import annotations + +import copy +import difflib +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterable, Iterator, Mapping, Sequence + +from agent_env.entity.refs import RefRole, ref_sites +from agent_env.store.ids import is_local_id +from agent_env.task.task import Task +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import TaskStep, attach_base_fields +from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, trajectory_mistakes_key +from agent_env.task_step.task_steps.verifiers.verifier import VerifierTaskStep + + +class Severity(str, Enum): + ERROR = "error" + WARNING = "warning" + + +@dataclass(frozen=True) +class Finding: + severity: Severity + rule: str + message: str + step_id: str | None = None + + @property + def text(self) -> str: + """The message with the step it's about, as a refusal lists it.""" + return f"step {self.step_id!r}: {self.message}" if self.step_id is not None else self.message + + def __str__(self) -> str: + return f"{self.severity.value}: {self.text}" + + +def lint(task: Task, step_dicts: Sequence[Mapping[str, Any]] | None = None, *, config: bool = True, + unpinned: bool = True) -> list[Finding]: + """``task``'s findings, errors first. ``step_dicts``, the task's steps as a file holds them, adds the unknown-field + rule, which a stored task can't break since storing drops unknown fields. ``config`` adds each step's own + ``config_problems`` as errors; a caller that runs ``Task.preflight`` already has them. ``unpinned`` warns for a store + entity read with no version; a bundle's plan leaves it out, since a plan records each version it reads and a + registration pins it. Reads no store.""" + findings = [ + *(unknown_fields(step_dicts) if step_dicts is not None else ()), + *(_step_configs(task.steps) if config else ()), + *_step_warnings(task.steps), + *_headlines(task), + *_aggregate_sources(task.steps), + *(_unpinned(task.steps) if unpinned else ()), + *_coverage(task), + ] + return sorted(findings, key=lambda finding: finding.severity is not Severity.ERROR) + + +def errors(findings: Iterable[Finding]) -> list[Finding]: + return [finding for finding in findings if finding.severity is Severity.ERROR] + + +def is_verifier(step: TaskStep) -> bool: + """A step that grades: one on the VerifierTaskStep base, or one that still writes a legacy entry under its + ``verifier_id`` itself, as agentenvhub-sdk's do until they move onto the base.""" + return isinstance(step, VerifierTaskStep) or bool(getattr(step, "verifier_id", None)) + + +class _Reads(dict): + """A step dict that records each key its type's ``from_dict`` asks for, read or absent; spreading or iterating it + counts as reading every key.""" + + def __init__(self, data: Mapping[str, Any]): + super().__init__(data) + self.asked: set[str] = set() + + def __getitem__(self, key): + self.asked.add(key) + return super().__getitem__(key) + + def get(self, key, default=None): + self.asked.add(key) + return super().get(key, default) + + def __contains__(self, key): + self.asked.add(key) + return super().__contains__(key) + + def pop(self, key, *default): + self.asked.add(key) + return super().pop(key, *default) + + def setdefault(self, key, default=None): + self.asked.add(key) + return super().setdefault(key, default) + + def __iter__(self): + self.asked.update(super().keys()) + return super().__iter__() + + def keys(self): + self.asked.update(super().keys()) + return super().keys() + + def items(self): + self.asked.update(super().keys()) + return super().items() + + def values(self): + self.asked.update(super().keys()) + return super().values() + + +def unknown_fields(step_dicts: Sequence[Mapping[str, Any]]) -> Iterator[Finding]: + """Each key of a step that its type never asks for when it reads the step, with the nearest field it asks for.""" + registry = get_task_step_registry() + for step in step_dicts: + cls = registry.get(step.get("type")) + if cls is None: + continue + tracked = _Reads(copy.deepcopy(dict(step))) + try: + attach_base_fields(cls.from_dict(tracked), tracked) + except Exception: + continue # a step that can't be read is reported where the task is built + for key in sorted(set(step) - tracked.asked - {"type"}): + near = difflib.get_close_matches(key, sorted(tracked.asked - set(step)), n=1) + hint = f" (did you mean {near[0]!r}?)" if near else "" + yield Finding(Severity.ERROR, "unknown-field", + f"{cls.type} doesn't read {key!r}{hint}, so it would be ignored", step.get("id")) + + +def _step_configs(steps: Sequence[TaskStep]) -> Iterator[Finding]: + for step in steps: + for problem in step.config_problems(): + yield Finding(Severity.ERROR, "step-config", problem, step.id) + + +def _step_warnings(steps: Sequence[TaskStep]) -> Iterator[Finding]: + for step in steps: + for doubt in step.config_warnings(): + yield Finding(Severity.WARNING, "step-config", doubt, step.id) + + +def _headlines(task: Task) -> Iterator[Finding]: + by_id = {step.id: step for step in task.steps} + for headline in task.headline_verifiers or (): + step = by_id.get(headline) + if step is None: + continue # Task refuses it when it's built + if not is_verifier(step): + yield Finding(Severity.ERROR, "headline-not-a-verifier", + f"headline_verifiers names {headline!r}, a {step.type} step, which grades nothing") + elif not isinstance(step, VerifierTaskStep): + yield Finding(Severity.WARNING, "headline-without-grade", + f"headline_verifiers names {headline!r}, a {step.type} step that writes no named grade yet, " + "so a run's result will read it as ungraded") + + +def _aggregate_sources(steps: Sequence[TaskStep]) -> Iterator[Finding]: + """An aggregate source that no verifier in the task writes, which fails the aggregate when it grades.""" + written = set() + for step in steps: + if is_verifier(step): + written.add(step.verifier_id) + if (isinstance(step, RubricsVerifierTaskStep) + and step.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES): + written.add(trajectory_mistakes_key(step.verifier_id)) + for step in steps: + if isinstance(step, AggregateVerifiersTaskStep): + missing = [source for source in step.verifier_ids if source not in written] + if missing: + yield Finding(Severity.ERROR, "aggregate-source", + f"verifier_ids names {missing!r}, which no verifier in this task writes (it has " + f"{sorted(written)})", step.id) + + +def _unpinned(steps: Sequence[TaskStep]) -> Iterator[Finding]: + """A store entity a step reads with no version, so each run takes whatever is latest. A bundle's own entities and + installed bundles' are local ids, which their bundles write and pin.""" + for step in steps: + for site in ref_sites(type(step).entity_refs or (), step.to_dict()): + if (site.ref.role is RefRole.INPUT and site.version_key is not None and site.version is None + and isinstance(site.value, str) and site.value and not is_local_id(site.value)): + yield Finding(Severity.WARNING, "unpinned-reference", + f"{site.path} names {site.ref.kind.value} {site.value!r} with no version, so each run " + f"takes its latest; set {site.version_key} to pin it", step.id) + + +def _coverage(task: Task) -> Iterator[Finding]: + verifiers = [step for step in task.steps if is_verifier(step)] + if not verifiers: + yield Finding(Severity.WARNING, "no-verifier", "no step grades this task, so a run has no score") + elif len(verifiers) > 1 and not task.headline_verifiers: + yield Finding(Severity.WARNING, "no-headlines", _no_headlines(verifiers)) + + +def _no_headlines(verifiers: list[TaskStep]) -> str: + """Several verifiers and no headline: every grade is top-line. Name the aggregate that covers the others, if one + does.""" + ids = {step.verifier_id for step in verifiers} + covering = [step.id for step in verifiers if isinstance(step, AggregateVerifiersTaskStep) + and ids - {step.verifier_id} <= set(step.verifier_ids)] + suggestion = (f"; {covering[0]!r} aggregates the others, so name it in headline_verifiers" if covering + else "; name the one that is its reward in headline_verifiers") + return (f"{len(verifiers)} verifiers and no headline_verifiers, so every grade is top-line and anything that needs " + f"one number must name one{suggestion}") diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index 1540d5ee4..141b77221 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -158,6 +158,11 @@ def config_problems(self) -> list[str]: ``preflight`` is, and also for a step whose preflight a dry run skips.""" return [] + def config_warnings(self) -> list[str]: + """Doubts about this step's own fields that don't stop it running, found without reading a store; empty when + fine. Task lint reports them as warnings, as it reports ``config_problems`` as errors.""" + return [] + def task_problems(self, steps: list["TaskStep"]) -> list[str]: """Problems this step has with the task's other steps (``steps``: all of them, this one included, in order), found without reading a store; empty when fine. A task with any is refused when it loads.""" diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index d27084736..c9406fbae 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -65,6 +65,14 @@ def to_dict(self) -> dict: base["verifier_id"] = self.verifier_id return base + def config_problems(self) -> list[str]: + """The criteria a run would refuse when it grades: an unknown type, an empty ``needles`` or ``pattern``, a + pattern that doesn't compile, or a flag ``re`` doesn't have.""" + problems = super().config_problems() + for idx, criterion in enumerate(self.criteria): + problems.extend(f"criterion #{idx}: {problem}" for problem in _criterion_problems(criterion)) + return problems + @classmethod def from_dict(cls, data: dict) -> "AgentPromptResponseVerifierTaskStep": return cls(**cls._base_from_dict(data), prompt_id=data["prompt_id"], criteria=data.get("criteria")) @@ -143,3 +151,25 @@ def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict: "justification": "pattern matched" if passed else f"no match for pattern {pattern!r}", } raise RuntimeError(f"unhandled criterion type {rtype}") + + +def _criterion_problems(criterion: dict) -> list[str]: + rtype = criterion.get("type") + if rtype not in _HANDLED_TYPES: + return [f"unknown type {rtype!r}; handled types: {sorted(_HANDLED_TYPES)}"] + if rtype == "response_contains": + return [] if criterion.get("needles") else ["response_contains needs non-empty needles"] + pattern = criterion.get("pattern") + if not pattern: + return ["response_regex_present needs a non-empty pattern"] + names = criterion.get("flags") + if names is not None and (unknown := [n for n in names if not isinstance(getattr(re, str(n), None), re.RegexFlag)]): + return [f"flags {unknown!r} aren't re flags"] + flags = re.DOTALL if names is None else 0 + for name in names or (): + flags |= getattr(re, name) + try: + re.compile(pattern, flags) + except re.error as e: + return [f"pattern {pattern!r} doesn't compile: {e}"] + return [] diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 9ebf72091..abe6bf11e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -145,6 +145,23 @@ def __init__( raise ValueError(f"run_container_unit_tests_verifier '{id}': result paths in env '{env_id}' are relative " f"to its session user's home, not {absolute}") + def config_problems(self) -> list[str]: + """A negative ``timeout_sec``, which ``timeout`` refuses, so the tests never run.""" + problems = super().config_problems() + if self.timeout_sec < 0: + problems.append(f"timeout_sec {self.timeout_sec} is negative; give the tests at least 1 s") + return problems + + def config_warnings(self) -> list[str]: + """What runs without testing anything: an empty ``command``, which exits 0 and so passes without a + ``reward_path``, and a ``timeout_sec`` of 0, which ``timeout`` reads as no limit.""" + warnings = super().config_warnings() + if not self.command.strip() and self.reward_path is None: + warnings.append("command is empty, so it exits 0 and the tests pass without running anything") + if self.timeout_sec == 0: + warnings.append("timeout_sec 0 means no time limit, so a hung test runs until the sandbox's own TTL") + return warnings + def to_dict(self) -> dict: base = super().to_dict() base["sandbox_name"] = self.sandbox_name diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index d73ae15e3..515f92655 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -103,6 +103,38 @@ def to_dict(self) -> dict: base["verifier_id"] = self.verifier_id return base + def config_problems(self) -> list[str]: + """The criteria that fail their row whatever the sandbox holds: a probe with no paths, a ``probe_file_contains`` + pattern that doesn't compile, a ``bash_cmd_succeeds`` with no command.""" + problems = super().config_problems() + for idx, criterion in enumerate(self.criteria): + rtype = criterion.get("type") + paths = criterion.get("paths") or [] + if rtype in ("probe_file_exists", "probe_dir_exists", "probe_file_contains") and not paths: + problems.append(f"criterion #{idx}: {rtype} needs paths") + if rtype == "probe_file_contains": + pattern = criterion.get("expected") or criterion.get("value") or "" + try: + re.compile(pattern) + except re.error as e: + problems.append(f"criterion #{idx}: expected {pattern!r} doesn't compile as a regex: {e}") + if rtype == "bash_cmd_succeeds" and not (criterion.get("bash_cmd") or "").strip(): + problems.append(f"criterion #{idx}: bash_cmd_succeeds needs a bash_cmd") + return problems + + def config_warnings(self) -> list[str]: + """The criteria a run grades as nothing: a type this step doesn't handle, which it skips, and a + ``probe_file_contains`` with no expected text, which any file passes.""" + warnings = super().config_warnings() + for idx, criterion in enumerate(self.criteria): + rtype = criterion.get("type") + if rtype not in _HANDLED_TYPES: + warnings.append(f"criterion #{idx}: type {rtype!r} isn't one verify_sandbox checks, so a run skips " + f"it; it checks {sorted(_HANDLED_TYPES)}") + elif rtype == "probe_file_contains" and not (criterion.get("expected") or criterion.get("value")): + warnings.append(f"criterion #{idx}: probe_file_contains has no expected text, so any file passes") + return warnings + @classmethod def from_dict(cls, data: dict) -> "VerifySandboxTaskStep": return cls( diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 22b58e4ed..c118d5062 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -20,7 +20,7 @@ from click.testing import CliRunner import agent_env -from agent_env.artifact import FileArtifactUniverse +from agent_env.artifact import Artifact, FileArtifactUniverse from agent_env.artifact.store import reset_artifact_store from agent_env.bundle.hub import HubSource from agent_env.bundle.installed import find_bundle @@ -35,6 +35,7 @@ ) from agent_env.providers.sandbox_providers.sandbox_provider import reset_sandbox_provider from agent_env.store import Filter, LocalSqliteDocumentStore +from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing from agent_env.task import Task from tst.integration.utils.registry import REGISTRY_IMAGE @@ -179,6 +180,27 @@ def test_a_task_file_naming_its_headline_verifiers_runs_and_its_stored_task_keep assert Task.get("@local/~/my-hello/hello", 1).headline_verifiers == ["hello"] +def test_a_task_file_with_a_field_its_step_doesnt_read_is_refused_before_anything_is_written(bundle, quiet_logs): + steps = json.loads((bundle / "tasks/hello.json").read_text()) + (bundle / "tasks/hello.json").write_text(json.dumps([*steps[:2], {**steps[2], "min_passing_scor": 1.0}])) + + refused = CliRunner().invoke(cli, ["run", str(bundle)]) + + assert refused.exit_code != 0 + assert ("tasks/hello.json: step 'hello': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?), so it would be ignored") in refused.output + with namespace_routing(): + for get in (lambda: Task.get("@local/~/my-hello/hello"), lambda: Artifact.get("@local/~/my-hello/greeting")): + with pytest.raises(NotFoundError): + get() + + (bundle / "tasks/hello.json").write_text(json.dumps([*steps, {**steps[2], "id": "again", "verifier_id": "again"}])) + output = _run(bundle) + + assert "tasks/hello.json: warning: 2 verifiers and no headline_verifiers" in output + assert "tasks/hello.json v1: passed" in output + + @skip_without_gnu_stat() def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs): steps = json.loads((bundle / "tasks/hello.json").read_text()) diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 558a17efc..4604b55b0 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -50,6 +50,9 @@ ARM64_HERE = (*LOCAL, "linux/arm64") # built here on Apple silicon, as the infra bootstrap builds it HERE = "localhost:5000/local/img:v1 is in a registry on this machine, which a sandbox elsewhere can't pull from" +# The lint warning a task with no verifier gets, which a run says before it writes. +_NO_GRADE = "tasks/t.json: warning: no step grades this task, so a run has no score" + @pytest.fixture def bundle_dir(tmp_path, monkeypatch, local_stores): @@ -1044,7 +1047,7 @@ async def deploy(self, context): run_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) - assert events[:3] == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"]), + assert events[:4] == [("say", _NO_GRADE), ("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"]), ("say", "[tasks/t.json] step 1/1 env (deploy_env)")] assert ("deploy", "crm") in events @@ -1065,7 +1068,7 @@ async def deploy(self, context): materialization = _write_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) - assert events == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] + assert events == [("say", _NO_GRADE), ("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] assert [done.write.kind for done in materialization.writes] == [BundleKind.TASK] # The walk resolves each deploy's provider as its step does diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py index de2abdc4d..47af797ed 100644 --- a/tst/unit/bundle/run_test.py +++ b/tst/unit/bundle/run_test.py @@ -366,8 +366,9 @@ def test_a_run_says_before_it_builds_an_image_and_names_the_image_apart_from_its run_bundle(bundle_dir, tasks=["agent"], on_progress=lines.append) - assert lines[:5] == [ + assert lines[:6] == [ CONFIGURED_WITH_AGENT, + "tasks/agent.json: warning: no step grades this task, so a run has no score", "agents/solver (Dockerfile image): building with docker, which can take minutes", "agents/solver (Dockerfile image): v1 (new)", "agents/solver: v1 (new)", @@ -553,6 +554,8 @@ def test_the_cli_dry_run_prints_what_the_run_would_write_and_run_and_exits_0(bun assert result.output == ( DRY_RUN + f"{CONFIGURED}\n" + "tasks/c.json: warning: no step grades this task, so a run has no score\n" + "tasks/code.json: warning: no step grades this task, so a run has no score\n" "artifacts/script: v1 (new)\n" "tasks/a.json: v1 (new)\n" "tasks/b.json: v1 (new)\n" @@ -596,6 +599,7 @@ def test_the_cli_dry_run_reads_another_bundles_entity_and_says_why_an_agent_pinn assert result.output.startswith( DRY_RUN + f"{CONFIGURED}\n" + "tasks/agent.json: warning: no step grades this task, so a run has no score\n" f"agents/solver: v2 (artifact {image} has a new version in the store (v1 → v2))\n" "tasks/agent.json: v1, unchanged\n" "\n" @@ -615,6 +619,7 @@ def test_the_cli_dry_run_lists_an_image_it_would_write_and_builds_nothing(bundle assert result.output == ( DRY_RUN + f"{CONFIGURED_WITH_AGENT}\n" + "tasks/agent.json: warning: no step grades this task, so a run has no score\n" "agents/solver (Dockerfile image): v1 (new)\n" "agents/solver: v1 (new)\n" "tasks/agent.json: v1 (new)\n" diff --git a/tst/unit/cli/eval_run_teardown_test.py b/tst/unit/cli/eval_run_teardown_test.py index e1025a91b..ceea94fc8 100644 --- a/tst/unit/cli/eval_run_teardown_test.py +++ b/tst/unit/cli/eval_run_teardown_test.py @@ -22,7 +22,7 @@ class _Deploys: """Records a sandbox on the fake backend, then naps.""" def __init__(self, id, nap): - self.id, self.version, self.steps, self.nap = id, 1, [], nap + self.id, self.version, self.steps, self.nap, self.headline_verifiers = id, 1, [], nap, None async def run(self, context, **_): context.deployed_sandboxes.append(DeployedSandbox( diff --git a/tst/unit/cli/run_role_models_test.py b/tst/unit/cli/run_role_models_test.py index b4837aefe..65de2a589 100644 --- a/tst/unit/cli/run_role_models_test.py +++ b/tst/unit/cli/run_role_models_test.py @@ -21,7 +21,7 @@ class _Recorded: """Records each context it runs with.""" - id, version, steps, requires = "t", 1, [], None + id, version, steps, requires, headline_verifiers = "t", 1, [], None, None def __init__(self): self.contexts = [] diff --git a/tst/unit/cli/submit_runs_test.py b/tst/unit/cli/submit_runs_test.py index 42805f6cf..dd2bfa8b2 100644 --- a/tst/unit/cli/submit_runs_test.py +++ b/tst/unit/cli/submit_runs_test.py @@ -28,7 +28,7 @@ class _Task: - id, version, steps, requires = "t", 1, [], None + id, version, steps, requires, headline_verifiers = "t", 1, [], None, None def __init__(self, nap=0.0, failed_steps=None): self.nap, self.failed_steps, self.contexts, self.calls = nap, failed_steps, [], [] diff --git a/tst/unit/cli/test_task_create_preflight.py b/tst/unit/cli/test_task_create_preflight.py index 19d336790..dc41c611e 100644 --- a/tst/unit/cli/test_task_create_preflight.py +++ b/tst/unit/cli/test_task_create_preflight.py @@ -97,3 +97,28 @@ def test_skip_validation_saves_and_still_prints_the_problems(tmp_path, store): assert "Saving anyway" in result.output put_step.assert_called_once() put_task.assert_called_once() + + +def test_a_field_no_step_reads_is_refused_naming_the_one_it_meant_and_persists_nothing(tmp_path, store): + put_step, put_task = store + path = tmp_path / "steps.json" + path.write_text(json.dumps([{"id": "c1", "type": "run_code", "script_artifact_id": "a", "agent_nme": "x"}])) + + result = CliRunner().invoke(create, [str(path), "--id", "t", "--skip-validation"]) + + assert result.exit_code == 1 + assert "Error: step 'c1': run_code doesn't read 'agent_nme' (did you mean 'agent_name'?)" in result.output + put_step.assert_not_called() + put_task.assert_not_called() + + +def test_a_lint_error_is_a_preflight_problem_and_a_lint_warning_is_only_printed(tmp_path, store): + put_step, put_task = store + with patch("agent_env.artifact.Artifact.get", side_effect=NotFoundError("nope")): + refused = CliRunner().invoke(create, [str(_object_file(tmp_path, headline_verifiers=["c1"])), "--id", "t"]) + + assert refused.exit_code == 1 + assert "Warning: no step grades this task, so a run has no score" in refused.output + assert " - headline_verifiers names 'c1', a run_code step, which grades nothing" in refused.output + put_step.assert_not_called() + put_task.assert_not_called() diff --git a/tst/unit/task/lint_test.py b/tst/unit/task/lint_test.py new file mode 100644 index 000000000..3ff20508d --- /dev/null +++ b/tst/unit/task/lint_test.py @@ -0,0 +1,272 @@ +"""agent-env task lint: each rule with a task that breaks it and one that doesn't, where its findings refuse a run, and +the bundles agent-env ships linting clean.""" + +import json +from typing import ClassVar + +import pytest +from click.testing import CliRunner + +from agent_env.bundle import BundleKind, parse_bundle +from agent_env.bundle.installed import CORE, installed_bundles +from agent_env.bundle.plan import plan_bundle +from agent_env.bundle.resolve import resolve_bundle +from agent_env.cli import cli +from agent_env.eval import Eval +from agent_env.eval.eval import EvalTask +from agent_env.store.routing import namespace_routing +from agent_env.task import Task +from agent_env.task.lint import Severity, lint +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( + RunContainerUnitTestsVerifierTaskStep, +) + +BOX = {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"} +TESTS = {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", + "criteria": [{"type": "probe_file_exists", "criterion": "has it", "paths": ["a.txt"]}]} +MORE = {**TESTS, "id": "more", "verifier_id": "more"} +FINAL = {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests", "more"]} +NOTES = {"id": "notes", "type": "load_artifact", "sandbox_name": "box", "artifact_id": "shared-notes"} + + +def _task(*steps, **fields) -> Task: + return Task.from_dict({"id": "t", "version": 1, "steps": [dict(step) for step in steps], **fields}) + + +def _found(task, *args, **kwargs) -> list[tuple[str, str, str]]: + return [(f.severity.value, f.rule, f.text) for f in lint(task, *args, **kwargs)] + + +def test_a_field_its_step_doesnt_read_is_an_error_naming_the_nearest_one_it_does(): + steps = [BOX, {**TESTS, "min_passing_scor": 0.8}] + + assert _found(_task(BOX, TESTS), steps) == [( + "error", "unknown-field", "step 'tests': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?), so it would be ignored")] + assert _found(_task(BOX, TESTS), [BOX, TESTS]) == [] + + +def test_a_verifier_config_that_fails_when_it_grades_is_an_error_and_one_a_run_skips_is_a_warning(): + no_paths = {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "has it"}]} + unknown = {**TESTS, "criteria": [{"type": "probe_file_smells", "criterion": "has it"}]} + + assert _found(_task(BOX, no_paths)) == [("error", "step-config", "step 'tests': criterion #0: probe_file_exists " + "needs paths")] + assert _found(_task(BOX, unknown)) == [("warning", "step-config", "step 'tests': criterion #0: type " + "'probe_file_smells' isn't one verify_sandbox checks, so a run skips it; it " + "checks ['bash_cmd_succeeds', 'probe_dir_exists', 'probe_file_contains', " + "'probe_file_exists']")] + assert _found(_task(BOX, no_paths), config=False) == [] + + +@pytest.mark.parametrize(("criterion", "problem"), [ + ({"type": "response_shouts"}, "unknown type 'response_shouts'"), + ({"type": "response_contains"}, "response_contains needs non-empty needles"), + ({"type": "response_regex_present"}, "response_regex_present needs a non-empty pattern"), + ({"type": "response_regex_present", "pattern": "(unclosed"}, "pattern '(unclosed' doesn't compile"), + ({"type": "response_regex_present", "pattern": "x", "flags": ["SHOUT"]}, "flags ['SHOUT'] aren't re flags"), +]) +def test_an_agent_prompt_response_criterion_a_run_would_refuse_is_a_config_problem(criterion, problem): + step = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[criterion]) + fine = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[ + {"type": "response_contains", "needles": ["42"]}, + {"type": "response_regex_present", "pattern": "4[0-9]", "flags": ["IGNORECASE", "IGNORECASE"]}]) + + (found,) = step.config_problems() + assert found.startswith(f"criterion #0: {problem}") + assert fine.config_problems() == [] + + +def test_a_unit_test_run_that_tests_nothing_is_a_warning_and_a_negative_timeout_an_error(): + def tests(**kw): + return RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + **{"command": "pytest", **kw}) + + assert tests(command=" ").config_warnings() == [ + "command is empty, so it exits 0 and the tests pass without running anything"] + assert tests(command="", reward_path="/r").config_warnings() == [] + assert tests(timeout_sec=0).config_warnings() == [ + "timeout_sec 0 means no time limit, so a hung test runs until the sandbox's own TTL"] + assert tests(timeout_sec=-1).config_problems() == ["timeout_sec -1 is negative; give the tests at least 1 s"] + assert (tests().config_problems(), tests().config_warnings()) == ([], []) + + +class _Legacy(TaskStep): + """A verifier that still writes its legacy entry itself, as agentenvhub-sdk's do until they move onto the base.""" + + type: ClassVar[str] = "legacy_lint_test" + entity_refs = () + + def __init__(self, id): + super().__init__(id, 1) + self.verifier_id = id + + async def execute(self, context): + return context + + +def test_a_headline_that_grades_nothing_is_an_error_and_one_that_writes_no_named_grade_a_warning(): + built = _task(BOX, TESTS).steps + legacy = Task(id="t", version=1, steps=[*built, _Legacy("cua")], headline_verifiers=["cua"]) + + assert ("error", "headline-not-a-verifier", + "headline_verifiers names 'box', a deploy_sandbox step, which grades nothing") in _found( + _task(BOX, TESTS, headline_verifiers=["box"])) + assert ("warning", "headline-without-grade", + "headline_verifiers names 'cua', a legacy_lint_test step that writes no named grade yet, so a run's result " + "will read it as ungraded") in _found(legacy) + assert _found(_task(BOX, TESTS, headline_verifiers=["tests"])) == [] + + +def test_an_aggregate_source_no_verifier_writes_is_an_error(): + assert _found(_task(BOX, TESTS, FINAL, headline_verifiers=["final"])) == [( + "error", "aggregate-source", "step 'final': verifier_ids names ['more'], which no verifier in this task writes " + "(it has ['final', 'tests'])")] + assert _found(_task(BOX, TESTS, MORE, FINAL, headline_verifiers=["final"])) == [] + + +def test_a_store_entity_read_with_no_version_warns_and_a_pinned_or_local_one_doesnt(): + pinned = {**NOTES, "artifact_id": None, "artifacts": [{"id": "shared-notes", "version": 2}]} + local = {**NOTES, "artifact_id": "@local/~/triage/notes"} + + assert _found(_task(BOX, NOTES, TESTS)) == [( + "warning", "unpinned-reference", "step 'notes': artifacts[0].id names artifact 'shared-notes' with no version, " + "so each run takes its latest; set version to pin it")] + assert _found(_task(BOX, NOTES, TESTS), unpinned=False) == [] + assert _found(_task(BOX, pinned, TESTS)) == _found(_task(BOX, local, TESTS)) == [] + + +def test_a_task_that_grades_nothing_warns_and_several_grades_without_a_headline_name_the_covering_aggregate(): + assert _found(_task(BOX)) == [("warning", "no-verifier", "no step grades this task, so a run has no score")] + assert _found(_task(BOX, TESTS, MORE, FINAL)) == [( + "warning", "no-headlines", "3 verifiers and no headline_verifiers, so every grade is top-line and anything that " + "needs one number must name one; 'final' aggregates the others, so name it in " + "headline_verifiers")] + assert _found(_task(BOX, TESTS, MORE))[0][2].endswith("name the one that is its reward in headline_verifiers") + assert _found(_task(BOX, TESTS)) == [] + + +def test_errors_come_before_warnings(): + found = lint(_task(BOX, NOTES, TESTS, FINAL), [BOX, NOTES, TESTS, {**FINAL, "sources": ["tests"]}]) + + assert [f.severity for f in found] == sorted((f.severity for f in found), key=lambda s: s is not Severity.ERROR) + assert {f.rule for f in found} == {"unknown-field", "aggregate-source", "unpinned-reference", "no-headlines"} + + +def _shipped(): + """agent-env's own bundles that hold tasks; the rest (drivers, computers, browsers) have none to lint.""" + return {bundle.name: bundle for bundle in installed_bundles() + if bundle.dist == CORE and any((bundle.root / "tasks").glob("*.json"))} + + +# The shipped examples that show env triggers and no grading, which lint rightly says grade nothing. +_UNGRADED = {"advance-to-next-event", "next-env-event"} + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.parametrize("name", sorted(_shipped())) +def test_a_shipped_bundles_tasks_lint_clean(name): + bundle = _shipped()[name] + with namespace_routing(): + parsed = parse_bundle(bundle.root, id_root=bundle.id_root) + tasks = [entry.name for entry in parsed.entries if entry.kind is BundleKind.TASK] + plan = plan_bundle(resolve_bundle(parsed), tasks=tasks, installed=installed_bundles) + + expected = [f"tasks/{task}.json: warning: no step grades this task, so a run has no score" for task in tasks + ] if name in _UNGRADED else [] + assert sorted(plan.lint_warnings) == sorted(expected) + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + root = tmp_path / "triage" + (root / "tasks").mkdir(parents=True) + return root + + +def _write(root, name, body): + (root / "tasks" / f"{name}.json").write_text(json.dumps(body)) + + +def test_the_cli_lints_a_bundle_folder_refusing_its_errors_and_printing_its_warnings(bundle_dir): + _write(bundle_dir, "ok", [BOX, TESTS, MORE]) + warned = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + _write(bundle_dir, "bad", [BOX, {**TESTS, "min_passing_scor": 0.8}]) + refused = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + + assert warned.exit_code == 0, warned.output + assert "tasks/ok.json: warning: 2 verifiers and no headline_verifiers" in warned.output + assert warned.output.endswith("task lint: 1 warning\n") + assert refused.exit_code == 1 + assert ("tasks/bad.json: step 'tests': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?)") in refused.output + + +def test_the_cli_lints_one_task_file_of_a_bundle_or_one_on_its_own(bundle_dir, tmp_path): + _write(bundle_dir, "ok", [BOX, TESTS]) + alone = tmp_path / "alone.json" + alone.write_text(json.dumps({"headline_verifiers": ["tests"], "steps": [BOX, NOTES, TESTS]})) + + in_bundle = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir / "tasks/ok.json")]) + on_its_own = CliRunner().invoke(cli, ["task", "lint", str(alone)]) + + assert (in_bundle.exit_code, in_bundle.output) == (0, "task lint: no problems\n") + assert on_its_own.exit_code == 0, on_its_own.output + assert f"{alone}: warning: step 'notes': artifacts[0].id names artifact 'shared-notes'" in on_its_own.output + + +def test_the_cli_lints_a_stored_task(local_stores): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + stored = Task.put(id="t", steps=task.steps, headline_verifiers=["box"]) + + result = CliRunner().invoke(cli, ["task", "lint", "--id", "t"]) + + assert result.exit_code == 1 + assert f"t v{stored.version}: headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in ( + result.output) + + +def test_task_validate_fails_on_a_lint_error_and_prints_a_warning(local_stores): + Task.put(id="graded", steps=_task(BOX, NOTES, TESTS).steps) + Task.put(id="headless", steps=_task(BOX, TESTS).steps, headline_verifiers=["box"]) + + warned = CliRunner().invoke(cli, ["task", "validate", "--id", "graded"]) + refused = CliRunner().invoke(cli, ["task", "validate", "--id", "headless"]) + + assert warned.exit_code == 0, warned.output + assert "graded v1: warning: step 'notes': artifacts[0].id names artifact 'shared-notes'" in warned.output + assert refused.exit_code == 1 + assert " - headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in refused.output + + +def test_eval_run_refuses_an_eval_whose_task_has_a_lint_error_before_any_run(local_stores, monkeypatch): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + monkeypatch.setattr(Task, "run", lambda *a, **k: pytest.fail("a run started")) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e"]) + + assert result.exit_code == 1, result.output + assert ("Task lint found 1 error(s), so no run started:\n - t v1: headline_verifiers names 'box', a deploy_sandbox " + "step, which grades nothing") in result.output + + +def test_the_cli_takes_a_path_or_an_id_but_not_both(bundle_dir): + assert CliRunner().invoke(cli, ["task", "lint"]).exit_code == 2 + assert CliRunner().invoke(cli, ["task", "lint", str(bundle_dir), "--id", "t"]).exit_code == 2 + + +def test_a_run_of_a_bundle_with_a_lint_error_is_refused(bundle_dir): + _write(bundle_dir, "t", [BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}]) + + result = CliRunner().invoke(cli, ["run", str(bundle_dir), "--dry-run"]) + + assert result.exit_code != 0 + assert "tasks/t.json: step 'tests': criterion #0: probe_file_exists needs paths" in result.output From 8cfaf120946993112527cc02c0f6d32c36b3f020 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 10:07:46 -0800 Subject: [PATCH 02/10] fix(task): lint reports values of the wrong type instead of crashing on them A chaos run of junk values through task lint found a verifier_id, command or criterion of the wrong type crashing the new checks, and an array or object step id or type crashing the bundle resolver, as it does agent-env run on main. Each is now a problem the command names. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/parse.py | 5 ++++ src/agent_env/cli/task/lint.py | 24 +++++++++++++++---- src/agent_env/task/lint.py | 4 ++-- .../run_container_unit_tests_verifier.py | 4 +++- .../task_steps/verifiers/verifier.py | 2 ++ .../task_steps/verifiers/verify_sandbox.py | 5 ++++ tst/unit/bundle/parse_test.py | 7 ++++++ tst/unit/task/lint_test.py | 20 ++++++++++++++++ 8 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index e1e8d9695..b07c1ef1c 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -82,6 +82,11 @@ def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]: headlines is None or isinstance(headlines, list) and all(isinstance(h, str) for h in headlines)): raise ValueError("a task file must be a JSON array of step objects, or an object whose steps is one and whose " "headline_verifiers lists step ids") + for index, step in enumerate(steps): + for key in ("id", "type"): + if isinstance(step.get(key), (list, dict)): + raise ValueError(f"step #{index}'s {key} is a JSON {'array' if isinstance(step[key], list) else 'object'}" + f"; it must be a string") return steps, headlines diff --git a/src/agent_env/cli/task/lint.py b/src/agent_env/cli/task/lint.py index f2f613661..c52058ff8 100644 --- a/src/agent_env/cli/task/lint.py +++ b/src/agent_env/cli/task/lint.py @@ -52,9 +52,9 @@ def _bundle(root: Path, task: str | None = None) -> tuple[list[str], list[str]]: from agent_env.store.routing import namespace_routing with namespace_routing(): - bundle = parse_bundle(root) - names = [task] if task else [entry.name for entry in bundle.entries if entry.kind is BundleKind.TASK] try: + bundle = parse_bundle(root) + names = [task] if task else [entry.name for entry in bundle.entries if entry.kind is BundleKind.TASK] plan = plan_bundle(resolve_bundle(bundle), tasks=names, installed=installed_bundles) except BundleError as e: return list(e.problems), [] @@ -65,8 +65,10 @@ def _file(path: Path) -> tuple[list[str], list[str]]: """A task file on its own: its references name store ids, as ``task create`` reads them.""" from agent_env.bundle.parse import task_file_parts from agent_env.bundle.resolve import build_step + from agent_env.plugins import _registration from agent_env.task import Task from agent_env.task.lint import Severity, lint as lint_task, unknown_fields + from agent_env.task_step.registry import get_task_step_registry try: steps, headlines = task_file_parts(json.loads(path.read_text())) @@ -74,10 +76,22 @@ def _file(path: Path) -> tuple[list[str], list[str]]: return [f"{path}: {e}"], [] if unknown := list(unknown_fields(steps)): return [f"{path}: {f.text}" for f in unknown], [] + registry, built, problems = get_task_step_registry(), [], [] + for step in steps: + step_type = step.get("type") + if not isinstance(step_type, str) or registry.get(step_type) is None: + note = _registration.failure_note(_registration.TASK_STEPS, step_type) if isinstance(step_type, str) else "" + problems.append(f"step {step.get('id')!r}: {step_type!r} is not a known step type{note}") + continue + try: + built.append(build_step(step)) + except Exception as e: + problems.append(f"step {step.get('id')!r}: {step_type} can't read it ({type(e).__name__}: {e})") + if problems: + return [f"{path}: {problem}" for problem in problems], [] try: - task = Task(id=path.stem, version=None, steps=[build_step(step) for step in steps], - headline_verifiers=headlines) - except Exception as e: + task = Task(id=path.stem, version=None, steps=built, headline_verifiers=headlines) + except ValueError as e: return [f"{path}: {e}"], [] findings = lint_task(task, steps) return ([f"{path}: {f.text}" for f in findings if f.severity is Severity.ERROR], diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py index f1245a1ab..c86e17599 100644 --- a/src/agent_env/task/lint.py +++ b/src/agent_env/task/lint.py @@ -165,7 +165,7 @@ def _aggregate_sources(steps: Sequence[TaskStep]) -> Iterator[Finding]: """An aggregate source that no verifier in the task writes, which fails the aggregate when it grades.""" written = set() for step in steps: - if is_verifier(step): + if is_verifier(step) and isinstance(step.verifier_id, str): written.add(step.verifier_id) if (isinstance(step, RubricsVerifierTaskStep) and step.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES): @@ -202,7 +202,7 @@ def _coverage(task: Task) -> Iterator[Finding]: def _no_headlines(verifiers: list[TaskStep]) -> str: """Several verifiers and no headline: every grade is top-line. Name the aggregate that covers the others, if one does.""" - ids = {step.verifier_id for step in verifiers} + ids = {step.verifier_id for step in verifiers if isinstance(step.verifier_id, str)} covering = [step.id for step in verifiers if isinstance(step, AggregateVerifiersTaskStep) and ids - {step.verifier_id} <= set(step.verifier_ids)] suggestion = (f"; {covering[0]!r} aggregates the others, so name it in headline_verifiers" if covering diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index abe6bf11e..5d621f6ba 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -148,6 +148,8 @@ def __init__( def config_problems(self) -> list[str]: """A negative ``timeout_sec``, which ``timeout`` refuses, so the tests never run.""" problems = super().config_problems() + if not isinstance(self.command, str): + problems.append(f"command must be a string, not {type(self.command).__name__}") if self.timeout_sec < 0: problems.append(f"timeout_sec {self.timeout_sec} is negative; give the tests at least 1 s") return problems @@ -156,7 +158,7 @@ def config_warnings(self) -> list[str]: """What runs without testing anything: an empty ``command``, which exits 0 and so passes without a ``reward_path``, and a ``timeout_sec`` of 0, which ``timeout`` reads as no limit.""" warnings = super().config_warnings() - if not self.command.strip() and self.reward_path is None: + if isinstance(self.command, str) and not self.command.strip() and self.reward_path is None: warnings.append("command is empty, so it exits 0 and the tests pass without running anything") if self.timeout_sec == 0: warnings.append("timeout_sec 0 means no time limit, so a hung test runs until the sandbox's own TTL") diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 71059dfd1..840b1620c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -161,6 +161,8 @@ def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: def config_problems(self) -> list[str]: problems = super().config_problems() + if not isinstance(self.verifier_id, str): + problems.append(f"verifier_id must be a string, not {type(self.verifier_id).__name__}") if not 0.0 < self.min_passing_score <= 1.0: problems.append(f"min_passing_score must be above 0 and at most 1, not {self.min_passing_score}") return problems diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index 515f92655..e87417a01 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -108,6 +108,9 @@ def config_problems(self) -> list[str]: pattern that doesn't compile, a ``bash_cmd_succeeds`` with no command.""" problems = super().config_problems() for idx, criterion in enumerate(self.criteria): + if not isinstance(criterion, dict): + problems.append(f"criterion #{idx} must be an object, not {type(criterion).__name__}") + continue rtype = criterion.get("type") paths = criterion.get("paths") or [] if rtype in ("probe_file_exists", "probe_dir_exists", "probe_file_contains") and not paths: @@ -127,6 +130,8 @@ def config_warnings(self) -> list[str]: ``probe_file_contains`` with no expected text, which any file passes.""" warnings = super().config_warnings() for idx, criterion in enumerate(self.criteria): + if not isinstance(criterion, dict): + continue # config_problems refuses it rtype = criterion.get("type") if rtype not in _HANDLED_TYPES: warnings.append(f"criterion #{idx}: type {rtype!r} isn't one verify_sandbox checks, so a run skips " diff --git a/tst/unit/bundle/parse_test.py b/tst/unit/bundle/parse_test.py index 5fe85ab29..bd05dfeff 100644 --- a/tst/unit/bundle/parse_test.py +++ b/tst/unit/bundle/parse_test.py @@ -657,6 +657,13 @@ def test_a_task_files_object_takes_only_steps_and_a_list_of_headline_verifiers(s f"tasks/b.json: {TASK_SHAPE}", ) + def test_a_step_whose_id_or_type_is_an_array_or_object_is_refused(self, tmp_path): + root = build(tmp_path / "b", {"tasks/a.json": '[{"id": "box", "type": "x"}, {"id": ["box"], "type": "x"}]', + "tasks/b.json": '[{"id": "box", "type": {"name": "x"}}]', + "tasks/c.json": '[{"id": 1, "type": "x"}]'}) + assert problems(root) == ("tasks/a.json: step #1's id is a JSON array; it must be a string", + "tasks/b.json: step #0's type is a JSON object; it must be a string") + @pytest.mark.parametrize("rel, content", [ ("tasks/t.json", "[" * 100_000 + "]" * 100_000), ("evals/e.toml", "a = " + "[" * 5000 + "]" * 5000 + "\n"), diff --git a/tst/unit/task/lint_test.py b/tst/unit/task/lint_test.py index 3ff20508d..53d66820c 100644 --- a/tst/unit/task/lint_test.py +++ b/tst/unit/task/lint_test.py @@ -81,6 +81,21 @@ def test_an_agent_prompt_response_criterion_a_run_would_refuse_is_a_config_probl assert fine.config_problems() == [] +def test_a_value_of_the_wrong_type_a_verifier_would_trip_on_is_a_config_problem(): + criteria = [{"type": "probe_file_exists", "criterion": "has it", "paths": ["a.txt"]}, None, "probe"] + tests = RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + command=["pytest"], verifier_id=["tests"]) + + assert _found(_task(BOX, {**TESTS, "criteria": criteria})) == [ + ("error", "step-config", "step 'tests': criterion #1 must be an object, not NoneType"), + ("error", "step-config", "step 'tests': criterion #2 must be an object, not str")] + assert tests.config_problems() == ["verifier_id must be a string, not list", "command must be a string, not list"] + assert tests.config_warnings() == [] + assert _found(Task(id="t", version=1, steps=[*_task(BOX).steps, tests])) == [ + ("error", "step-config", "step 'tests': verifier_id must be a string, not list"), + ("error", "step-config", "step 'tests': command must be a string, not list")] + + def test_a_unit_test_run_that_tests_nothing_is_a_warning_and_a_negative_timeout_an_error(): def tests(**kw): return RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", @@ -206,6 +221,11 @@ def test_the_cli_lints_a_bundle_folder_refusing_its_errors_and_printing_its_warn assert ("tasks/bad.json: step 'tests': verify_sandbox doesn't read 'min_passing_scor' (did you mean " "'min_passing_score'?)") in refused.output + _write(bundle_dir, "bad", [BOX, {**TESTS, "id": ["tests"]}]) + unparsed = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + assert (unparsed.exit_code, unparsed.output.splitlines()[-1]) == ( + 1, " - tasks/bad.json: step #1's id is a JSON array; it must be a string") + def test_the_cli_lints_one_task_file_of_a_bundle_or_one_on_its_own(bundle_dir, tmp_path): _write(bundle_dir, "ok", [BOX, TESTS]) From 973cf851b7059781c55210eff10ce437717ec4e4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 10:11:50 -0800 Subject: [PATCH 03/10] fix(bundle): bundle register says each lint warning once The command checks, then writes, registering twice, and each pass said the plan's warnings. The check's result carries them, so the command says them once after it. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/register.py | 2 -- src/agent_env/cli/bundle.py | 2 ++ tst/unit/bundle/register_test.py | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 19bac4eaf..36d546c20 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -115,8 +115,6 @@ def register_bundle( raise BundleError(problems) requires = tuple(declare(resolved, declaring or declaring_for(bundle.root, installed=False, dry_run=dry_run))) say = on_progress or (lambda line: None) - for warning in plan.lint_warnings: - say(f" {warning}") materialization = materialize( registered, dry_run=dry_run, ledger=ledger, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index b33996231..194e7eec8 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -126,6 +126,8 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi if not dry_run: offer_to_install(e) raise + for line in planned.plan.lint_warnings: + click.echo(f" {line}") for line in planned.floating: click.echo(f" floating, since its field takes no version: {line}") for line in planned.requires: diff --git a/tst/unit/bundle/register_test.py b/tst/unit/bundle/register_test.py index 2be9159d9..917211e18 100644 --- a/tst/unit/bundle/register_test.py +++ b/tst/unit/bundle/register_test.py @@ -395,6 +395,22 @@ def test_the_cli_says_what_it_would_write_asks_before_a_shared_store_and_dry_run assert [task.version for task in [Task.get("triage-t")]] == [1] +def test_the_cli_says_each_lint_warning_once_and_a_lint_error_refuses_before_anything_is_written(home): + root = layout(home / "triage", TRIAGE) + warning = " tasks/t.json: warning: no step grades this task, so a run has no score" + + dry = CliRunner().invoke(cli, ["bundle", "register", str(root), "--dry-run"]) + written = CliRunner().invoke(cli, ["bundle", "register", str(root), "--yes"]) + (root / "tasks/t.json").write_text(json.dumps([BOX, {**_load("greet", "greeting"), "destinaton_path": "/app"}])) + refused = CliRunner().invoke(cli, ["bundle", "register", str(root), "--yes"]) + + assert dry.output.splitlines().count(warning) == written.output.splitlines().count(warning) == 1 + assert refused.exit_code == 1 + assert ("tasks/t.json: step 'greet': load_artifact doesn't read 'destinaton_path' (did you mean " + "'destination_path'?), so it would be ignored") in refused.output + assert Task.get("triage-t").version == 1 + + def test_a_bundle_with_no_tasks_registers_every_entity(home): registered = register_bundle(layout(home / "library", {"artifacts/greeting/hello.txt": "hello\n"})) From 1dbde3d240190176076adfee62ca9e87fce5450b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 10:33:12 -0800 Subject: [PATCH 04/10] fix(task): lint grades only what it can see, and checks types before use A missing aggregate source, a headline that grades nothing and a task with no verifier are now sure only for agent-env's own steps: a validator writes its entry under a key of its own, and an outside step such as agentenvhub-sdk's run_lighthouse_audit may write any, so with one in the task each is a warning. Each verifier's config checks read a criterion's fields only once their types are right, and say what's wrong with the rest; and task lint's help says which inputs read the stores. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/cli/task/lint.py | 5 +- src/agent_env/task/lint.py | 47 +++++--- .../agent_prompt_response_verifier.py | 21 ++-- .../run_container_unit_tests_verifier.py | 7 +- .../task_steps/verifiers/verify_sandbox.py | 53 +++++---- tst/unit/bundle/run_test.py | 5 +- tst/unit/task/lint_test.py | 106 ++++++++++++++++-- 7 files changed, 188 insertions(+), 56 deletions(-) diff --git a/src/agent_env/cli/task/lint.py b/src/agent_env/cli/task/lint.py index c52058ff8..f6f625ea1 100644 --- a/src/agent_env/cli/task/lint.py +++ b/src/agent_env/cli/task/lint.py @@ -17,8 +17,9 @@ def lint(path: Path | None, task_id: str | None, task_version: int | None): Errors are configs that can't run or grade correctly: an unknown field, a step's own config problems, a headline that isn't a verifier, an aggregate source no verifier writes. They also refuse agent-env run, bundle register, task create and - eval run. Warnings block nothing. Exits non-zero when there are errors. Reads no - store, but a bundle folder is planned as a run plans it. + eval run. Warnings block nothing. Exits non-zero when there are errors. A task + file on its own is checked without a store; a bundle folder is planned as a run + plans it and --id reads the stored task, so both read the configured stores. """ if (path is None) == (task_id is None): raise click.UsageError("give a bundle folder or task file, or --id, but not both") diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py index c86e17599..c3cb714e5 100644 --- a/src/agent_env/task/lint.py +++ b/src/agent_env/task/lint.py @@ -13,7 +13,7 @@ from agent_env.store.ids import is_local_id from agent_env.task.task import Task from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_step import TaskStep, attach_base_fields +from agent_env.task_step.task_step import StepRole, TaskStep, attach_base_fields from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, trajectory_mistakes_key @@ -65,9 +65,22 @@ def errors(findings: Iterable[Finding]) -> list[Finding]: def is_verifier(step: TaskStep) -> bool: - """A step that grades: one on the VerifierTaskStep base, or one that still writes a legacy entry under its - ``verifier_id`` itself, as agentenvhub-sdk's do until they move onto the base.""" - return isinstance(step, VerifierTaskStep) or bool(getattr(step, "verifier_id", None)) + """A step that grades: one on the VerifierTaskStep base, or one that still writes its own legacy entry, a grader (as + agent-env's validators are) or a step with a ``verifier_id`` (as agentenvhub-sdk's are).""" + return (isinstance(step, VerifierTaskStep) or step.step_role is StepRole.GRADER + or bool(getattr(step, "verifier_id", None))) + + +def _own(step: TaskStep) -> bool: + """One of agent-env's own steps, whose behaviour lint knows.""" + return type(step).__module__.partition(".")[0] == "agent_env" + + +def _seen_into(step: TaskStep) -> bool: + """A step whose verification writes lint knows: one on the base writes under its ``verifier_id``, and another of + agent-env's own writes none unless it's a grader, whose keys are its own. An outside step off the base may write + any key, as agentenvhub-sdk's ``run_lighthouse_audit`` writes under its ``verification_id``.""" + return isinstance(step, VerifierTaskStep) or (_own(step) and step.step_role is not StepRole.GRADER) class _Reads(dict): @@ -152,20 +165,23 @@ def _headlines(task: Task) -> Iterator[Finding]: step = by_id.get(headline) if step is None: continue # Task refuses it when it's built - if not is_verifier(step): + if isinstance(step, VerifierTaskStep): + continue + if _own(step) and not is_verifier(step): yield Finding(Severity.ERROR, "headline-not-a-verifier", f"headline_verifiers names {headline!r}, a {step.type} step, which grades nothing") - elif not isinstance(step, VerifierTaskStep): + else: yield Finding(Severity.WARNING, "headline-without-grade", - f"headline_verifiers names {headline!r}, a {step.type} step that writes no named grade yet, " + f"headline_verifiers names {headline!r}, a {step.type} step that writes no named grade, " "so a run's result will read it as ungraded") def _aggregate_sources(steps: Sequence[TaskStep]) -> Iterator[Finding]: - """An aggregate source that no verifier in the task writes, which fails the aggregate when it grades.""" - written = set() + """An aggregate source that no verifier in the task writes, which fails the aggregate when it grades: an error when + lint sees every key the task's steps write, else a warning naming the steps it can't see into.""" + written, unseen = set(), [step.id for step in steps if not _seen_into(step)] for step in steps: - if is_verifier(step) and isinstance(step.verifier_id, str): + if isinstance(getattr(step, "verifier_id", None), str): written.add(step.verifier_id) if (isinstance(step, RubricsVerifierTaskStep) and step.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES): @@ -173,10 +189,15 @@ def _aggregate_sources(steps: Sequence[TaskStep]) -> Iterator[Finding]: for step in steps: if isinstance(step, AggregateVerifiersTaskStep): missing = [source for source in step.verifier_ids if source not in written] - if missing: + if missing and not unseen: yield Finding(Severity.ERROR, "aggregate-source", f"verifier_ids names {missing!r}, which no verifier in this task writes (it has " f"{sorted(written)})", step.id) + elif missing: + yield Finding(Severity.WARNING, "aggregate-source", + f"verifier_ids names {missing!r}, which no verifier in this task names as its " + f"verifier_id; unless {unseen!r} write it, under keys lint can't see, the aggregate fails " + "when it grades", step.id) def _unpinned(steps: Sequence[TaskStep]) -> Iterator[Finding]: @@ -193,7 +214,7 @@ def _unpinned(steps: Sequence[TaskStep]) -> Iterator[Finding]: def _coverage(task: Task) -> Iterator[Finding]: verifiers = [step for step in task.steps if is_verifier(step)] - if not verifiers: + if not verifiers and all(_own(step) for step in task.steps): yield Finding(Severity.WARNING, "no-verifier", "no step grades this task, so a run has no score") elif len(verifiers) > 1 and not task.headline_verifiers: yield Finding(Severity.WARNING, "no-headlines", _no_headlines(verifiers)) @@ -202,7 +223,7 @@ def _coverage(task: Task) -> Iterator[Finding]: def _no_headlines(verifiers: list[TaskStep]) -> str: """Several verifiers and no headline: every grade is top-line. Name the aggregate that covers the others, if one does.""" - ids = {step.verifier_id for step in verifiers if isinstance(step.verifier_id, str)} + ids = {step.verifier_id for step in verifiers if isinstance(getattr(step, "verifier_id", None), str)} covering = [step.id for step in verifiers if isinstance(step, AggregateVerifiersTaskStep) and ids - {step.verifier_id} <= set(step.verifier_ids)] suggestion = (f"; {covering[0]!r} aggregates the others, so name it in headline_verifiers" if covering diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index c9406fbae..6ea92b35e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -20,7 +20,7 @@ import logging import re import uuid -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.task_step.task_step import TaskStepDependency @@ -153,17 +153,24 @@ def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict: raise RuntimeError(f"unhandled criterion type {rtype}") -def _criterion_problems(criterion: dict) -> list[str]: +def _criterion_problems(criterion: Any) -> list[str]: + if not isinstance(criterion, dict): + return [f"must be an object, not {type(criterion).__name__}"] rtype = criterion.get("type") - if rtype not in _HANDLED_TYPES: + if not isinstance(rtype, str) or rtype not in _HANDLED_TYPES: return [f"unknown type {rtype!r}; handled types: {sorted(_HANDLED_TYPES)}"] if rtype == "response_contains": - return [] if criterion.get("needles") else ["response_contains needs non-empty needles"] + needles = criterion.get("needles") + if isinstance(needles, list) and needles and all(isinstance(needle, str) for needle in needles): + return [] + return [f"response_contains needs non-empty needles, a list of strings, not {needles!r}"] pattern = criterion.get("pattern") - if not pattern: - return ["response_regex_present needs a non-empty pattern"] + if not (isinstance(pattern, str) and pattern): + return [f"response_regex_present needs a non-empty pattern, a string, not {pattern!r}"] names = criterion.get("flags") - if names is not None and (unknown := [n for n in names if not isinstance(getattr(re, str(n), None), re.RegexFlag)]): + if names is not None and not (isinstance(names, list) and all(isinstance(name, str) for name in names)): + return [f"flags must be a list of re flag names, not {names!r}"] + if unknown := [name for name in names or () if not isinstance(getattr(re, name, None), re.RegexFlag)]: return [f"flags {unknown!r} aren't re flags"] flags = re.DOTALL if names is None else 0 for name in names or (): diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 5d621f6ba..2bac2b130 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -146,11 +146,14 @@ def __init__( f"to its session user's home, not {absolute}") def config_problems(self) -> list[str]: - """A negative ``timeout_sec``, which ``timeout`` refuses, so the tests never run.""" + """A ``command`` or ``timeout_sec`` a run can't use, and a negative ``timeout_sec``, which ``timeout`` + refuses, so the tests never run.""" problems = super().config_problems() if not isinstance(self.command, str): problems.append(f"command must be a string, not {type(self.command).__name__}") - if self.timeout_sec < 0: + if not isinstance(self.timeout_sec, (int, float)) or isinstance(self.timeout_sec, bool): + problems.append(f"timeout_sec must be a number of seconds, not {self.timeout_sec!r}") + elif self.timeout_sec < 0: problems.append(f"timeout_sec {self.timeout_sec} is negative; give the tests at least 1 s") return problems diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index e87417a01..e25333904 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -26,7 +26,7 @@ import re import shlex import uuid -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from agent_env.providers.sandbox_providers.sandbox import VmSandbox from agent_env.providers.sandbox_providers.sandbox_provider import ( @@ -49,10 +49,36 @@ "probe_file_contains", "bash_cmd_succeeds", } +_PROBES = ("probe_file_exists", "probe_dir_exists", "probe_file_contains") _DEFAULT_BASE_DIR = "/app" _DEFAULT_SHELL_TIMEOUT_S = 120 +def _criterion_problems(criterion: Any) -> list[str]: + if not isinstance(criterion, dict): + return [f"must be an object, not {type(criterion).__name__}"] + rtype = criterion.get("type") + if not isinstance(rtype, str): + return [f"type must be a string, not {rtype!r}"] + problems = [] + paths = criterion.get("paths") + if rtype in _PROBES and not (isinstance(paths, list) and paths and all(isinstance(path, str) for path in paths)): + problems.append(f"{rtype} needs paths, a non-empty list of strings, not {paths!r}") + if rtype == "probe_file_contains": + pattern = criterion.get("expected") or criterion.get("value") or "" + if not isinstance(pattern, str): + problems.append(f"expected must be a string, not {pattern!r}") + else: + try: + re.compile(pattern) + except re.error as e: + problems.append(f"expected {pattern!r} doesn't compile as a regex: {e}") + command = criterion.get("bash_cmd") + if rtype == "bash_cmd_succeeds" and not (isinstance(command, str) and command.strip()): + problems.append(f"bash_cmd_succeeds needs a bash_cmd, a non-empty string, not {command!r}") + return problems + + def _outcome(passed: bool, justification: str) -> dict: return {"score": 1.0 if passed else 0.0, "passed": passed, "justification": justification} @@ -104,25 +130,12 @@ def to_dict(self) -> dict: return base def config_problems(self) -> list[str]: - """The criteria that fail their row whatever the sandbox holds: a probe with no paths, a ``probe_file_contains`` - pattern that doesn't compile, a ``bash_cmd_succeeds`` with no command.""" + """The criteria a run can't read, or that fail their row whatever the sandbox holds: one that isn't an object or + whose type isn't a string, a probe without a list of paths, a ``probe_file_contains`` pattern that doesn't + compile, a ``bash_cmd_succeeds`` with no command.""" problems = super().config_problems() for idx, criterion in enumerate(self.criteria): - if not isinstance(criterion, dict): - problems.append(f"criterion #{idx} must be an object, not {type(criterion).__name__}") - continue - rtype = criterion.get("type") - paths = criterion.get("paths") or [] - if rtype in ("probe_file_exists", "probe_dir_exists", "probe_file_contains") and not paths: - problems.append(f"criterion #{idx}: {rtype} needs paths") - if rtype == "probe_file_contains": - pattern = criterion.get("expected") or criterion.get("value") or "" - try: - re.compile(pattern) - except re.error as e: - problems.append(f"criterion #{idx}: expected {pattern!r} doesn't compile as a regex: {e}") - if rtype == "bash_cmd_succeeds" and not (criterion.get("bash_cmd") or "").strip(): - problems.append(f"criterion #{idx}: bash_cmd_succeeds needs a bash_cmd") + problems.extend(f"criterion #{idx}: {problem}" for problem in _criterion_problems(criterion)) return problems def config_warnings(self) -> list[str]: @@ -130,9 +143,9 @@ def config_warnings(self) -> list[str]: ``probe_file_contains`` with no expected text, which any file passes.""" warnings = super().config_warnings() for idx, criterion in enumerate(self.criteria): - if not isinstance(criterion, dict): + if not (isinstance(criterion, dict) and isinstance(criterion.get("type"), str)): continue # config_problems refuses it - rtype = criterion.get("type") + rtype = criterion["type"] if rtype not in _HANDLED_TYPES: warnings.append(f"criterion #{idx}: type {rtype!r} isn't one verify_sandbox checks, so a run skips " f"it; it checks {sorted(_HANDLED_TYPES)}") diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py index 47af797ed..fec9aa31e 100644 --- a/tst/unit/bundle/run_test.py +++ b/tst/unit/bundle/run_test.py @@ -366,9 +366,8 @@ def test_a_run_says_before_it_builds_an_image_and_names_the_image_apart_from_its run_bundle(bundle_dir, tasks=["agent"], on_progress=lines.append) - assert lines[:6] == [ + assert lines[:5] == [ CONFIGURED_WITH_AGENT, - "tasks/agent.json: warning: no step grades this task, so a run has no score", "agents/solver (Dockerfile image): building with docker, which can take minutes", "agents/solver (Dockerfile image): v1 (new)", "agents/solver: v1 (new)", @@ -554,7 +553,6 @@ def test_the_cli_dry_run_prints_what_the_run_would_write_and_run_and_exits_0(bun assert result.output == ( DRY_RUN + f"{CONFIGURED}\n" - "tasks/c.json: warning: no step grades this task, so a run has no score\n" "tasks/code.json: warning: no step grades this task, so a run has no score\n" "artifacts/script: v1 (new)\n" "tasks/a.json: v1 (new)\n" @@ -619,7 +617,6 @@ def test_the_cli_dry_run_lists_an_image_it_would_write_and_builds_nothing(bundle assert result.output == ( DRY_RUN + f"{CONFIGURED_WITH_AGENT}\n" - "tasks/agent.json: warning: no step grades this task, so a run has no score\n" "agents/solver (Dockerfile image): v1 (new)\n" "agents/solver: v1 (new)\n" "tasks/agent.json: v1 (new)\n" diff --git a/tst/unit/task/lint_test.py b/tst/unit/task/lint_test.py index 53d66820c..85653c342 100644 --- a/tst/unit/task/lint_test.py +++ b/tst/unit/task/lint_test.py @@ -10,7 +10,7 @@ from agent_env.bundle import BundleKind, parse_bundle from agent_env.bundle.installed import CORE, installed_bundles from agent_env.bundle.plan import plan_bundle -from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle.resolve import build_step, resolve_bundle from agent_env.cli import cli from agent_env.eval import Eval from agent_env.eval.eval import EvalTask @@ -31,6 +31,8 @@ MORE = {**TESTS, "id": "more", "verifier_id": "more"} FINAL = {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests", "more"]} NOTES = {"id": "notes", "type": "load_artifact", "sandbox_name": "box", "artifact_id": "shared-notes"} +# A validator: it grades, under a key of its own, with no verifier_id. +ASSESS = {"id": "assess", "type": "verify_mcp_env_assessment", "env_id": "mcp", "prompt_id": "ask"} def _task(*steps, **fields) -> Task: @@ -55,7 +57,7 @@ def test_a_verifier_config_that_fails_when_it_grades_is_an_error_and_one_a_run_s unknown = {**TESTS, "criteria": [{"type": "probe_file_smells", "criterion": "has it"}]} assert _found(_task(BOX, no_paths)) == [("error", "step-config", "step 'tests': criterion #0: probe_file_exists " - "needs paths")] + "needs paths, a non-empty list of strings, not None")] assert _found(_task(BOX, unknown)) == [("warning", "step-config", "step 'tests': criterion #0: type " "'probe_file_smells' isn't one verify_sandbox checks, so a run skips it; it " "checks ['bash_cmd_succeeds', 'probe_dir_exists', 'probe_file_contains', " @@ -81,14 +83,73 @@ def test_an_agent_prompt_response_criterion_a_run_would_refuse_is_a_config_probl assert fine.config_problems() == [] +@pytest.mark.parametrize(("criterion", "problem"), [ + (None, "must be an object, not NoneType"), + ({"type": ["response_contains"]}, "unknown type ['response_contains']"), + ({"type": "response_contains", "needles": "abc"}, "response_contains needs non-empty needles, a list of strings"), + ({"type": "response_contains", "needles": [7]}, "response_contains needs non-empty needles, a list of strings"), + ({"type": "response_regex_present", "pattern": 7}, "response_regex_present needs a non-empty pattern, a string"), + ({"type": "response_regex_present", "pattern": "x", "flags": "IGNORECASE"}, "flags must be a list of re flag"), +]) +def test_an_agent_prompt_response_criterion_of_the_wrong_type_is_a_config_problem(criterion, problem): + step = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[criterion]) + + (found,) = step.config_problems() + assert found.startswith(f"criterion #0: {problem}") + + +@pytest.mark.parametrize(("criterion", "problem"), [ + ({"type": ["probe_file_exists"], "criterion": "x"}, "type must be a string, not ['probe_file_exists']"), + ({"type": "probe_file_exists", "criterion": "x", "paths": "a.txt"}, + "probe_file_exists needs paths, a non-empty list of strings, not 'a.txt'"), + ({"type": "probe_file_contains", "criterion": "x", "paths": ["a.txt"], "expected": 7}, + "expected must be a string, not 7"), + ({"type": "bash_cmd_succeeds", "criterion": "x", "bash_cmd": ["make"]}, + "bash_cmd_succeeds needs a bash_cmd, a non-empty string, not ['make']"), +]) +def test_a_verify_sandbox_criterion_of_the_wrong_type_is_a_config_problem_and_no_warning(criterion, problem): + step = build_step({**TESTS, "criteria": [criterion]}) + + assert (step.config_problems(), step.config_warnings()) == ([f"criterion #0: {problem}"], []) + + +@pytest.mark.parametrize(("timeout", "problems"), [ + ("300", ["timeout_sec must be a number of seconds, not '300'"]), + (None, ["timeout_sec must be a number of seconds, not None"]), + (True, ["timeout_sec must be a number of seconds, not True"]), + (1.5, []), +]) +def test_a_unit_test_timeout_is_a_number_of_seconds(timeout, problems): + tests = RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + command="pytest", timeout_sec=timeout) + + assert tests.config_problems() == problems + + +def test_the_cli_names_each_value_of_the_wrong_type_in_a_task_file_and_exits_1(tmp_path): + path = tmp_path / "bad.json" + path.write_text(json.dumps([BOX, {**TESTS, "criteria": [None, {"type": "probe_file_exists", "paths": "a.txt"}]}, + {"id": "check", "type": "agent_prompt_response_verifier", "prompt_id": "ask", + "criteria": [{"type": "response_regex_present", "pattern": 7}]}])) + + result = CliRunner().invoke(cli, ["task", "lint", str(path)]) + + assert result.exit_code == 1, result.output + assert result.exception is None or isinstance(result.exception, SystemExit) + assert [line.split(": ", 1)[1] for line in result.output.splitlines() if line.startswith(" - ")] == [ + "step 'tests': criterion #0: must be an object, not NoneType", + "step 'tests': criterion #1: probe_file_exists needs paths, a non-empty list of strings, not 'a.txt'", + "step 'check': criterion #0: response_regex_present needs a non-empty pattern, a string, not 7"] + + def test_a_value_of_the_wrong_type_a_verifier_would_trip_on_is_a_config_problem(): criteria = [{"type": "probe_file_exists", "criterion": "has it", "paths": ["a.txt"]}, None, "probe"] tests = RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", command=["pytest"], verifier_id=["tests"]) assert _found(_task(BOX, {**TESTS, "criteria": criteria})) == [ - ("error", "step-config", "step 'tests': criterion #1 must be an object, not NoneType"), - ("error", "step-config", "step 'tests': criterion #2 must be an object, not str")] + ("error", "step-config", "step 'tests': criterion #1: must be an object, not NoneType"), + ("error", "step-config", "step 'tests': criterion #2: must be an object, not str")] assert tests.config_problems() == ["verifier_id must be a string, not list", "command must be a string, not list"] assert tests.config_warnings() == [] assert _found(Task(id="t", version=1, steps=[*_task(BOX).steps, tests])) == [ @@ -124,23 +185,50 @@ async def execute(self, context): return context +class _Outside(TaskStep): + """An outside step off the base with no verifier_id, as agentenvhub-sdk's run_lighthouse_audit, which writes its + entry under a verification_id: lint can't tell what it writes.""" + + type: ClassVar[str] = "outside_lint_test" + entity_refs = () + + def __init__(self, id): + super().__init__(id, 1) + + async def execute(self, context): + return context + + def test_a_headline_that_grades_nothing_is_an_error_and_one_that_writes_no_named_grade_a_warning(): built = _task(BOX, TESTS).steps legacy = Task(id="t", version=1, steps=[*built, _Legacy("cua")], headline_verifiers=["cua"]) + outside = Task(id="t", version=1, steps=[*built, _Outside("audit")], headline_verifiers=["audit"]) assert ("error", "headline-not-a-verifier", "headline_verifiers names 'box', a deploy_sandbox step, which grades nothing") in _found( _task(BOX, TESTS, headline_verifiers=["box"])) - assert ("warning", "headline-without-grade", - "headline_verifiers names 'cua', a legacy_lint_test step that writes no named grade yet, so a run's result " - "will read it as ungraded") in _found(legacy) + for task, (name, kind) in [(legacy, ("cua", "legacy_lint_test")), (outside, ("audit", "outside_lint_test")), + (_task(BOX, ASSESS, headline_verifiers=["assess"]), + ("assess", "verify_mcp_env_assessment"))]: + assert ("warning", "headline-without-grade", + f"headline_verifiers names {name!r}, a {kind} step that writes no named grade, so a run's result will " + "read it as ungraded") in _found(task) assert _found(_task(BOX, TESTS, headline_verifiers=["tests"])) == [] -def test_an_aggregate_source_no_verifier_writes_is_an_error(): +def test_an_aggregate_source_no_verifier_writes_is_an_error_unless_a_step_lint_cant_see_into_may_write_it(): + on_assessment = {**FINAL, "verifier_ids": ["tests", "mcp_tool_correctness"]} + assert _found(_task(BOX, TESTS, FINAL, headline_verifiers=["final"])) == [( "error", "aggregate-source", "step 'final': verifier_ids names ['more'], which no verifier in this task writes " "(it has ['final', 'tests'])")] + assert ("warning", "aggregate-source", + "step 'final': verifier_ids names ['mcp_tool_correctness'], which no verifier in this task names as its " + "verifier_id; unless ['assess'] write it, under keys lint can't see, the aggregate fails when it grades" + ) in _found(_task(BOX, ASSESS, TESTS, on_assessment, headline_verifiers=["final"])) + outside = Task(id="t", version=1, steps=[*_task(BOX, TESTS, FINAL).steps, _Outside("audit")], + headline_verifiers=["final"]) + assert [f.severity for f in lint(outside) if f.rule == "aggregate-source"] == [Severity.WARNING] assert _found(_task(BOX, TESTS, MORE, FINAL, headline_verifiers=["final"])) == [] @@ -157,6 +245,8 @@ def test_a_store_entity_read_with_no_version_warns_and_a_pinned_or_local_one_doe def test_a_task_that_grades_nothing_warns_and_several_grades_without_a_headline_name_the_covering_aggregate(): assert _found(_task(BOX)) == [("warning", "no-verifier", "no step grades this task, so a run has no score")] + assert _found(_task(BOX, ASSESS)) == [] + assert _found(Task(id="t", version=1, steps=[*_task(BOX).steps, _Outside("audit")])) == [] assert _found(_task(BOX, TESTS, MORE, FINAL)) == [( "warning", "no-headlines", "3 verifiers and no headline_verifiers, so every grade is top-line and anything that " "needs one number must name one; 'final' aggregates the others, so name it in " From 3e22dfcc5131781a5521d631e945d0bd4ffd48c4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 12:10:01 -0800 Subject: [PATCH 05/10] test(drivers): the idle REPL test holds its REPL until both calls are done On a busy runner the gap between the test's two calls passed the one-second idle window, so the sweep closed the REPL before the second call read it back. The window is long for the two calls and short once they're done. Co-Authored-By: Claude Opus 5.5 --- tst/unit/drivers/codex_computer_use_driver_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index ec1c805dd..e4b88beec 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -98,9 +98,6 @@ async def main(): anyio.run(main) ''') -#: How long a REPL may sit unused before the driver closes it, in the idle test: well over two calls in a row on a busy -#: runner, so the second still finds the first's REPL. -IDLE_S = 1.0 MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") WINDOWS = Platform(os="windows", version="11", arch="amd64", display="dwm") @@ -359,11 +356,13 @@ async def js(caller: str, code: str) -> str: @pytest.mark.asyncio async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): + """The REPL may sit idle for as long as two calls take on a busy runner, then for a moment once they're done.""" monkeypatch.setattr(module, "SWEEP_S", 0.05) - monkeypatch.setattr(module, "IDLE_S", IDLE_S) + monkeypatch.setattr(module, "IDLE_S", 60) async with mcp_session(_driver(module, monkeypatch)) as session: await session.call_tool("js", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) before = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + monkeypatch.setattr(module, "IDLE_S", 0.2) closed = await _gone(int(before.rpartition(" in ")[2])) after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) From 81733522403e5420d1a67d8cc4e9a74dfe43c2c2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 12:42:10 -0800 Subject: [PATCH 06/10] fix(local): processes with their own state roots hand out host ports in turn The host-port cursor lived in the state root, so processes with their own, as each xdist worker has, kept cursors of their own and could hand out one port in the gap before docker binds it: docker then refused it as already allocated (the slow tier's browser clock test on this PR, run_bundle_test on main). The cursor is now one per user per machine, in a folder of the user's own in the temp folder: 8 processes drawing 200 ports each drew 22 twice before and none after. Co-Authored-By: Claude Opus 5.5 --- .../local_docker/provider.py | 25 ++++++++--- .../local_sandbox_host_ports_test.py | 42 +++++++++++++++---- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index f8b20301e..68aa21d1b 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -22,6 +22,7 @@ import shutil import signal import socket +import stat import subprocess import tempfile from ipaddress import ip_address @@ -126,8 +127,8 @@ def trust_env(directory: str) -> dict[str, str]: # up on Linux, the VM Docker Desktop or Rancher Desktop runs Docker in included; 49152 up on macOS and Windows) and # below Kubernetes' node ports (30000-32767). _HOST_PORTS = range(20000, 30000) -# Holds the last host port handed out, in the per-user state root's locks, so processes sharing that root hand them -# out in turn. +# Holds the last host port handed out, in a folder of the user's own in this machine's temp folder, so every process the +# user runs here hands them out in turn, whatever its state root. _HOST_PORT_CURSOR = "host-ports" @@ -155,13 +156,11 @@ def _local_sandbox_root() -> Path: def _free_host_port() -> int: """The next port in ``_HOST_PORTS`` after the last one handed out, skipping those in use. - Docker binds it later, so nothing may take it meanwhile: a deploy sharing this state root draws it again only after + Docker binds it later, so nothing may take it meanwhile: another deploy on this machine draws it again only after the whole range has gone round, and outgoing connections don't use the range by default. The probe alone could not keep it: it releases the port, and can't see the sockets of a VM Docker runs in. """ - locks = state_root() / "locks" - ensure_state_dir(locks) - fd = os.open(locks / _HOST_PORT_CURSOR, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + fd = os.open(_host_port_cursor(), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) try: fcntl.flock(fd, fcntl.LOCK_EX) last = os.read(fd, 16).strip() @@ -177,6 +176,20 @@ def _free_host_port() -> int: raise RuntimeError(f"Every host port from {_HOST_PORTS.start} to {_HOST_PORTS.stop - 1} is in use") +def _host_port_cursor() -> Path: + """The cursor's file: in this user's own folder of the machine's temp folder, shared by processes whose state roots + differ, as each xdist worker's does; in the state root's locks when that folder isn't this user's alone.""" + folder = Path(tempfile.gettempdir()) / f"agentenv-{os.getuid()}" + with contextlib.suppress(FileExistsError): + folder.mkdir(mode=0o700) + held = folder.lstat() + if stat.S_ISDIR(held.st_mode) and held.st_uid == os.getuid() and not held.st_mode & 0o077: + return folder / _HOST_PORT_CURSOR + locks = state_root() / "locks" + ensure_state_dir(locks) + return locks / _HOST_PORT_CURSOR + + def _bindable(port: int) -> bool: """Whether ``port`` is free on every interface, as docker binds it when publishing: a port free on loopback alone can still be taken on another interface. A bind that fails for another reason raises.""" diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py index 9016cc6d0..f792f8740 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py @@ -15,6 +15,7 @@ import platform import random import socket +import tempfile import threading import pytest @@ -110,6 +111,15 @@ async def test_tunnel_urls_stay_keyed_by_container_port(fake_ports): assert sandbox.tunnel_urls[port] == f"http://127.0.0.1:{sandbox.host_port(port)}" +@pytest.fixture(autouse=True) +def machine_temp(tmp_path, monkeypatch): + """The machine's temp folder, which holds the cursor: this test's own, so parallel workers don't share it.""" + folder = tmp_path / "machine-tmp" + folder.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(folder)) + return folder + + @pytest.fixture def busy(monkeypatch): """Ports the stand-in bind probe finds in use; every other port is free.""" @@ -119,9 +129,7 @@ def busy(monkeypatch): def _last_handed_out(port: int | str) -> None: - cursor = state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR - cursor.parent.mkdir(parents=True, exist_ok=True) - cursor.write_text(str(port)) + local_sandbox_module._host_port_cursor().write_text(str(port)) def test_host_ports_are_handed_out_in_turn_skipping_those_in_use(busy): @@ -146,17 +154,37 @@ def test_a_missing_or_unreadable_cursor_starts_at_a_random_port(busy, monkeypatc assert local_sandbox_module._free_host_port() == 25000 -def test_the_first_port_of_a_state_root_does_not_follow_a_reseeded_random(busy, tmp_path, monkeypatch): - """A trainer reseeds ``random``; fresh state roots must still start apart in the range.""" +def test_the_first_port_on_a_machine_does_not_follow_a_reseeded_random(busy, tmp_path, monkeypatch): + """A trainer reseeds ``random``; fresh machines must still start apart in the range.""" starts = [] - for root in ("a", "b", "c"): - monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / root)) + for machine in ("a", "b", "c"): + (tmp_path / machine).mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path / machine)) random.seed(0) starts.append(local_sandbox_module._free_host_port()) assert len(set(starts)) > 1 +def test_processes_with_their_own_state_roots_hand_out_host_ports_in_turn(busy, tmp_path, monkeypatch): + """As each xdist worker does: a port one drew isn't drawn by the next before docker binds it.""" + _last_handed_out(20100) + drawn = [] + for root in ("a", "b", "c"): + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / root)) + drawn.append(local_sandbox_module._free_host_port()) + + assert drawn == [20101, 20102, 20103] + + +def test_a_machine_folder_others_can_write_leaves_the_cursor_in_the_state_root(busy, machine_temp, monkeypatch): + folder = machine_temp / f"agentenv-{os.getuid()}" + folder.mkdir(mode=0o777) + folder.chmod(0o777) + + assert local_sandbox_module._host_port_cursor() == state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR + + def test_concurrent_draws_never_share_a_host_port(busy): """Each draw opens the cursor afresh, so the lock serialises threads as it does processes.""" drawn: list[int] = [] From 97beb0313d6608094458ceb4d98c1c21cdf1d20a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 12:42:11 -0800 Subject: [PATCH 07/10] test(drivers): name the idle test's two windows Co-Authored-By: Claude Opus 5.5 --- tst/unit/drivers/codex_computer_use_driver_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index e4b88beec..f50062d48 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -98,6 +98,9 @@ async def main(): anyio.run(main) ''') +#: How long the idle test's REPL may sit unused: through its two calls, however slow a busy runner makes them, then for +#: a moment once they're done, so the sweep closes it. +IDLE_THROUGH_CALLS_S, IDLE_AFTER_CALLS_S = 60, 0.2 MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") WINDOWS = Platform(os="windows", version="11", arch="amd64", display="dwm") @@ -356,13 +359,12 @@ async def js(caller: str, code: str) -> str: @pytest.mark.asyncio async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): - """The REPL may sit idle for as long as two calls take on a busy runner, then for a moment once they're done.""" monkeypatch.setattr(module, "SWEEP_S", 0.05) - monkeypatch.setattr(module, "IDLE_S", 60) + monkeypatch.setattr(module, "IDLE_S", IDLE_THROUGH_CALLS_S) async with mcp_session(_driver(module, monkeypatch)) as session: await session.call_tool("js", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) before = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) - monkeypatch.setattr(module, "IDLE_S", 0.2) + monkeypatch.setattr(module, "IDLE_S", IDLE_AFTER_CALLS_S) closed = await _gone(int(before.rpartition(" in ")[2])) after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) From 0f0647c784926e866a08c3ab038f831be166b517 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 14:48:11 -0800 Subject: [PATCH 08/10] feat(task): a run says each kind of lint finding once, and --skip-lint runs past lint's own rules A bundle run, a registration and an eval run said every task's lint warnings, one line each: 1,355 lines for the cua-driver bench, 600 for Online-Mind2Web. They now say each kind once, with how many tasks have it and the first few, which takes those to two lines and one; agent-env task lint still lists each. --skip-lint on run, bundle register and eval run lifts lint's own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them, so a lint mistake never blocks a run; a step's own config problems still refuse it, as they did before lint. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/bundle/plan.py | 37 +++++++++----- src/agent_env/bundle/register.py | 13 +++-- src/agent_env/bundle/run.py | 25 +++++---- src/agent_env/cli/bundle.py | 10 ++-- src/agent_env/cli/eval/run.py | 25 +++++---- src/agent_env/cli/run.py | 12 +++-- src/agent_env/cli/task/lint.py | 2 +- src/agent_env/task/lint.py | 22 ++++++++ tst/unit/bundle/installed_test.py | 6 +-- tst/unit/task/lint_test.py | 84 +++++++++++++++++++++++++++---- 11 files changed, 176 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 99d21b02f..a00b743dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. `task/lint.py` is `agent-env task lint` (a bundle folder, a task file or a stored task): an error is what a task gets wrong before it runs (a field its step doesn't read, a step's own `config_problems`, a headline that grades nothing, an aggregate source no verifier writes) and refuses `agent-env run`, `bundle register`, `task create`, `task validate` and `eval run`; a warning is what may be wrong (a step's `config_warnings`, a store entity read with no version, no verifier, several and no headline) and is only printed. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. `task/lint.py` is `agent-env task lint` (a bundle folder, a task file or a stored task): an error is what a task gets wrong before it runs (a field its step doesn't read, a step's own `config_problems`, a headline that grades nothing, an aggregate source no verifier writes) and refuses `agent-env run`, `bundle register`, `task create`, `task validate` and `eval run` (the run, register and eval run take `--skip-lint`, which lifts lint's own rules but not a step's `config_problems`); a warning is what may be wrong (a step's `config_warnings`, a store entity read with no version, no verifier, several and no headline) and is only printed, each kind once by a run (`task.lint.said`), every one by `task lint`. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 1ec64c843..995106d94 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -57,7 +57,7 @@ from agent_env.store.base import NotFoundError from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.routing import namespace_routing_enabled -from agent_env.task.lint import Severity, lint +from agent_env.task.lint import Finding, lint, refuses from agent_env.task.task import Task from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep @@ -199,7 +199,9 @@ class Plan: alias_uses: Mapping[EntityKind, Mapping[str, AliasUse]] = field(default_factory=dict) agent: AgentOverride | None = None # the agent --agent names for the solver variant: Varied | None = None # what --set changes, written under its variant's ids - lint_warnings: tuple[str, ...] = () # each selected task's task lint warnings, as ": warning: ..." + # Each selected task's task lint findings that didn't refuse it, with its path: its warnings, and its errors too + # when the plan skipped lint. + lint_findings: tuple[tuple[str, Finding], ...] = () def bundle_of(self, entry: BundleEntry) -> Bundle: """The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's.""" @@ -212,14 +214,17 @@ def path(self, entry: BundleEntry) -> str: def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: Sequence[str] = (), - installed: Installed | None = None, agent: str | None = None, default_agents: bool = True) -> Plan: + installed: Installed | None = None, agent: str | None = None, default_agents: bool = True, + skip_lint: bool = False) -> Plan: """Plan a run of ``tasks`` and ``evals``, each a name or id in this bundle. With neither, the run is every eval or, in a bundle without evals, every task. ``installed`` lists the installed bundles whose entities a reference pulls in (``installed_bundles``); without it, an id under one is a store id. ``agent`` names the agent the solver runs as, as a task's reference would, leaving the tasks as written (``Plan.agent``). Without ``default_agents``, a step that leaves its agent unset pulls in no configured default, as for a task written to - run wherever another's default applies. Raises BundleError listing every problem found.""" - return _Planner(resolved, installed, default_agents).plan(tasks, evals, agent) + run wherever another's default applies. With ``skip_lint``, the errors of task lint's own rules are findings it + keeps (``Plan.lint_findings``), not problems; a step's config problems stay problems. Raises BundleError listing + every problem found.""" + return _Planner(resolved, installed, default_agents, skip_lint).plan(tasks, evals, agent) def plan_entities(resolved: ResolvedBundle, ids: Sequence[str], *, installed: Installed | None = None) -> Plan: @@ -246,11 +251,14 @@ class _Pullable: class _Planner: - def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True): + def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True, + skip_lint: bool = False): self.resolved = resolved self.default_agents = default_agents + self.skip_lint = skip_lint self.problems: list[str] = [] - self.lint_warnings: list[str] = [] + self.lint_findings: list[tuple[str, Finding]] = [] + self.lint_refused = 0 self.nodes: dict[_Key, ResolvedEntry | BundleImage] = {} self.rank: dict[_Key, int] = {} self.edges: dict[_Key, list[_Key]] = {} @@ -586,12 +594,15 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr self._check_environment_names(closure, store_latest) self._check_loaded_environments(closure, store_latest) if self.problems: - raise NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems) + refused = NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems) + if self.lint_refused: + refused.add_note(f"task lint found {self.lint_refused} of these; --skip-lint runs the tasks anyway") + raise refused writes = self._ordered(closure) return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest, dict(self.origins), self._published(writes), {kind: dict(uses) for kind, uses in self.alias_uses.items()}, override, - lint_warnings=tuple(self.lint_warnings)) + lint_findings=tuple(self.lint_findings)) def check(self) -> None: """The checks a run makes before it reads a store. A bundle with nothing to run passes only when its @@ -1128,7 +1139,8 @@ def _store_entity(self, ref: Reference, store_latest: dict[tuple[EntityKind, str def _build_tasks(self, closure: set[_Key], *, linted: bool = False) -> None: """Build each task's steps the way a write will, so a bad step fails before anything is written. ``linted``, - for a run or registration, adds task lint: its errors are problems and its warnings ``lint_warnings``. The + for a run or registration, adds task lint: its errors are problems, unless the plan skips lint, and the rest + ``lint_findings``. The store-free check leaves it out, since a folder ``bundle init`` writes leaves user-scoped fields for its runner to fill in.""" registry = get_task_step_registry() @@ -1153,10 +1165,11 @@ def _build_tasks(self, closure: set[_Key], *, linted: bool = False) -> None: self.problems.append(f"{self._path(source.entry)}: {e}") continue for finding in lint(task, source.config, unpinned=False) if linted else (): - if finding.severity is Severity.ERROR: + if refuses(finding, skip_lint=self.skip_lint): self.problems.append(f"{self._path(source.entry)}: {finding.text}") + self.lint_refused += not refuses(finding, skip_lint=True) else: - self.lint_warnings.append(f"{self._path(source.entry)}: {finding}") + self.lint_findings.append((self._path(source.entry), finding)) # Order diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 36d546c20..d9b401619 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -82,6 +82,7 @@ def register_bundle( on_progress: Callable[[str], None] | None = None, sandbox: str | None = None, emulate: bool = True, + skip_lint: bool = False, ) -> Registration: """Register the bundle at ``root``: ``tasks`` and ``evals`` by name, or with neither every task and eval it has, or every entity when it has none, with what they reference. ``prefix`` opens each entry's registry id, the folder's @@ -91,7 +92,8 @@ def register_bundle( run of the folder would (``declaring_for``). ``sandbox`` is the provider its tasks deploy on, as a run's is: each computer env composes only the forms its deploys there boot, and without it every form its base declares. Either way it composes on the env's provider, emulating an arch no link of that one runs natively unless ``emulate`` is - False, which leaves it skipped. + False, which leaves it skipped. ``skip_lint`` registers tasks task lint's own rules + refuse (``Registration.plan.lint_findings``). It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse. So does @@ -101,7 +103,7 @@ def register_bundle( with namespace_routing(): bundle = parse_bundle(Path(root)) resolved = resolve_bundle(bundle) - plan = _plan(resolved, tasks, evals) + plan = _plan(resolved, tasks, evals, skip_lint) ids, problems = registry_ids(plan, _prefix(bundle, prefix)) if problems: raise BundleError(problems) @@ -130,7 +132,7 @@ def registered_line(plan: Plan, done: Materialized) -> str: return f"{_label(plan, done.write)} -> {done.write.kind.store} {done.write.id} v{done.version} ({why})" -def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str]) -> Plan: +def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str], skip_lint: bool = False) -> Plan: """The plan of ``tasks`` and ``evals``, or of every task and eval, or of every entity of a bundle with none.""" entries = resolved.bundle.entries if not tasks and not evals: @@ -138,7 +140,8 @@ def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str]) evals = [entry.name for entry in entries if entry.kind is BundleKind.EVAL] if not tasks and not evals: return plan_entities(resolved, [entry.id for entry in entries], installed=installed_bundles) - return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False) + return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False, + skip_lint=skip_lint) def _prefix(bundle: Bundle, prefix: str | None) -> str: @@ -243,7 +246,7 @@ def _slug(name: str) -> str: # The fields of a Plan that hold no ``@local`` id, so a translated plan keeps them: the store's latest versions, by store # id, the agent ``--agent`` names, which a registration plans without, and task lint's warnings, which name the bundle's # files. -_KEPT = frozenset({"store_latest", "agent", "variant", "lint_warnings"}) +_KEPT = frozenset({"store_latest", "agent", "variant", "lint_findings"}) def translated(plan: Plan, ids: Mapping[str, str]) -> Plan: diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 7521d1e00..c0c63af89 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -32,6 +32,7 @@ from agent_env.store.routing import namespace_routing, run_scope from agent_env.task.store import record_task_cancelled from agent_env.task.interrupts import Interrupts +from agent_env.task.lint import said from agent_env.utils.progress import run_progress from agent_env.task.task import Task, recorded_scores from agent_env.task.teardown import TeardownReport, attempt_epoch, record_teardown, teardown_run @@ -225,6 +226,7 @@ def run_bundle( emulate: bool = True, source: HubSource | None = None, variant: Variant | None = None, + skip_lint: bool = False, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the @@ -237,7 +239,8 @@ def run_bundle( (``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come from its path. ``source`` says where a Hub bundle came from (``HubBundle.source``), which each run's context records under ``metadata["bundle_source"]``. ``variant`` sets env keys for this run (``with_variant``), writing - what it changes under ids of their own and leaving the bundle as written. + what it changes under ids of their own and leaving the bundle as written. ``skip_lint`` runs tasks task lint's own + rules refuse, saying why (``plan_bundle``). Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. @@ -264,7 +267,7 @@ def run_bundle( with namespace_routing(): prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say, declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing, - variant=variant) + variant=variant, skip_lint=skip_lint) plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks] with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) @@ -352,6 +355,7 @@ def dry_run_bundle( id_root: str | None = None, emulate: bool = True, variant: Variant | None = None, + skip_lint: bool = False, ) -> DryRun: """Check what ``run_bundle`` with these arguments checks before its first task runs, and say what it would write and run, writing and running nothing: no entity, ledger row, lock or instance. Each write's version is @@ -365,7 +369,7 @@ def dry_run_bundle( composing = _composing(sandbox, emulate) say = _progress(on_progress) with namespace_routing(): - plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant) + plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant, skip_lint=skip_lint) _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True)) materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), composing=composing, sandbox=sandbox, @@ -408,9 +412,9 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None, agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring, composing: Composing = Composing(), *, solver_endpoint: bool = False, - variant: Variant | None = None) -> PreparedBundle: + variant: Variant | None = None, skip_lint: bool = False) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played, - solver_endpoint=solver_endpoint, variant=variant) + solver_endpoint=solver_endpoint, variant=variant, skip_lint=skip_lint) _say_plan(plan, say, declaring) materialization = _written_with_infra(plan, preflight, say, composing, sandbox) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) @@ -419,14 +423,15 @@ def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandb def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_root: str | None, sandbox: str | None, agent: str | None = None, *, played: bool = False, - solver_endpoint: bool = False, variant: Variant | None = None) -> tuple[Plan, Preflight]: + solver_endpoint: bool = False, variant: Variant | None = None, + skip_lint: bool = False) -> tuple[Plan, Preflight]: bundle = parse_bundle(Path(root), id_root=id_root) varied = with_variant(bundle, variant) if variant is not None else None if varied is not None: tasks, evals = varied.selected(tasks, BundleKind.TASK), varied.selected(evals, BundleKind.EVAL) resolved = resolve_bundle(varied.bundle if varied is not None else bundle) plan = plan_bundle(without_played_agents(resolved) if played else resolved, tasks=tasks, evals=evals, - installed=installed_bundles, agent=agent) + installed=installed_bundles, agent=agent, skip_lint=skip_lint) refuse_store_tasks(plan) if varied is not None: plan = replace(plan, variant=varied) @@ -485,7 +490,7 @@ def refuse_store_tasks(plan: Plan) -> None: def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> None: """Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, the ids what --set changes is written under, what plugin packages the bundle uses that its bundle.toml doesn't - require, as ``declaring`` says (``declare``), and each task's lint warnings.""" + require, as ``declaring`` says (``declare``), and its tasks' lint findings, each kind once.""" for use in (use for uses in plan.alias_uses.values() for use in uses.values()): say(_aliased(use)) if plan.agent is not None: @@ -498,8 +503,8 @@ def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> N say(f"{variant.typed}: {_changed(plan)} written under ids ending in ({variant.label})") for line in declare(plan.bundle, declaring): say(line) - for warning in plan.lint_warnings: - say(warning) + for line in said(plan.lint_findings): + say(line) def _changed(plan: Plan) -> str: diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index 194e7eec8..cd8031c33 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -13,6 +13,7 @@ from agent_env.bundle.requires import Declaring, UnmetRequirements, declaring_for from agent_env.cli._install_prompt import offer_to_install from agent_env.cli.utils import interruptible +from agent_env.task.lint import said from agent_env.entity.scopes import FieldScope @@ -94,8 +95,10 @@ def _report(imported: BundleImport, folder: Path) -> None: @click.option("--no-emulate", is_flag=True, help="Skip, rather than emulate, a computer env's arch the provider it composes on doesn't run " "natively; a later register or env build builds it.") +@click.option("--skip-lint", is_flag=True, + help="Register tasks agent-env task lint refuses for its own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them; a step's own config problems still refuse it.") def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefix: str | None, - dry_run: bool, yes: bool, sandbox: str | None, no_emulate: bool): + dry_run: bool, yes: bool, sandbox: str | None, no_emulate: bool, skip_lint: bool): """Write the bundle at FOLDER into the configured stores under registry ids, so whatever runs the stores' tasks runs what the folder runs. @@ -113,7 +116,8 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi whoever wrote it. A store other than this machine's is written only after you confirm, or with --yes. Registering again writes only what changed since. A computer env whose compose leaves no build a deploy can boot fails the registration once every entity is written, before any task: registering again composes it again.""" - options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox, "emulate": not no_emulate} + options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox, "emulate": not no_emulate, + "skip_lint": skip_lint} click.echo(f"Registering {folder} into:") for line in describe_target(): click.echo(f" {line}") @@ -126,7 +130,7 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi if not dry_run: offer_to_install(e) raise - for line in planned.plan.lint_warnings: + for line in said(planned.plan.lint_findings): click.echo(f" {line}") for line in planned.floating: click.echo(f" floating, since its field takes no version: {line}") diff --git a/src/agent_env/cli/eval/run.py b/src/agent_env/cli/eval/run.py index a34f260c0..5d22af952 100644 --- a/src/agent_env/cli/eval/run.py +++ b/src/agent_env/cli/eval/run.py @@ -35,7 +35,9 @@ def _write_context(context, task_id, output_dir, prefix=""): @click.option("--user-model", default=None, type=str, help="Override the model the user simulator runs on") @click.option("--judge-model", default=None, type=str, help="Override the model rubrics_verifier judges grade with") @click.option("--agent-artifact-id", default=None, type=str, help="Override the agent image artifact for deploy_agent steps") -def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, max_concurrency: int | None, agent_model: str | None, user_model: str | None, judge_model: str | None, agent_artifact_id: str | None): +@click.option("--skip-lint", is_flag=True, + help="Run tasks agent-env task lint refuses for its own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them; a step's own config problems still refuse it.") +def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, max_concurrency: int | None, agent_model: str | None, user_model: str | None, judge_model: str | None, agent_artifact_id: str | None, skip_lint: bool): """Run all tasks in an eval through the configured [runner], concurrently.""" if k < 1: raise click.BadParameter("must be at least 1", param_hint="'--k'") @@ -44,7 +46,7 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, from agent_env.eval import Eval from agent_env.task import Task - from agent_env.task.lint import Severity, lint + from agent_env.task.lint import lint, refuses, said click.echo(f"Fetching eval: id={eval_id} version={eval_version or 'latest'}...") eval_obj = Eval.get(eval_id, version=eval_version) @@ -57,19 +59,16 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, click.echo(f" Loaded task: {task.id} v{task.version} ({len(task.steps)} steps){version_note}") tasks.append(task) - refused = [] - for task in tasks: - for finding in lint(task): - line = f"{task.id} v{task.version}: {finding.text}" - if finding.severity is Severity.ERROR: - refused.append(line) - else: - click.echo(f" Warning: {line}") + findings = [(f"{task.id} v{task.version}", finding) for task in tasks for finding in lint(task)] + refused = [(task, f) for task, f in findings if refuses(f, skip_lint=skip_lint)] if refused: - click.echo(f"Task lint found {len(refused)} error(s), so no run started:", err=True) - for line in refused: - click.echo(f" - {line}", err=True) + liftable = " (--skip-lint runs anyway)" if any(not refuses(f, skip_lint=True) for _, f in refused) else "" + click.echo(f"Task lint found {len(refused)} error(s), so no run started{liftable}:", err=True) + for task, finding in refused: + click.echo(f" - {task}: {finding.text}", err=True) raise SystemExit(1) + for line in said(findings): + click.echo(f" {line}") print_banner() diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index fd1182662..5c373cc3d 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -56,10 +56,12 @@ @click.option("--no-emulate", is_flag=True, help="Skip, rather than emulate, a computer env's arch the provider it composes on doesn't run " "natively; a later run or env build builds it.") +@click.option("--skip-lint", is_flag=True, + help="Run tasks agent-env task lint refuses for its own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them; a step's own config problems still refuse it.") @click.pass_context def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: tuple[str, ...], model: str | None, user_model: str | None, judge_model: str | None, agent: str | None, settings: tuple[str, ...], - sandbox: str | None, keep: bool, dry_run: bool, no_emulate: bool): + sandbox: str | None, keep: bool, dry_run: bool, no_emulate: bool, skip_lint: bool): """Run a bundle's tasks and evals, or list the installed bundles. BUNDLE is a folder, the name of a bundle an installed package provides (PACKAGE/NAME when several @@ -103,9 +105,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t """ if bundle is None: if tasks or evals or model or user_model or judge_model or agent or settings or sandbox or keep or dry_run \ - or no_emulate: + or no_emulate or skip_lint: raise click.UsageError("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, " - "--keep, --dry-run and --no-emulate need a BUNDLE to run") + "--keep, --dry-run, --no-emulate and --skip-lint need a BUNDLE to run") _list_installed() return verbose = ctx.find_root().params.get("verbose") @@ -117,12 +119,12 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t click.echo(_DRY_RUN) _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, agent=agent, on_progress=click.echo, id_root=id_root, emulate=not no_emulate, - variant=variant)) + variant=variant, skip_lint=skip_lint)) return result = run_bundle(root, tasks=tasks, evals=evals, model=model, user_model=user_model, judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, id_root=id_root, keep=keep, emulate=not no_emulate, source=hub.source if hub else None, - variant=variant) + variant=variant, skip_lint=skip_lint) except BundleError as e: if isinstance(e, NotInStore) and hub is not None and hub.setup_note: e.add_note(hub.setup_note) diff --git a/src/agent_env/cli/task/lint.py b/src/agent_env/cli/task/lint.py index f6f625ea1..3e22fc48f 100644 --- a/src/agent_env/cli/task/lint.py +++ b/src/agent_env/cli/task/lint.py @@ -59,7 +59,7 @@ def _bundle(root: Path, task: str | None = None) -> tuple[list[str], list[str]]: plan = plan_bundle(resolve_bundle(bundle), tasks=names, installed=installed_bundles) except BundleError as e: return list(e.problems), [] - return [], list(plan.lint_warnings) + return [], [f"{task}: {finding}" for task, finding in plan.lint_findings] def _file(path: Path) -> tuple[list[str], list[str]]: diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py index c3cb714e5..5ec15557a 100644 --- a/src/agent_env/task/lint.py +++ b/src/agent_env/task/lint.py @@ -60,6 +60,28 @@ def lint(task: Task, step_dicts: Sequence[Mapping[str, Any]] | None = None, *, c return sorted(findings, key=lambda finding: finding.severity is not Severity.ERROR) +def said(findings: Iterable[tuple[str, Finding]]) -> list[str]: + """``findings``, each with the task it's in, as a run says them: each kind once, errors first, after its task when + only one has it, else with how many and the first few, so a bundle of a thousand tasks says each kind once.""" + where: dict[tuple[Severity, str], list[str]] = {} + for task, finding in findings: + where.setdefault((finding.severity, finding.text), []).append(task) + lines = [] + for (severity, text), tasks in sorted(where.items(), key=lambda item: item[0][0] is not Severity.ERROR): + if len(tasks) == 1: + lines.append(f"{tasks[0]}: {severity.value}: {text}") + else: + shown = ", ".join(tasks[:3]) + (f" and {len(tasks) - 3} more" if len(tasks) > 3 else "") + lines.append(f"{severity.value}: {text} ({len(tasks)} tasks: {shown}; agent-env task lint lists each)") + return lines + + +def refuses(finding: Finding, *, skip_lint: bool = False) -> bool: + """Whether ``finding`` refuses a run. ``skip_lint`` lifts lint's own rules, not a step's ``config_problems``, which + refused a run before lint did and fail it when it runs.""" + return finding.severity is Severity.ERROR and not (skip_lint and finding.rule != "step-config") + + def errors(findings: Iterable[Finding]) -> list[Finding]: return [finding for finding in findings if finding.severity is Severity.ERROR] diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index f20354e69..ed5d3b8ba 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -281,11 +281,11 @@ def test_the_cli_says_so_when_no_bundles_are_installed_and_needs_a_bundle_for_it assert CliRunner().invoke(cli, ["run"]).output == "No bundles are installed. agent-env run PATH runs a folder.\n" for option in (["--task", "t"], ["--keep"], ["--dry-run"], ["--user-model", "m"], ["--judge-model", "m"], - ["--agent", "pi"], ["--set", "envs/desktop.driver=reference"], ["--no-emulate"]): + ["--agent", "pi"], ["--set", "envs/desktop.driver=reference"], ["--no-emulate"], ["--skip-lint"]): usage = CliRunner().invoke(cli, ["run", *option]) assert usage.exit_code == 2 - assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, --keep, --dry-run " - "and --no-emulate need a BUNDLE to run") in " ".join(usage.output.split()) + assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, --keep, --dry-run, " + "--no-emulate and --skip-lint need a BUNDLE to run") in " ".join(usage.output.split()) def test_the_cli_runs_an_installed_bundle_by_name_with_ids_rooted_at_its_package(site, monkeypatch, quiet_logs, diff --git a/tst/unit/task/lint_test.py b/tst/unit/task/lint_test.py index 85653c342..717d46b6b 100644 --- a/tst/unit/task/lint_test.py +++ b/tst/unit/task/lint_test.py @@ -1,6 +1,7 @@ """agent-env task lint: each rule with a task that breaks it and one that doesn't, where its findings refuse a run, and the bundles agent-env ships linting clean.""" +import importlib import json from typing import ClassVar @@ -16,7 +17,7 @@ from agent_env.eval.eval import EvalTask from agent_env.store.routing import namespace_routing from agent_env.task import Task -from agent_env.task.lint import Severity, lint +from agent_env.task.lint import Finding, Severity, lint, said from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( AgentPromptResponseVerifierTaskStep, @@ -283,7 +284,7 @@ def test_a_shipped_bundles_tasks_lint_clean(name): expected = [f"tasks/{task}.json: warning: no step grades this task, so a run has no score" for task in tasks ] if name in _UNGRADED else [] - assert sorted(plan.lint_warnings) == sorted(expected) + assert sorted(f"{task}: {finding}" for task, finding in plan.lint_findings) == sorted(expected) @pytest.fixture @@ -364,8 +365,35 @@ def test_eval_run_refuses_an_eval_whose_task_has_a_lint_error_before_any_run(loc result = CliRunner().invoke(cli, ["eval", "run", "--id", "e"]) assert result.exit_code == 1, result.output - assert ("Task lint found 1 error(s), so no run started:\n - t v1: headline_verifiers names 'box', a deploy_sandbox " - "step, which grades nothing") in result.output + assert ("Task lint found 1 error(s), so no run started (--skip-lint runs anyway):\n - t v1: headline_verifiers " + "names 'box', a deploy_sandbox step, which grades nothing") in result.output + + +def test_eval_run_with_skip_lint_says_the_error_and_goes_on_to_run(local_stores, monkeypatch): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + monkeypatch.setattr(importlib.import_module("agent_env.cli.eval.run"), "print_banner", + lambda: (_ for _ in ()).throw(SystemExit(7))) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e", "--skip-lint"]) + + assert result.exit_code == 7, result.output + assert " t v1: error: headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in result.output + + +def test_eval_run_with_skip_lint_still_refuses_a_steps_config_problem(local_stores, monkeypatch): + task = _task(BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e", "--skip-lint"]) + + assert result.exit_code == 1, result.output + assert "Task lint found 1 error(s), so no run started:\n - t v1: step 'tests': criterion #0: probe_file_exists" in ( + result.output) def test_the_cli_takes_a_path_or_an_id_but_not_both(bundle_dir): @@ -373,10 +401,48 @@ def test_the_cli_takes_a_path_or_an_id_but_not_both(bundle_dir): assert CliRunner().invoke(cli, ["task", "lint", str(bundle_dir), "--id", "t"]).exit_code == 2 -def test_a_run_of_a_bundle_with_a_lint_error_is_refused(bundle_dir): - _write(bundle_dir, "t", [BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}]) +_HEADLESS = "headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" + + +def test_a_run_of_a_bundle_with_a_lint_error_is_refused_unless_it_skips_lint_but_not_for_a_steps_config(bundle_dir): + _write(bundle_dir, "t", {"headline_verifiers": ["box"], "steps": [BOX, TESTS]}) + _write(bundle_dir, "u", [BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}]) + + refused = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "t", "--dry-run"]) + skipped = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "t", "--dry-run", "--skip-lint"]) + config = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "u", "--dry-run", "--skip-lint"]) + + assert refused.exit_code != 0 + assert f"tasks/t.json: {_HEADLESS}" in refused.output + assert "task lint found 1 of these; --skip-lint runs the tasks anyway" in refused.output + assert skipped.exit_code == 0, skipped.output + assert f"tasks/t.json: error: {_HEADLESS}" in skipped.output + assert "Would run:\n tasks/t.json v1\n" in skipped.output + assert config.exit_code != 0 + assert "tasks/u.json: step 'tests': criterion #0: probe_file_exists needs paths" in config.output + assert "--skip-lint" not in config.output + + +def test_bundle_register_with_skip_lint_registers_a_task_lint_refuses_saying_why_once(bundle_dir): + _write(bundle_dir, "t", {"headline_verifiers": ["box"], "steps": [BOX, TESTS]}) + + refused = CliRunner().invoke(cli, ["bundle", "register", str(bundle_dir), "--yes"]) + skipped = CliRunner().invoke(cli, ["bundle", "register", str(bundle_dir), "--yes", "--skip-lint"]) + + assert refused.exit_code != 0 + assert skipped.exit_code == 0, skipped.output + assert skipped.output.count(f"tasks/t.json: error: {_HEADLESS}") == 1 + assert Task.get("triage-t").headline_verifiers == ["box"] + - result = CliRunner().invoke(cli, ["run", str(bundle_dir), "--dry-run"]) +def test_a_run_says_each_kind_of_finding_once_with_how_many_tasks_have_it_errors_first(): + doubt, other = (Finding(Severity.WARNING, "no-verifier", text) for text in ("grades nothing", "unpinned")) + wrong = Finding(Severity.ERROR, "step-config", "needs paths", "tests") + findings = [*((f"tasks/{n}.json", doubt) for n in "abcde"), ("tasks/a.json", other), ("tasks/c.json", wrong), + ("tasks/d.json", wrong)] - assert result.exit_code != 0 - assert "tasks/t.json: step 'tests': criterion #0: probe_file_exists needs paths" in result.output + assert said(findings) == [ + "error: step 'tests': needs paths (2 tasks: tasks/c.json, tasks/d.json; agent-env task lint lists each)", + "warning: grades nothing (5 tasks: tasks/a.json, tasks/b.json, tasks/c.json and 2 more; agent-env task lint " + "lists each)", + "tasks/a.json: warning: unpinned"] From 8a761bf3e2a52ce251116d81b21ef12cc662ebf5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:03:14 -0800 Subject: [PATCH 09/10] fix(local): the host-port cursor lives in /tmp, not the process's TMPDIR A process may set TMPDIR for itself, so processes with their own would each keep a cursor, and runs sharing a state root, which coordinated before, would stop. The cursor is now in the user's folder of the machine's /tmp, and in the state root's locks when that folder can't be made or isn't the user's alone. The tasks a run names for a finding before how many more is a named constant. Co-Authored-By: Claude Opus 5.5 --- .../local_docker/provider.py | 18 ++++++++++-------- src/agent_env/task/lint.py | 7 ++++++- .../local_sandbox_host_ports_test.py | 19 ++++++++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 68aa21d1b..051cf62ba 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -127,9 +127,10 @@ def trust_env(directory: str) -> dict[str, str]: # up on Linux, the VM Docker Desktop or Rancher Desktop runs Docker in included; 49152 up on macOS and Windows) and # below Kubernetes' node ports (30000-32767). _HOST_PORTS = range(20000, 30000) -# Holds the last host port handed out, in a folder of the user's own in this machine's temp folder, so every process the -# user runs here hands them out in turn, whatever its state root. +# Holds the last host port handed out, in a folder of the user's own in this machine's /tmp, so every process the user +# runs here hands them out in turn, whatever its state root or TMPDIR. _HOST_PORT_CURSOR = "host-ports" +_MACHINE_TMP = Path("/tmp") def _local_sandbox_root_path() -> Path: @@ -177,13 +178,14 @@ def _free_host_port() -> int: def _host_port_cursor() -> Path: - """The cursor's file: in this user's own folder of the machine's temp folder, shared by processes whose state roots - differ, as each xdist worker's does; in the state root's locks when that folder isn't this user's alone.""" - folder = Path(tempfile.gettempdir()) / f"agentenv-{os.getuid()}" - with contextlib.suppress(FileExistsError): + """The cursor's file: in this user's own folder of the machine's /tmp, not of ``TMPDIR``, which a process may set + for itself, so processes whose state roots or temp folders differ, as each xdist worker's state root does, share + it; in the state root's locks when that folder can't be made or isn't this user's alone.""" + folder = _MACHINE_TMP / f"agentenv-{os.getuid()}" + with contextlib.suppress(OSError): folder.mkdir(mode=0o700) - held = folder.lstat() - if stat.S_ISDIR(held.st_mode) and held.st_uid == os.getuid() and not held.st_mode & 0o077: + held = folder.lstat() if folder.exists() else None + if held and stat.S_ISDIR(held.st_mode) and held.st_uid == os.getuid() and not held.st_mode & 0o077: return folder / _HOST_PORT_CURSOR locks = state_root() / "locks" ensure_state_dir(locks) diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py index 5ec15557a..8f88d95c8 100644 --- a/src/agent_env/task/lint.py +++ b/src/agent_env/task/lint.py @@ -60,6 +60,10 @@ def lint(task: Task, step_dicts: Sequence[Mapping[str, Any]] | None = None, *, c return sorted(findings, key=lambda finding: finding.severity is not Severity.ERROR) +#: How many of the tasks a finding is in a run names, before how many more. +SHOWN_TASKS = 3 + + def said(findings: Iterable[tuple[str, Finding]]) -> list[str]: """``findings``, each with the task it's in, as a run says them: each kind once, errors first, after its task when only one has it, else with how many and the first few, so a bundle of a thousand tasks says each kind once.""" @@ -71,7 +75,8 @@ def said(findings: Iterable[tuple[str, Finding]]) -> list[str]: if len(tasks) == 1: lines.append(f"{tasks[0]}: {severity.value}: {text}") else: - shown = ", ".join(tasks[:3]) + (f" and {len(tasks) - 3} more" if len(tasks) > 3 else "") + more = len(tasks) - SHOWN_TASKS + shown = ", ".join(tasks[:SHOWN_TASKS]) + (f" and {more} more" if more > 0 else "") lines.append(f"{severity.value}: {text} ({len(tasks)} tasks: {shown}; agent-env task lint lists each)") return lines diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py index f792f8740..e4aa42428 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py @@ -15,7 +15,6 @@ import platform import random import socket -import tempfile import threading import pytest @@ -113,10 +112,10 @@ async def test_tunnel_urls_stay_keyed_by_container_port(fake_ports): @pytest.fixture(autouse=True) def machine_temp(tmp_path, monkeypatch): - """The machine's temp folder, which holds the cursor: this test's own, so parallel workers don't share it.""" + """The machine's /tmp, which holds the cursor: this test's own, so parallel workers don't share it.""" folder = tmp_path / "machine-tmp" folder.mkdir() - monkeypatch.setattr(tempfile, "tempdir", str(folder)) + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", folder) return folder @@ -159,24 +158,34 @@ def test_the_first_port_on_a_machine_does_not_follow_a_reseeded_random(busy, tmp starts = [] for machine in ("a", "b", "c"): (tmp_path / machine).mkdir() - monkeypatch.setattr(tempfile, "tempdir", str(tmp_path / machine)) + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", tmp_path / machine) random.seed(0) starts.append(local_sandbox_module._free_host_port()) assert len(set(starts)) > 1 -def test_processes_with_their_own_state_roots_hand_out_host_ports_in_turn(busy, tmp_path, monkeypatch): +def test_processes_with_their_own_state_roots_or_temp_folders_hand_out_host_ports_in_turn(busy, tmp_path, monkeypatch): """As each xdist worker does: a port one drew isn't drawn by the next before docker binds it.""" _last_handed_out(20100) drawn = [] for root in ("a", "b", "c"): monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / root)) + monkeypatch.setenv("TMPDIR", str(tmp_path / root)) drawn.append(local_sandbox_module._free_host_port()) assert drawn == [20101, 20102, 20103] +def test_a_machine_tmp_it_cant_write_leaves_the_cursor_in_the_state_root(busy, machine_temp): + machine_temp.chmod(0o500) + try: + assert local_sandbox_module._host_port_cursor() == ( + state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR) + finally: + machine_temp.chmod(0o700) + + def test_a_machine_folder_others_can_write_leaves_the_cursor_in_the_state_root(busy, machine_temp, monkeypatch): folder = machine_temp / f"agentenv-{os.getuid()}" folder.mkdir(mode=0o777) From 5f00b479cdaf248ae80380f623c5db58de697066 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:15:52 -0800 Subject: [PATCH 10/10] test(local): the cursor's fallback test holds when the tests run as root CodeBuild runs the suite as root, which writes a 0500 folder all the same; a file where /tmp would be stops anyone. Co-Authored-By: Claude Opus 5.5 --- .../local_sandbox_host_ports_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py index e4aa42428..9d860b0c2 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py @@ -177,13 +177,12 @@ def test_processes_with_their_own_state_roots_or_temp_folders_hand_out_host_port assert drawn == [20101, 20102, 20103] -def test_a_machine_tmp_it_cant_write_leaves_the_cursor_in_the_state_root(busy, machine_temp): - machine_temp.chmod(0o500) - try: - assert local_sandbox_module._host_port_cursor() == ( - state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR) - finally: - machine_temp.chmod(0o700) +def test_a_machine_tmp_it_cant_make_its_folder_in_leaves_the_cursor_in_the_state_root(busy, tmp_path, monkeypatch): + """A file where /tmp would be, so not even root makes the folder.""" + (tmp_path / "not-a-folder").write_text("") + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", tmp_path / "not-a-folder") + + assert local_sandbox_module._host_port_cursor() == state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR def test_a_machine_folder_others_can_write_leaves_the_cursor_in_the_state_root(busy, machine_temp, monkeypatch):