Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ dependencies = [
]

[project.optional-dependencies]
# cua_verifier renders a CUA task's spreadsheets, documents, slides and PDFs as text for its judge; without them it names
# each such file as not rendered.
cua = [
"openpyxl>=3.1",
"python-docx>=1.1",
"python-pptx>=1.0",
"pypdf>=6",
]
# agent_env.openenv: bundle tasks served as OpenEnv envs. No cap: OpenEnv ships a minor release a week.
openenv = [
"openenv>=0.9",
Expand Down Expand Up @@ -152,7 +160,7 @@ coreweave = [
"cwsandbox>=1.19.6,<2", # Reject exec streams that close without an exit status.
]
dev = [
"agentenv-framework[aws,boat,boxd,coreweave,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,trl,vercel,verl]", # exercise portable optional providers and stores in CI
"agentenv-framework[aws,boat,boxd,coreweave,cua,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,trl,vercel,verl]", # exercise portable optional providers and stores in CI
"fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x
"Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs
"av>=17", # the protocol's computer extra, whose tests the unit tier runs and which the recording tests decode with
Expand Down
11 changes: 6 additions & 5 deletions src/agent_env/env/envs/computer/utils/inside.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,12 @@ async def read_file(deployed: DeployedSandboxEnv, path: str) -> bytes:


async def run(deployed: DeployedSandboxEnv, argv: list[str], *, timeout_s: float,
env: Optional[dict[str, str]] = None) -> dict:
"""computer-exec/v1 ``run``'s answer for ``argv``, under a request id of its own. A call whose connection breaks
fails rather than being sent again: a driver that restarted would run the command a second time, beside the first,
which outlives the driver that started it."""
params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {})}
env: Optional[dict[str, str]] = None, cwd: Optional[str] = None) -> dict:
"""computer-exec/v1 ``run``'s answer for ``argv``, in ``cwd`` relative to the home, the home by default, under a
request id of its own. A call whose connection breaks fails rather than being sent again: a driver that restarted
would run the command a second time, beside the first, which outlives the driver that started it."""
params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {}),
**({"cwd": cwd} if cwd else {})}
try:
return await deployed.invoke(COMPUTER_EXEC, "run", params, timeout=timeout_s + CALL_MARGIN_S)
except httpx.TransportError as e:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const STEP_COLORS: Record<string, string> = {
collect_artifacts: '#14b8a6',
rubrics_verifier: '#10b981',
always_on_agent_verifier: '#10b981',
cua_verifier: '#10b981',
env_outcome_verifier: '#10b981',
run_code: '#84cc16',
publish: '#6366f1',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export type VerifierKind = 'aggregate' | 'sandbox' | 'rubric' | 'other';
const RUBRICS_VERIFIER_STEP_TYPES: ReadonlySet<string> = new Set([
'rubrics_verifier',
'always_on_agent_verifier',
'cua_verifier',
]);

export function isRubricsVerifierStep(
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/task/episode/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s
"peer_agents": _peer_agents,
"rubrics_verifier": _rubrics_verifier,
"always_on_agent_verifier": _rubrics_verifier,
"cua_verifier": _rubrics_verifier,
**dict.fromkeys((
"verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_builtin_tools",
"verify_a2a_core_protocol", "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents",
Expand All @@ -156,7 +157,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s
"aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage",
"browser_close_tab", "browser_delete_cookies", "browser_lease_context", "browser_open_tab",
"browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli",
"combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording",
"combine_universe_verdicts", "computer_exec", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording",
"end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier",
"modify_env_tool_access", "reset_env", "review", "user_sim_judge",
"run_container_unit_tests_verifier", "run_docker_container", "set_env_clock_rate", "snapshot_env", "snapshot_computer_env",
Expand Down
4 changes: 4 additions & 0 deletions src/agent_env/task_step/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudge
from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier
from agent_env.task_step.task_steps.verifiers.cua_verifier import CuaVerifierTaskStep
from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep
from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep
from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import VerifyMCPToolSchemaTaskStep
Expand Down Expand Up @@ -104,6 +105,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep
from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep
from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep
from agent_env.task_step.task_steps.computer.computer_exec import ComputerExecTaskStep
from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep
from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep
from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep
Expand Down Expand Up @@ -151,6 +153,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
_get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep,
_get_type(UserSimJudge): UserSimJudge,
_get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier,
_get_type(CuaVerifierTaskStep): CuaVerifierTaskStep,
_get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep,
_get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep,
_get_type(VerifyMCPToolSchemaTaskStep): VerifyMCPToolSchemaTaskStep,
Expand All @@ -174,6 +177,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
_get_type(SnapshotAgentStateTaskStep): SnapshotAgentStateTaskStep,
_get_type(SnapshotEnvTaskStep): SnapshotEnvTaskStep,
_get_type(SnapshotComputerEnvTaskStep): SnapshotComputerEnvTaskStep,
_get_type(ComputerExecTaskStep): ComputerExecTaskStep,
_get_type(RunCodeTaskStep): RunCodeTaskStep,
_get_type(PeerAgentsTaskStep): PeerAgentsTaskStep,
_get_type(VerifyA2ASnapshotStep): VerifyA2ASnapshotStep,
Expand Down
188 changes: 188 additions & 0 deletions src/agent_env/task_step/task_steps/computer/computer_exec.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
"""computer_exec: a command run inside a deployed computer as its session user, through its driver's computer-exec/v1
``run``, a control route the agent can't call, as a task sets a computer up: a program run to its end, its exit status
and output recorded, or with ``background`` an app started and left running."""

from __future__ import annotations

import logging
import shlex
from typing import ClassVar, Optional, Union

from agentenv_protocol.computer import COMPUTER_EXEC

from agent_env.entity.refs import EntityRef
from agent_env.env.env import DeployedEnv, DeployedSandboxEnv
from agent_env.env.envs.computer.env import ComputerEnv
from agent_env.env.envs.computer.utils import inside
from agent_env.env.envs.multi_env import MultiEnv
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.preflight import stored_env
from agent_env.task_step.task_step import TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)

#: Where the step records what each run did, by step id.
RECORDS_KEY = "computer_exec"
#: How long a command may run when the step doesn't say.
DEFAULT_TIMEOUT_S = 120.0
#: The most of each stream the record keeps, and the most of stderr a failure says: the end of it.
RECORDED_CHARS = 4000
FAILURE_CHARS = 300
#: How a ``background`` command starts: its program found and executable, else it fails as the shell does with 127,
#: then detached from the run's pipes, so the run returns as the shell does.
_BACKGROUND = ('case "$1" in */*) [ -f "$1" ] && [ -x "$1" ] ;; *) command -v "$1" >/dev/null ;; esac'
' || { echo "computer_exec: $1: not found or not executable" >&2; exit 127; }; '
'"$@" </dev/null >/dev/null 2>&1 &')

Command = Union[list[str], str]


class ComputerExecTaskStep(TaskStep):
"""Run ``command`` inside the computer of the env ``env_id`` as its session user, in ``cwd`` (relative to the home,
the home by default), with ``env`` added to its environment, for at most ``timeout_s``. ``command`` is a program and
its arguments, or a string run by ``sh -c``. For a multi env, ``computer`` names the computer, by its environment
name, and may be left out when the env has one.

The run is recorded in ``context.metadata["computer_exec"][step_id]``: its exit code, the end of its stdout and
stderr, and whether it timed out or was cut. A run that exits non-zero or times out fails the step, unless
``check`` is false. With ``background``, the command is started and left running, its output discarded, as an app
launched for the agent is: the step waits for the start alone, and fails as a run does when its program isn't
there to start."""

type: ClassVar[str] = "computer_exec"
entity_refs = (EntityRef.env("env_id"),)

def __init__(self, id: str, version: Optional[int], env_id: str, command: Command,
computer: Optional[str] = None, cwd: Optional[str] = None, env: Optional[dict[str, str]] = None,
timeout_s: float = DEFAULT_TIMEOUT_S, background: bool = False, check: bool = True,
depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True):
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error)
self.env_id = env_id
self.command = command
self.computer = computer
self.cwd = cwd
self.env = env
self.timeout_s = timeout_s
self.background = background
self.check = check

def config_problems(self) -> list[str]:
problems = []
if not isinstance(self.env_id, str) or not self.env_id:
problems.append(f"step {self.id!r}: env_id must be a non-empty string")
if isinstance(self.command, str):
if not self.command.strip():
problems.append(f"step {self.id!r}: command must not be blank")
elif not (isinstance(self.command, list) and self.command and all(isinstance(a, str) for a in self.command)):
problems.append(f"step {self.id!r}: command must be a non-empty list of strings, or a string run by sh -c")
if self.computer is not None and (not isinstance(self.computer, str) or not self.computer):
problems.append(f"step {self.id!r}: computer, when given, must be a non-empty string")
if self.cwd is not None and (not isinstance(self.cwd, str) or self.cwd.startswith("/")):
problems.append(f"step {self.id!r}: cwd is a folder relative to the session user's home, not {self.cwd!r}")
if self.env is not None and not (isinstance(self.env, dict) and all(
isinstance(k, str) and k and isinstance(v, str) for k, v in self.env.items())):
problems.append(f"step {self.id!r}: env maps variable names to string values")
if isinstance(self.timeout_s, bool) or not isinstance(self.timeout_s, (int, float)) or self.timeout_s <= 0:
problems.append(f"step {self.id!r}: timeout_s must be a number of seconds above 0")
for name in ("background", "check"):
if not isinstance(getattr(self, name), bool):
problems.append(f"step {self.id!r}: {name} must be true or false")
return problems

def preflight(self) -> list[str]:
"""The env this step names when it has no computer, or none of that name, or several when it names none."""
problems = super().preflight()
if not isinstance(self.env_id, str) or not self.env_id or (env := stored_env(self.env_id)) is None:
return problems
if problem := computer_problem(env, self.computer):
problems.append(f"step {self.id!r}: {problem}")
return problems

def to_dict(self) -> dict:
return {**super().to_dict(), "env_id": self.env_id, "command": self.command, "computer": self.computer,
"cwd": self.cwd, "env": self.env, "timeout_s": self.timeout_s, "background": self.background,
"check": self.check}

@classmethod
def from_dict(cls, data: dict) -> ComputerExecTaskStep:
return cls(**cls._base_from_dict(data), env_id=data["env_id"], command=data["command"],
computer=data.get("computer"), cwd=data.get("cwd"), env=data.get("env"),
timeout_s=data.get("timeout_s", DEFAULT_TIMEOUT_S), background=data.get("background", False),
check=data.get("check", True))

def argv(self) -> list[str]:
"""What the driver runs: the command, through ``sh -c`` when it's a string, started detached with
``background``."""
argv = ["sh", "-c", self.command] if isinstance(self.command, str) else list(self.command)
return ["sh", "-c", _BACKGROUND, "computer_exec", *argv] if self.background else argv

async def execute(self, context: TaskStepContext) -> TaskStepContext:
deployed = find_computer(context, self.env_id, self.computer)
if missing := [m for m in inside.missing(deployed) if m.startswith(COMPUTER_EXEC)]:
raise RuntimeError(f"{self.type} {self.id!r}: env {self.env_id!r}'s computer serves no {missing[0]}")
if self.timeout_s > (limit := inside.max_timeout_s(deployed)):
raise ValueError(f"{self.type} {self.id!r}: timeout_s {self.timeout_s} is above the {limit} s the computer "
f"of env {self.env_id!r} lets a command run")
argv = self.argv()
answer = await inside.run(deployed, argv, timeout_s=self.timeout_s, env=self.env, cwd=self.cwd)
record = {"env_id": self.env_id, "computer": self.computer, "argv": argv, "background": self.background,
"exit_code": answer.get("exit_code"), "timed_out": bool(answer.get("timed_out")),
"truncated": bool(answer.get("truncated")),
"stdout": _tail(answer.get("stdout") or "", RECORDED_CHARS),
"stderr": _tail(answer.get("stderr") or "", RECORDED_CHARS)}
context.metadata.setdefault(RECORDS_KEY, {})[self.id] = record
shown = shlex.join(argv)
if record["timed_out"]:
outcome = f"timed out after {self.timeout_s} s"
elif record["exit_code"] != 0:
outcome = f"exited {record['exit_code']}"
else:
logger.info(f"{self.id}: {'started' if self.background else 'ran'} {shown} in env {self.env_id!r}")
return context
message = f"{self.type} {self.id!r}: {shown} {outcome} in env {self.env_id!r}"
if stderr := _tail(record["stderr"].strip(), FAILURE_CHARS):
message += f": {stderr}"
if self.check:
raise RuntimeError(message)
logger.warning(f"{message} (check is off)")
return context


def find_computer(context: TaskStepContext, env_id: str, computer: Optional[str]) -> DeployedSandboxEnv:
"""The deploy record of the computer in ``env_id`` that serves computer-exec/v1 ``run``: the env's own, or one of a
multi env's computers' (``DeployedGatewayEnv.child_envs``), the one ``computer`` names else its only one."""
deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None)
if deployed is None:
raise RuntimeError(f"env {env_id!r} isn't in context.deployed_envs")
children = {name: DeployedEnv.from_dict(data) for name, data in (getattr(deployed, "child_envs", None) or {}).items()}
if computer is not None:
found = children.get(computer)
if found is None:
raise RuntimeError(f"env {env_id!r} has no computer named {computer!r} (it has {sorted(children)})")
return found
if deployed.supports(COMPUTER_EXEC, "run"):
return deployed
computers = [record for record in children.values() if record.supports(COMPUTER_EXEC, "run")]
if len(computers) != 1:
raise RuntimeError(f"env {env_id!r} has {len(computers) or 'no'} computers serving {COMPUTER_EXEC} run"
+ (f": name one as computer, of {sorted(children)}" if computers else ""))
return computers[0]


def computer_problem(env: object, computer: Optional[str]) -> Optional[str]:
if isinstance(env, ComputerEnv):
return None if computer is None else f"env {env.id!r} is a computer env, so it takes no computer name"
if not isinstance(env, MultiEnv):
return f"env {getattr(env, 'id', '?')!r} is no computer env, nor a multi env with one"
browser = env.website_browser().computer
names = [computer_env.environment_name for computer_env in (*env.computer_envs, *([browser] if browser else []))]
if computer is not None and computer not in names:
return f"env {env.id!r} has no computer named {computer!r} (it has {names})"
if computer is None and len(names) != 1:
return (f"env {env.id!r} has {len(names) or 'no'} computers" +
(f": name one as computer, of {names}" if names else ""))
return None


def _tail(text: str, chars: int) -> str:
return text if len(text) <= chars else "…" + text[-chars:]
Loading
Loading