Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/agent_env/bundle/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,16 +296,18 @@ def prepare_bundle(
played: bool = False,
agent: str | None = None,
solver_endpoint: bool = False,
emulate: bool = True,
) -> PreparedBundle:
"""Do what ``run_bundle`` with these arguments does before its first task runs, and stop there. ``played``
leaves out what each task's only prompted agent would deploy, for a caller that plays that agent itself
(``EpisodeSpec``), so the agent needn't exist. With ``solver_endpoint`` the caller gives the solver's deploys their
model endpoint in each run's ``step_params``, so the config needn't set one.
model endpoint in each run's ``step_params``, so the config needn't set one. ``emulate`` controls foreign-arch
builds as it does for ``run_bundle``.

It doesn't turn namespace routing on: the caller keeps it on for as long as it runs the tasks
(``agent_env.store.routing.namespace_routing``), which suits a service that runs nothing else. Raises what
``run_bundle`` raises before its first task runs, except for a running event loop, which it doesn't need."""
composing = _composing(sandbox)
composing = _composing(sandbox, emulate)
return _prepare(root, tasks, evals, sandbox, id_root, agent, played, _progress(on_progress), _hint(id_root),
composing, solver_endpoint=solver_endpoint)

Expand Down
4 changes: 4 additions & 0 deletions src/agent_env/computers/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ are, under a tag of its recipe, so another machine composing on it pulls it rath
no ISO. A compose installs a base afresh once fewer than 7 days of its evaluation are left, from the ISO Microsoft's
link serves then. Name it as `windows-11`, or by its id, `@local/agentenv-framework/computers/windows-11`.

Docker opens `/dev/kvm` for the QEMU launcher. The user running agent-env needs access to Docker,
but does not need read or write permission on that device. If Docker cannot pass the device to the launcher,
the container fails to start and the compose reports the error.

A Windows computer env is composed on the same machine as a thin disk (`env/envs/computer/utils/thin_disk.py`, with
what's Windows' own in `windows_disk.py`): the base booted on an overlay kept in a folder of its own, each layer
installed over ssh as `agentadmin` by a PowerShell script composing writes, as its `[install.windows]` says (its files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@


def kvm_platforms() -> frozenset[str]:
"""This machine's platform when its Docker can pass KVM to a container: /dev/kvm here, readable and writable; and
windows/amd64 on an amd64 machine, whose launcher boots a Windows guest too."""
if not os.access(KVM, os.R_OK | os.W_OK):
"""Disk platforms when this machine has a KVM device. Docker opens it for the launcher, so the client's user
need not have device access. A daemon that cannot pass the device fails when it starts the launcher."""
if not Path(KVM).is_char_device():
return frozenset()
here = host_platform()
return frozenset({here, "windows/amd64"} if here == "linux/amd64" else {here})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here
from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, forward, reset_input, unresettable
from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker
from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_for
from agent_env.providers.sandbox_providers.local_docker.disk import KVM, kvm_platforms, launcher_args, machine_for
from agent_env.providers.sandbox_providers.local_docker.relay import relay_image
from agent_env.providers.sandbox_providers.sandbox import (
SANDBOX_LABEL,
Expand Down Expand Up @@ -116,6 +116,8 @@ def trust_env(directory: str) -> dict[str, str]:
# Marker dropped in the work dir when this sandbox runs a container, so a later get_sandbox()
# (post-run teardown reconstructs the sandbox from disk) knows it owns that container.
_CONTAINER_MODE_MARKER = ".agent-container-mode"
# A CLI failure can leave the daemon still creating this container; absence alone does not confirm its removal.
_CONTAINER_START_MARKER = ".container-start-pending"
# The sandbox's container port -> host port map, so a sandbox rebuilt from its id publishes and finds the same ports.
_PORT_MAP_FILE = ".port-map.json"
# The internal network a container with no network runs on, so a sandbox rebuilt from its id removes it and its relay.
Expand Down Expand Up @@ -364,9 +366,8 @@ async def terminate(self) -> None:
this, local runs leak their containers/compose stacks, which squat host ports and block the
next deploy. Prod backends override terminate() to tear the whole VM down.
Processes its commands left running on this machine are stopped first (on Linux; see ``_marked_pids``),
then, once none is left to run from it, what its steps staged in /tmp is removed. With no Docker installed,
nothing of it can have been started; with Docker unreachable, the teardown fails only for a sandbox whose folder
records Docker objects of its own.
then, once none is left to run from it, what its steps staged in /tmp is removed. Unavailable Docker fails
teardown for a sandbox whose folder records Docker objects of its own, even if its CLI is now missing.
"""
failures: list[Exception] = []
try:
Expand All @@ -377,19 +378,18 @@ async def terminate(self) -> None:
failures.append(error)
try:
code, _, stderr = await self.exec_with_output("docker", "version", "--format", "{{.Server.Version}}")
except FileNotFoundError:
code, stderr = None, ""
except FileNotFoundError as error:
code, stderr = None, str(error)
if code == 0:
await self._remove_docker_objects(failures)
elif code is not None:
elif self._made_docker_objects:
reason = stderr.strip()[-_ERROR_TAIL_CHARS:]
if self._made_docker_objects:
failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, "
f"compose stack or network: {reason}"))
else:
logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. "
"Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)",
reason, self.sandbox_id, SANDBOX_LABEL, self.sandbox_id)
failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, "
f"compose stack or network: {reason}"))
elif code is not None:
logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. "
"Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)",
stderr.strip()[-_ERROR_TAIL_CHARS:], self.sandbox_id, SANDBOX_LABEL, self.sandbox_id)
if failures:
raise RuntimeError("Local sandbox teardown incomplete: " + "; ".join(str(error) for error in failures)) from failures[0]

Expand Down Expand Up @@ -419,14 +419,18 @@ async def _remove_docker_objects(self, failures: list[Exception]) -> None:
failures.append(error)

async def _remove_own_container(self) -> None:
"""Remove this sandbox's named container, or verify that it is absent."""
"""Remove its named container; an unconfirmed start may still create one after an empty lookup."""
name = f"name=^{re.escape(self.container_name)}$"
code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", name)
if code != 0:
raise RuntimeError(f"Could not list sandbox {self.sandbox_id}'s container: "
f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}")
if containers := stdout.split():
await self.exec_script(f"docker rm -f {shlex.join(containers)} >/dev/null")
(self._work_dir / _CONTAINER_START_MARKER).unlink(missing_ok=True)
elif (self._work_dir / _CONTAINER_START_MARKER).exists():
raise RuntimeError(f"Container creation for sandbox {self.sandbox_id} is not confirmed; "
f"retain {self.work_dir} and retry teardown")

async def _remove_labeled(self) -> None:
"""Remove labeled containers and networks; images can still be shared and are best-effort."""
Expand Down Expand Up @@ -806,11 +810,14 @@ async def create_vm(
(sandbox.work_dir / _VM_SIZE_FILE).write_text(json.dumps({"cpu": cpu, "memory": memory}))
try:
await self._boot_disk(sandbox, image=image, env=env or {})
(sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name)
except BaseException:
with contextlib.suppress(Exception):
except BaseException as error:
try:
await sandbox.terminate()
await asyncio.to_thread(remove_local_work_dir, sandbox.sandbox_id)
except Exception as cleanup_error:
note = f"Cleanup of sandbox {sandbox.sandbox_id} incomplete; retained {sandbox.work_dir}: {cleanup_error}"
error.add_note(note)
logger.warning("%s", note)
raise
return sandbox

Expand All @@ -834,15 +841,29 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str,
publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port)
for arg in ("-p", spec)]
grants = await asyncio.to_thread(local_grants)
running = asyncio.ensure_future(asyncio.to_thread(
docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}",
*publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]),
*(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_QUERY_SECONDS))
state = sandbox.work_dir / "vm-state"
state.mkdir(mode=0o700)
# The launcher owns its files as this user, so teardown needs no privileged recursive cleanup.
state_args = ("--mount", f"type=bind,src={state},dst=/var/lib/agentenv-vm",
"--user", f"{os.getuid()}:{os.getgid()}", "--group-add", str(os.stat(KVM).st_gid))
(sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name)
pending = sandbox.work_dir / _CONTAINER_START_MARKER
pending.write_text(sandbox.container_name)

def start() -> None:
docker("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}",
*publish, *machine.args, *state_args, *launcher_args(ports, env, size["cpu"], size["memory"]),
*(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_START_SECONDS)
pending.unlink(missing_ok=True)

running = asyncio.ensure_future(asyncio.to_thread(start))
try:
await asyncio.shield(running)
except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it
with contextlib.suppress(Exception):
except asyncio.CancelledError as cancelled:
try:
await running
except Exception as error:
cancelled.add_note(f"Docker launcher failed during cancellation: {type(error).__name__}")
raise

async def create_container(
Expand Down Expand Up @@ -1107,6 +1128,8 @@ def _forward_grants(container: str, port: int) -> None:


_DOCKER_QUERY_SECONDS = 30
# Starting a disk container also mounts its image; give creation its own bounded budget.
_DOCKER_START_SECONDS = 120
#: How long a reset may take: longer than the minute the launcher's reset waits for the computer to start again.
_RESET_SECONDS = 90
#: Where a disk's launcher finds the local transfer CA's trust files.
Expand Down
45 changes: 41 additions & 4 deletions tst/unit/bundle/run_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,17 +267,54 @@ def test_the_model_and_sandbox_reach_each_run_and_one_invocation_shares_a_run_gr
]


def test_preparing_for_a_caller_that_plays_the_agent_needs_only_that_agent_to_exist_no_more(bundle_dir):
@pytest.mark.parametrize("options, expected_emulation", [({}, True), ({"emulate": False}, False)],
ids=["default", "native-only"])
def test_preparing_passes_composition_options_and_keeps_selection_without_running_tasks(
bundle_dir, monkeypatch, options, expected_emulation):
compositions, progress = [], []
write = run_module.materialize
provider = object()
providers = []

def selected_provider(name):
providers.append(name)
return provider

def record_composition(plan, **hooks):
compositions.append((hooks["composing"], hooks["sandbox"]))
return write(plan, **hooks)

monkeypatch.setattr(run_module, "build_sandbox_provider", selected_provider)
monkeypatch.setattr(run_module, "materialize", record_composition)
with namespace_routing():
prepared = prepare_bundle(
bundle_dir, tasks=["unnamed"], evals=["smoke"], sandbox="local",
id_root="@local/prepared", on_progress=progress.append, **options)

[(composing, sandbox)] = compositions
assert composing.emulate is expected_emulation
assert composing.forms_of == "deploys"
assert composing.sandbox_provider is None and composing.deploys_on is provider
assert sandbox == "local" and providers == ["local"]
assert [(entry.name, task.id, task.version) for entry, task in prepared.tasks] == [
("a", "@local/prepared/a", 1), ("b", "@local/prepared/b", 1),
("unnamed", "@local/prepared/unnamed", 1)]
assert progress
assert _Scored.seen == []


@pytest.mark.parametrize("options", [{}, {"emulate": False}], ids=["default", "native-only"])
def test_preparing_for_a_caller_that_plays_the_agent_needs_only_that_agent_to_exist_no_more(bundle_dir, options):
layout(bundle_dir, {"tasks/play.json": json.dumps(PLAYED)})
helper = {"id": "helper", "type": "deploy_agent", "agent_name": "helper", "a2a_agent_id": "nobody-else"}

with namespace_routing():
with pytest.raises(BundleError, match="'nobody'"):
prepare_bundle(bundle_dir, tasks=["play"])
prepared = prepare_bundle(bundle_dir, tasks=["play"], played=True)
prepare_bundle(bundle_dir, tasks=["play"], **options)
prepared = prepare_bundle(bundle_dir, tasks=["play"], played=True, **options)
layout(bundle_dir, {"tasks/play.json": json.dumps([*PLAYED, helper])})
with pytest.raises(BundleError) as refused:
prepare_bundle(bundle_dir, tasks=["play"], played=True)
prepare_bundle(bundle_dir, tasks=["play"], played=True, **options)

[(entry, task)] = prepared.tasks
assert (entry.name, [step.type for step in task.steps]) == (
Expand Down
Loading