From 27f442fa0515718cac068580cc381a29167c6311 Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Thu, 19 Nov 2026 17:57:46 -0500 Subject: [PATCH 1/3] feat(env-providers): tear down one slot of a shared VM tear_down_slot ends one slot's stack, leaving the VM and every other slot running: it removes the containers, volumes and networks its compose project labels, then its dir and scratch files. It needs no compose file, so it also clears a slot whose deploy failed early, and it is idempotent, so a retry is safe. It refuses the default stack and any layout not made by for_slot, so its file glob can't reach another stack's files. Nothing calls it yet. Co-Authored-By: Claude Opus 5.5 --- .../env_providers/env_gateway_provider.py | 17 ++++ .../providers/env_providers/stack_layout.py | 27 +++++++ .../env_providers/slot_teardown_test.py | 79 +++++++++++++++++++ .../env_providers/stack_layout_test.py | 55 +++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 tst/integration/providers/env_providers/slot_teardown_test.py diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index bafe2b816..c352bbfff 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -135,6 +135,23 @@ def uses_database(self) -> bool: return _ENV_NETWORK in self.networks +TEARDOWN_RETRIES = 2 + + +async def tear_down_slot(sandbox: Sandbox, layout: StackLayout) -> None: + """End one slot's stack on a VM other slots share, leaving the VM and every other slot running. Idempotent, so a + retry after a dropped connection is safe. Raises for the default stack, or for a sandbox that is not a VM, since + only a VM holds slots; raises too if the script fails, so the caller never counts a slot gone that isn't. + + The caller must first cancel or await the slot's own deploy: a ``compose up`` still running can create containers + after the script has listed them, and the slot would be counted gone while it still holds its ports.""" + if not layout.is_slot: + raise ValueError("tear_down_slot ends one slot of a shared VM; the default stack ends with its VM") + if sandbox.mode != SANDBOX_MODE_VM: + raise ValueError(f"slot {layout.compose_project} can only be on a VM, not a {sandbox.mode} sandbox") + await sandbox.exec_script(layout.teardown_script, max_retries=TEARDOWN_RETRIES) + + @dataclass class DeployedGateway: """Result of a gateway deployment.""" diff --git a/src/agent_env/providers/env_providers/stack_layout.py b/src/agent_env/providers/env_providers/stack_layout.py index f3277ec36..04ff5c723 100644 --- a/src/agent_env/providers/env_providers/stack_layout.py +++ b/src/agent_env/providers/env_providers/stack_layout.py @@ -8,6 +8,7 @@ import posixpath import re +import shlex from dataclasses import dataclass from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR @@ -133,6 +134,32 @@ def from_record(cls, data: object) -> StackLayout: raise ValueError(f"a recorded stack layout must be a slot's, got {data!r}") return layout + @property + def teardown_script(self) -> str: + """A root shell script that removes this slot's stack and nothing else on the VM: every container, volume and + network its compose project labels, then its dir and scratch files. Images stay, since every stack on the VM + shares them. Idempotent, and it needs no compose file, so it also clears a slot whose deploy failed before + writing one. Raises for the default stack, which has the VM to itself and goes with it, and for any layout + not made by ``for_slot``: a hand-built project such as ``s1-x`` would have its files caught by ``s1``'s glob.""" + if not self.is_slot: + raise ValueError("only a slot's stack is torn down on its own; the default stack ends with its VM") + index = self.compose_project[1:] + if not (self.compose_project.startswith("s") and index.isascii() and index.isdigit() + and self == StackLayout.for_slot(int(index))): + raise ValueError(f"only a layout made by StackLayout.for_slot is torn down, so its dir and scratch glob " + f"can't reach another stack's files; got {self}") + label = shlex.quote(f"label=com.docker.compose.project={self.compose_project}") + return "\n".join([ + "set -eu", + f"containers=$(docker ps -aq --filter {label})", + '[ -z "$containers" ] || docker rm -f -v $containers >/dev/null', + f"volumes=$(docker volume ls -q --filter {label})", + '[ -z "$volumes" ] || docker volume rm -f $volumes >/dev/null', + f"networks=$(docker network ls -q --filter {label})", + '[ -z "$networks" ] || docker network rm $networks >/dev/null', + f"rm -rf {shlex.quote(self.app_dir)} {shlex.quote(self.scratch_prefix)}*", + ]) + def scratch_path(self, name: str) -> str: return f"{self.scratch_prefix}{name}" diff --git a/tst/integration/providers/env_providers/slot_teardown_test.py b/tst/integration/providers/env_providers/slot_teardown_test.py new file mode 100644 index 000000000..b9303700a --- /dev/null +++ b/tst/integration/providers/env_providers/slot_teardown_test.py @@ -0,0 +1,79 @@ +"""``StackLayout.teardown_script`` against a real Docker daemon: it ends one slot's containers, volumes, networks, dir and +scratch files, and leaves a neighbouring slot whole. The script runs in a throwaway Docker CLI container that holds the +slots' dirs, so it never touches this machine's /app or /tmp. Needs Docker.""" + +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest + +from agent_env.providers.env_providers.stack_layout import StackLayout +from tst.util.capabilities import missing_capability_reason + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BUSYBOX = "mirror.gcr.io/library/busybox:1.37" +DOCKER_CLI = "mirror.gcr.io/library/docker:28-cli" +COMPOSE = f"""services: + web: + image: {BUSYBOX} + command: ["sleep", "3600"] + volumes: ["data:/data", "/scratch"] +volumes: + data: {{}} +""" +GONE, KEPT = StackLayout.for_slot(43), StackLayout.for_slot(44) + + +def _docker(*args: str) -> str: + return subprocess.run(["docker", *args], check=True, capture_output=True, text=True).stdout + + +def _labelled(kind: str, layout: StackLayout) -> list[str]: + listing = ["ps", "-aq"] if kind == "container" else [kind, "ls", "-q"] + return _docker(*listing, "--filter", f"label=com.docker.compose.project={layout.compose_project}").split() + + +@pytest.fixture +def two_slots(tmp_path: Path): + if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0: + pytest.skip(missing_capability_reason("docker_daemon")) + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text(COMPOSE) + for layout in (GONE, KEPT): + _docker("compose", "-p", layout.compose_project, "-f", str(compose_file), "up", "-d", "--quiet-pull") + try: + yield + finally: + for layout in (GONE, KEPT): + subprocess.run(["docker", "compose", "-p", layout.compose_project, "-f", str(compose_file), "down", "-v"], + capture_output=True) + + +def _run_teardown_twice_beside_both_slots_files() -> str: + files = f"{GONE.app_dir} {KEPT.app_dir} {GONE.scratch_path('pgdata')} {KEPT.scratch_path('pgdata')} /tmp/s430-other" + script = "\n".join([ + f"mkdir -p {GONE.app_dir} {KEPT.app_dir} && touch {GONE.scratch_path('pgdata')} {KEPT.scratch_path('pgdata')} " + "/tmp/s430-other", + GONE.teardown_script, + GONE.teardown_script, + f"for f in {files}; do [ -e $f ] && echo kept $f || echo gone $f; done", + ]) + return _docker("run", "--rm", "--name", f"slot-teardown-{uuid.uuid4().hex[:8]}", + "-v", "/var/run/docker.sock:/var/run/docker.sock", DOCKER_CLI, "sh", "-c", script) + + +def test_tearing_down_one_slot_ends_its_stack_and_files_and_leaves_its_neighbour_whole(two_slots): + kept_before = {kind: _labelled(kind, KEPT) for kind in ("container", "volume", "network")} + assert all(kept_before.values()) and all(_labelled(kind, GONE) for kind in kept_before) + + report = _run_teardown_twice_beside_both_slots_files() + + assert {kind: _labelled(kind, GONE) for kind in kept_before} == {"container": [], "volume": [], "network": []} + assert {kind: _labelled(kind, KEPT) for kind in kept_before} == kept_before + assert report.split("\n")[:-1] == [ + f"gone {GONE.app_dir}", f"kept {KEPT.app_dir}", f"gone {GONE.scratch_path('pgdata')}", + f"kept {KEPT.scratch_path('pgdata')}", "kept /tmp/s430-other", + ] diff --git a/tst/unit/providers/env_providers/stack_layout_test.py b/tst/unit/providers/env_providers/stack_layout_test.py index 9a6ccf1f0..199c71de4 100644 --- a/tst/unit/providers/env_providers/stack_layout_test.py +++ b/tst/unit/providers/env_providers/stack_layout_test.py @@ -11,9 +11,11 @@ from agent_env.env.envs.service_db import DB_MCP_PORT, DB_WEB_PORT, ServiceDBConfig from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT from agent_env.providers.env_providers.env_gateway_provider import ( + TEARDOWN_RETRIES, EnvironmentGatewayProvider, MCPServerConfig, SidecarConfig, + tear_down_slot, ) from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout @@ -395,3 +397,56 @@ def test_the_default_stack_records_nothing_and_a_record_without_a_layout_is_the_ def test_a_damaged_recorded_layout_is_refused_rather_than_read_as_the_default_stack(data): with pytest.raises((ValueError, TypeError)): StackLayout.from_record(data) + + +def test_the_default_stack_is_never_torn_down_on_its_own(): + with pytest.raises(ValueError, match="default stack"): + DEFAULT_STACK_LAYOUT.teardown_script + + +@pytest.mark.parametrize("layout", [ + StackLayout(compose_project="s1"), + StackLayout(app_dir="/app/s2", compose_project="s1", scratch_prefix="/tmp/s1-"), + StackLayout(app_dir="/app/s1", compose_project="s1", scratch_prefix="/tmp/"), + StackLayout(app_dir="/app/s1", compose_project="s1", scratch_prefix="/tmp/s1"), + StackLayout(app_dir="/app/s1-x", compose_project="s1-x", scratch_prefix="/tmp/s1-x-", host_port_offset=2000), + StackLayout(app_dir="/app/s1", compose_project="s1", scratch_prefix="/tmp/s1-", host_port_offset=5000), + StackLayout(app_dir="/app/web", compose_project="web", scratch_prefix="/tmp/web-", host_port_offset=1000), + StackLayout(app_dir="/app/s01", compose_project="s01", scratch_prefix="/tmp/s01-", host_port_offset=2000), +]) +def test_a_slot_layout_not_made_by_for_slot_is_never_torn_down(layout): + with pytest.raises(ValueError, match="for_slot"): + layout.teardown_script + + +def test_a_slots_teardown_names_only_its_own_project_dir_and_scratch_files(): + script = StackLayout.for_slot(7).teardown_script + assert script.count("label=com.docker.compose.project=s7") == 3 + assert script.splitlines()[-1] == "rm -rf /app/s7 /tmp/s7-*" + + +@pytest.mark.asyncio +async def test_tearing_down_a_slot_runs_its_script_on_the_vm_and_may_retry(): + vm = _RecordingVm([], set()) + vm.exec_script = AsyncMock(return_value="") + await tear_down_slot(vm, StackLayout.for_slot(2)) + vm.exec_script.assert_awaited_once_with(StackLayout.for_slot(2).teardown_script, max_retries=TEARDOWN_RETRIES) + + +@pytest.mark.asyncio +async def test_a_failed_slot_teardown_raises_so_the_slot_is_never_counted_gone(): + vm = _RecordingVm([], set()) + vm.exec_script = AsyncMock(side_effect=RuntimeError("Script failed (exit 1)")) + with pytest.raises(RuntimeError, match="exit 1"): + await tear_down_slot(vm, StackLayout.for_slot(2)) + + +@pytest.mark.asyncio +async def test_only_a_slot_on_a_vm_can_be_torn_down(): + vm = _RecordingVm([], set()) + with pytest.raises(ValueError, match="default stack"): + await tear_down_slot(vm, DEFAULT_STACK_LAYOUT) + vm.mode = "container" + with pytest.raises(ValueError, match="only be on a VM"): + await tear_down_slot(vm, StackLayout.for_slot(2)) + assert vm.calls == [] From 7b8f7cb42bed967fc309eb2255cd3ebaa6ef997f Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Thu, 19 Nov 2026 18:39:17 -0500 Subject: [PATCH 2/3] feat(sandbox-providers): a shared Modal VM is ended only by terminate_shared_box Several tasks' slots will share one Modal VM, a shared box, so no single task's close may end it. Every Modal terminate now goes through one check: a handle this process created terminates as before; one reattached by id, and terminate_sandbox, first read the sandbox's tags and refuse, with SharedBoxTerminateRefused, to end one tagged agent_env_shared_box. A tag read that keeps failing raises and terminates nothing. create_shared_box tags the VM at create, and terminate_shared_box ends a box only when its tag names that box. Callers already log a failed terminate, and a teardown reports the box as not terminated. Nothing creates boxes yet. Co-Authored-By: Claude Opus 5.5 --- .../providers/sandbox_providers/AGENTS.md | 1 + .../sandbox_providers/modal_sandbox.py | 86 +++++- .../sandbox_providers/modal_vm_sandbox.py | 48 +++- .../sandbox_providers/sandbox_provider.py | 15 + tst/unit/env/envs/shared_box_close_test.py | 130 +++++++++ .../sandbox_providers/modal_sandbox_test.py | 1 + .../modal_shared_box_test.py | 269 ++++++++++++++++++ .../modal_vm_sandbox_test.py | 1 + tst/unit/task/teardown_test.py | 1 + 9 files changed, 542 insertions(+), 10 deletions(-) create mode 100644 tst/unit/env/envs/shared_box_close_test.py create mode 100644 tst/unit/providers/sandbox_providers/modal_shared_box_test.py diff --git a/src/agent_env/providers/sandbox_providers/AGENTS.md b/src/agent_env/providers/sandbox_providers/AGENTS.md index 4727eaff5..dc9e92321 100644 --- a/src/agent_env/providers/sandbox_providers/AGENTS.md +++ b/src/agent_env/providers/sandbox_providers/AGENTS.md @@ -54,6 +54,7 @@ with `exposed_ports=[port]` and returns the VM, and the deploy then loads and st | `create_sandbox(*, image_name, port, env, cpu, memory, disk_size_gb, timeout, attribution, network_policy) -> Sandbox` | yes (a VM backend wraps `create_vm`) | an agent's sandbox (`provider.create_sandbox`); `create_container` of a container-only backend | | `get_sandbox(sandbox_id) -> Sandbox` | yes | every step after the one that created the sandbox, and teardown | | `terminate_sandbox(sandbox_id)` | no | teardown. The default calls `get_sandbox` then `terminate()`, and treats `SandboxNotFoundError` as done; override when you can terminate without a live handle | +| `create_shared_box(box_id, *, cpu, memory, timeout, exposed_ports, setup_for_gateway, attribution, network_policy) -> VmSandbox` | no | nothing yet: a VM for several tasks' slots. The default raises `NotImplementedError`; `modal_vm` tags the VM with `SHARED_BOX_TAG` at create, refuses to end it from its handle's `terminate()` or `terminate_sandbox`, raising `SharedBoxTerminateRefused`, and ends it only through `terminate_shared_box(sandbox_id, box_id)` | | `create_container(...)` | inherited | containers that run an env or agent image. Override only when the platform boots images natively. Its `shm_mb` is the MiB the container's `/dev/shm` holds (`--shm-size` on the default VM path), passed only where `SHARED_MEMORY` is on | | `reset_container(sandbox, *, image_name, port, env, shm_mb)` | inherited | a computer env's reset, which replaces the container `create_container` ran with a fresh one on the same sandbox and port, its `/dev/shm` as before. The default works on a VM backend; one whose sandbox is the container itself raises `NotImplementedError` | | `native_platforms()` | no | a computer env's deploy and gate 3, which boot a build only on a provider that runs its arch without emulation. The default is `linux/amd64`; local Docker returns this machine's | diff --git a/src/agent_env/providers/sandbox_providers/modal_sandbox.py b/src/agent_env/providers/sandbox_providers/modal_sandbox.py index 43c9907de..4438d2b13 100644 --- a/src/agent_env/providers/sandbox_providers/modal_sandbox.py +++ b/src/agent_env/providers/sandbox_providers/modal_sandbox.py @@ -6,6 +6,7 @@ import base64 import collections import contextlib +import enum import functools import hashlib import logging @@ -30,6 +31,7 @@ if TYPE_CHECKING: from agent_env.artifact.artifacts.docker_image import DockerImageArtifact + from agent_env.providers.sandbox_providers.modal_vm_sandbox import ModalVmSandbox logger = logging.getLogger(__name__) @@ -95,14 +97,86 @@ def _tunnel_url(t) -> str: return f"https://{t.host}" + (f":{t.port}" if t.port != 443 else "") -async def _terminate_by_id(client, sandbox_id: str) -> None: +#: The Modal sandbox tag that marks a VM as a shared box, several tasks' slots on one VM; its value is the box's id. +SHARED_BOX_TAG = "agent_env_shared_box" +SHARED_BOX_TAG_READ_FAILED_EVENT = "agent_env.modal_shared_box_tag_read_failed" +_TAG_READ_ATTEMPTS, _TAG_READ_BACKOFF_SECONDS = 3, 1.0 + + +class _Unread(enum.Enum): + UNREAD = "unread" + + +#: A handle's ``shared_box_id`` before its tags are read: one this process reattached to by id. +UNREAD = _Unread.UNREAD +SharedBox = Optional[str] | _Unread + + +class SharedBoxTerminateRefused(Exception): + """A terminate that would have ended a shared box, or a ``terminate_shared_box`` naming a box the sandbox isn't; + the sandbox was left running.""" + + def __init__(self, sandbox_id: str, box_id: Optional[str], ending_box: Optional[str] = None): + self.sandbox_id, self.box_id = sandbox_id, box_id + if ending_box is None: + why = f"is shared box {box_id!r}, which only terminate_shared_box ends" + else: + why = f"is {f'shared box {box_id!r}' if box_id else 'no shared box'}, not shared box {ending_box!r}" + super().__init__(f"Modal sandbox {sandbox_id} {why}; it was not terminated") + + +async def _read_shared_box(sb: modal.Sandbox) -> Optional[str]: + """The shared box ``sb`` is tagged as, if any. A read that keeps failing raises, so a terminate fails closed; + NotFoundError raises at once.""" + attempt = 1 + while True: + try: + return (await sb.get_tags.aio()).get(SHARED_BOX_TAG) + except modal.exception.NotFoundError: + raise + except Exception as e: + if attempt == _TAG_READ_ATTEMPTS: + logger.warning(f"Modal sandbox {sb.object_id} was not terminated: its tags, which say whether it's a " + f"shared box, couldn't be read: {fmt_exc(e)}", + extra={"event": SHARED_BOX_TAG_READ_FAILED_EVENT, "modal_sandbox_id": sb.object_id}) + raise + await asyncio.sleep(_TAG_READ_BACKOFF_SECONDS * 2 ** (attempt - 1)) + attempt += 1 + + +async def _terminate(sb: modal.Sandbox, box: SharedBox, *, ending_box: Optional[str] = None) -> None: + """Terminate ``sb`` when it is the shared box ``ending_box``, or, with none, when it is no shared box; else raise + SharedBoxTerminateRefused. Every Modal terminate goes through here but the cleanup of a create that failed, whose + sandbox holds no slots yet. ``box`` is what the caller knows of ``sb``; UNREAD reads its tags, and one gone already + counts as terminated.""" + if box is UNREAD: + try: + box = await _read_shared_box(sb) + except modal.exception.NotFoundError: + logger.info("Modal sandbox %s was already gone at terminate", sb.object_id) + return + if box != ending_box: + raise SharedBoxTerminateRefused(sb.object_id, box, ending_box) + await sb.terminate.aio() + + +async def _terminate_by_id(client, sandbox_id: str, *, ending_box: Optional[str] = None) -> None: try: sb = await modal.Sandbox.from_id.aio(sandbox_id, client=client) - await sb.terminate.aio() + await _terminate(sb, UNREAD, ending_box=ending_box) except modal.exception.NotFoundError: logger.info("Modal sandbox %s was already gone at terminate", sandbox_id) +async def _terminate_handle(handle: ModalSandbox | ModalVmSandbox) -> None: + """Terminate a handle's sandbox through ``_terminate``, keeping the shared box a refusal found on the handle.""" + try: + await _terminate(handle._sb, handle.shared_box_id) + except SharedBoxTerminateRefused as refused: + handle.shared_box_id = refused.box_id + raise + + def _modal_network_kwargs(policy: NetworkPolicy) -> dict: """``policy`` as Modal sandbox creation kwargs. @@ -194,8 +268,9 @@ class ModalSandbox(Sandbox): type = "modal" def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str], i6pn_address: Optional[str] = None, - network_policy: NetworkPolicy | None = None): + network_policy: NetworkPolicy | None = None, shared_box_id: SharedBox = UNREAD): self._sb = sb + self.shared_box_id = shared_box_id self.sandbox_id = sb.object_id self.tunnel_urls = tunnel_urls self.i6pn_address = i6pn_address @@ -210,7 +285,7 @@ def private_host(self) -> str | None: return f"[{self.i6pn_address}]" if self.i6pn_address else None async def terminate(self) -> None: - await self._sb.terminate.aio() + await _terminate_handle(self) async def exit_status(self) -> Optional[int]: return await self._sb.poll.aio() @@ -511,6 +586,7 @@ async def create_rebuilding_once() -> modal.Sandbox: tunnel_urls, i6pn_address=i6pn_address, network_policy=effective, + shared_box_id=None, ) if vnc_port in tunnel_urls: sandbox.vnc_url = f"{tunnel_urls[vnc_port]}/vnc.html" @@ -589,6 +665,8 @@ def _attribution_tags(attribution: Attribution) -> dict[str, str]: continue if not _MODAL_TAG_KEY.fullmatch(key): raise ValueError(f"attribution key {key!r} can't be a Modal tag: use 1-63 of a-z A-Z 0-9 . _ -") + if key == SHARED_BOX_TAG: + raise ValueError(f"attribution key {key!r} is the tag that marks a shared box; only create_shared_box sets it") tags[key] = _modal_tag_value(value) return tags diff --git a/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py b/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py index f54d90f1a..ec6df5558 100644 --- a/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py +++ b/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py @@ -34,6 +34,10 @@ import modal from agent_env.providers.sandbox_providers.modal_sandbox import ( + _MODAL_TAG_KEY, + SHARED_BOX_TAG, + UNREAD, + SharedBox, _ModalProcessAdapter, _attribution_tags, _ended_output, @@ -41,6 +45,7 @@ _log_sandbox_started, _resolve_app_base_name, _terminate_by_id, + _terminate_handle, _tunnel_url, _modal_network_kwargs, ) @@ -144,8 +149,9 @@ class ModalVmSandbox(VmSandbox): _WFT_CHUNK_BYTES = 32 * 1024 def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str], - network_policy: NetworkPolicy | None = None): + network_policy: NetworkPolicy | None = None, shared_box_id: SharedBox = UNREAD): self._sb = sb + self.shared_box_id = shared_box_id self.sandbox_id = sb.object_id self.tunnel_urls = tunnel_urls self.vnc_url = None @@ -155,7 +161,7 @@ def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str], self._ended: Optional[tuple[list[str], list[str]]] = None async def terminate(self) -> None: - await self._sb.terminate.aio() + await _terminate_handle(self) async def exit_status(self) -> Optional[int]: return await self._sb.poll.aio() @@ -399,6 +405,33 @@ async def create_vm( experimental_options={"enable_nested_virt": True}, secrets=[modal.Secret.from_dict(variables)], idle_timeout=self._idle_timeout or _DISK_IDLE_TIMEOUT_S) + async def create_shared_box( + self, + box_id: str, + *, + cpu: float = 0.5, + memory: int = 8192, + timeout: int = 3600 * 2, + exposed_ports: Optional[list[int]] = None, + setup_for_gateway: bool = True, + attribution: Optional[Attribution] = None, + network_policy: Optional[NetworkPolicy] = None, + ) -> ModalVmSandbox: + """A VM running Docker, as ``create_vm`` makes, tagged as shared box ``box_id`` at create, since a later + ``set_tags`` would replace its attribution tags. Its handle's terminate, and ``terminate_sandbox``, refuse to end + it: only ``terminate_shared_box`` does.""" + if not _MODAL_TAG_KEY.fullmatch(box_id): + raise ValueError(f"shared box id {box_id!r} can't be a Modal tag: use 1-63 of a-z A-Z 0-9 . _ -") + return await self._created(_KEEPALIVE_CMD, _VM_IMAGE, cpu=cpu, memory=memory, timeout=timeout, + ports=list(exposed_ports or []), attribution=attribution, + network_policy=network_policy, setup_for_gateway=setup_for_gateway, + idle_timeout=self._idle_timeout, runtime="vm", shared_box_id=box_id) + + async def terminate_shared_box(self, sandbox_id: str, box_id: str) -> None: + """End shared box ``box_id``, the sandbox ``sandbox_id``, and every slot on it. A sandbox tagged as another box, + or as none, raises SharedBoxTerminateRefused, and one whose tags can't be read raises; neither is terminated.""" + await _terminate_by_id(await self._get_client(), sandbox_id, ending_box=box_id) + async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: """The launcher in the sandbox booting its disk afresh with ``env``, at the same URLs.""" code, _, stderr = await sandbox._exec_with_stdin(shlex.join(RESET), _chunks(reset_input(env))) @@ -408,13 +441,16 @@ async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) - async def _created(self, command: tuple[str, ...], image: modal.Image, *, cpu: float, memory: int, timeout: int, ports: list[int], attribution: Optional[Attribution], network_policy: Optional[NetworkPolicy], - setup_for_gateway: bool, idle_timeout: Optional[int], **create) -> ModalVmSandbox: + setup_for_gateway: bool, idle_timeout: Optional[int], shared_box_id: Optional[str] = None, + **create) -> ModalVmSandbox: """A sandbox of ``image`` running ``command``, in the configured region, its ports' tunnels up, ended once idle - ``idle_timeout`` seconds when given, set up for a gateway when ``setup_for_gateway``; one that fails before - it's returned is terminated.""" + ``idle_timeout`` seconds when given, set up for a gateway when ``setup_for_gateway``, tagged as shared box + ``shared_box_id`` when given; one that fails before it's returned is terminated.""" attribution = dict(attribution or {}) app_tags = _attribution_tags({}) sandbox_tags = _attribution_tags(attribution) + if shared_box_id is not None: + sandbox_tags[SHARED_BOX_TAG] = shared_box_id app_name = self._app_name app = await self._get_app(app_name, app_tags) client = await self._get_client() @@ -449,7 +485,7 @@ async def _created(self, command: tuple[str, ...], image: modal.Image, *, cpu: f for p in ports: if p not in tunnel_urls: raise RuntimeError(f"No Modal tunnel for exposed port {p}; got {sorted(tunnel_urls)}") - sandbox = ModalVmSandbox(sb, tunnel_urls, network_policy=effective) + sandbox = ModalVmSandbox(sb, tunnel_urls, network_policy=effective, shared_box_id=shared_box_id) logger.info(f"Modal VM sandbox created: {sb.object_id} (tunnels={tunnel_urls})") await _log_sandbox_started( sb, app_name=app_name, sandbox_tags=sandbox_tags, diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index 0d314371c..a6f9f9ae4 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -97,6 +97,21 @@ async def create_vm( channel.""" raise NotImplementedError(f"{type(self).__name__} does not support create_vm") + async def create_shared_box( + self, + box_id: str, + *, + cpu: float = 1.0, + memory: int = 8192, + timeout: int = 3600 * 2, + exposed_ports: Optional[list[int]] = None, + setup_for_gateway: bool = True, + attribution: Optional[Attribution] = None, + network_policy: Optional[NetworkPolicy] = None, + ) -> VmSandbox: + """A VM for several tasks' slots, shared box ``box_id``, which no single task's terminate ends.""" + raise NotImplementedError(f"{type(self).__name__} does not support shared boxes") + async def create_container( self, *, diff --git a/tst/unit/env/envs/shared_box_close_test.py b/tst/unit/env/envs/shared_box_close_test.py new file mode 100644 index 000000000..26fbaa91b --- /dev/null +++ b/tst/unit/env/envs/shared_box_close_test.py @@ -0,0 +1,130 @@ +"""No way an env, its provider or a run's teardown closes reaches Modal's terminate for a shared box: each is refused, +logged, and, in a teardown, reported as not terminated.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import modal +import pytest + +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.envs._deployment import close_deployed, close_replaced +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.env.envs.website import WebsiteEnv +from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider +from agent_env.providers.sandbox_providers.modal_sandbox import SHARED_BOX_TAG, ModalSandbox, ModalSandboxProvider +from agent_env.providers.sandbox_providers.modal_vm_sandbox import ModalVmSandbox, ModalVmSandboxProvider +from agent_env.task import teardown +from agent_env.task.teardown import RecordedSandbox +from agent_env.task_step.context import TaskStepContext + + +@pytest.fixture +def box(monkeypatch) -> MagicMock: + """The Modal sandbox of shared box ``box-1``, which ``from_id`` finds, and Modal clients that never connect.""" + sb = MagicMock(object_id="sb-box") + sb.get_tags.aio = AsyncMock(return_value={SHARED_BOX_TAG: "box-1"}) + sb.terminate.aio = AsyncMock() + sb.tunnels.aio = AsyncMock(return_value={}) + sb.exec.aio = AsyncMock(side_effect=RuntimeError("no exec here")) + monkeypatch.setattr(modal.Sandbox, "from_id", MagicMock(aio=AsyncMock(return_value=sb))) + for cls in (ModalVmSandboxProvider, ModalSandboxProvider): + monkeypatch.setattr(cls, "_get_client", AsyncMock(return_value=MagicMock())) + return sb + + +HANDLES = { + "created vm": lambda sb: ModalVmSandbox(sb, {}, shared_box_id="box-1"), + "reattached vm": lambda sb: ModalVmSandbox(sb, {}), + "reattached container": lambda sb: ModalSandbox(sb, {}), +} + + +def _mcp() -> MCPServerEnv: + return MCPServerEnv(id="mcp-email", version=1, docker_image_artifact=None, environment_name="email") + + +def _website() -> WebsiteEnv: + return WebsiteEnv(id="web-email", version=1, backend_docker_image_artifact=None, + frontend_docker_image_artifact=None, environment_name="email") + + +def _multi() -> MultiEnv: + return MultiEnv(id="crm-suite", version=1, mcp_server_envs=[_mcp()]) + + +def _computer() -> ComputerEnv: + return ComputerEnv("desk", 1, "desk", MagicMock(), MagicMock()) + + +async def _env_close(make_env, handle): + env = make_env() + env._env_provider, env._sandbox = None, handle + await env.close() + + +async def _close_replaced(handle): + env = _mcp() + env._replaced = [(None, handle)] + await close_replaced(env) + + +async def _env_provider_close(handle): + provider = EnvironmentGatewayProvider() + provider._container_sandboxes = [handle] + await provider.close() + + +HANDLE_PATHS = { + "MCPServerEnv.close": lambda handle: _env_close(_mcp, handle), + "WebsiteEnv.close": lambda handle: _env_close(_website, handle), + "MultiEnv.close": lambda handle: _env_close(_multi, handle), + "ComputerEnv.close": lambda handle: _env_close(_computer, handle), + "close_replaced": _close_replaced, + "EnvProvider.close": _env_provider_close, +} + + +@pytest.mark.parametrize("handle", HANDLES.values(), ids=HANDLES.keys()) +@pytest.mark.parametrize("close", HANDLE_PATHS.values(), ids=HANDLE_PATHS.keys()) +@pytest.mark.asyncio +async def test_closing_a_handle_to_a_shared_box_leaves_it_running(box, close, handle): + await close(handle(box)) + + box.terminate.aio.assert_not_called() + + +@pytest.mark.parametrize("sandbox_type", ["modal_vm", "modal"]) +@pytest.mark.asyncio +async def test_close_deployed_of_a_plugins_deployment_on_a_shared_box_leaves_it_running(box, monkeypatch, sandbox_type): + env = _mcp() + env._env_provider = object() + monkeypatch.setattr(MCPServerEnv, "from_deployed_env", AsyncMock(return_value=env)) + deployed = DeployedGatewayEnv(env_id="mcp-email", env_version=1, gateway_url="http://g", sandbox_id="sb-box", + sandbox_type=sandbox_type) + + await close_deployed(deployed, MCPServerEnv) + + box.terminate.aio.assert_not_called() + + +@pytest.mark.parametrize("sandbox_type, default", [ + ("modal_vm", None), ("modal", None), (None, ModalVmSandboxProvider), (None, ModalSandboxProvider)], + ids=["by id on modal_vm", "by id on modal", "reattached on modal_vm", "reattached on modal"]) +@pytest.mark.asyncio +async def test_a_teardown_reports_a_shared_box_not_terminated_and_leaves_it_running(box, monkeypatch, sandbox_type, + default): + if default is not None: + monkeypatch.setitem(teardown._DEFAULT_PROVIDERS, "env", default) + context = TaskStepContext() + sandbox = RecordedSandbox("sb-box", sandbox_type, "env") + + report = await teardown.tear_down(context, [sandbox], [], timeout=5) + + box.terminate.aio.assert_not_called() + assert report.terminated == () + assert [(failed, "shared box 'box-1'" in why) for failed, why in report.failed] == [(sandbox, True)] + assert teardown.TORN_DOWN_KEY not in context.metadata diff --git a/tst/unit/providers/sandbox_providers/modal_sandbox_test.py b/tst/unit/providers/sandbox_providers/modal_sandbox_test.py index 5c745ec58..d24c42068 100644 --- a/tst/unit/providers/sandbox_providers/modal_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/modal_sandbox_test.py @@ -463,6 +463,7 @@ async def collect(chunks, destination_path): async def test_terminate_sandbox_terminates_by_id_without_reading_its_tunnels(_patched_provider): sb = MagicMock() sb.tunnels.aio = AsyncMock(side_effect=modal.exception.ConflictError("Sandbox has already finished with status terminated")) + sb.get_tags.aio = AsyncMock(return_value={}) sb.terminate.aio = AsyncMock() with patch("agent_env.providers.sandbox_providers.modal_sandbox.modal.Sandbox.from_id") as from_id: from_id.aio = AsyncMock(return_value=sb) diff --git a/tst/unit/providers/sandbox_providers/modal_shared_box_test.py b/tst/unit/providers/sandbox_providers/modal_shared_box_test.py new file mode 100644 index 000000000..24e1f8654 --- /dev/null +++ b/tst/unit/providers/sandbox_providers/modal_shared_box_test.py @@ -0,0 +1,269 @@ +"""A shared box, one Modal VM holding several tasks' slots, is ended only by ``terminate_shared_box``: every other Modal +terminate reads whether its sandbox is one, unless this process made it, and refuses to end one.""" + +from __future__ import annotations + +import ast +import logging +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import modal +import pytest + +from agent_env.attribution import PIPELINE_STEP_KEY +from agent_env.config import get_config, reset_config +from agent_env.providers.sandbox_providers import modal_sandbox +from agent_env.providers.sandbox_providers.modal_sandbox import ( + SHARED_BOX_TAG, + SHARED_BOX_TAG_READ_FAILED_EVENT, + ModalSandboxProvider, + SharedBoxTerminateRefused, +) +from agent_env.providers.sandbox_providers.modal_vm_sandbox import ModalVmSandboxProvider +from tst.unit.store.fakes import FakeImageStore + +BOX = {SHARED_BOX_TAG: "box-1"} + + +@pytest.fixture(autouse=True) +def _offline(monkeypatch): + get_config().set_image_store(FakeImageStore()) + monkeypatch.setattr(modal_sandbox, "_TAG_READ_BACKOFF_SECONDS", 0) + yield + reset_config() + + +def modal_sb(tags=None, *, tag_reads=None, sandbox_id="sb-1") -> MagicMock: + """A Modal sandbox whose tags are ``tags``, or whose reads go as ``tag_reads``, recording its terminate.""" + sb = MagicMock(object_id=sandbox_id) + sb.get_tags.aio = AsyncMock(side_effect=tag_reads) if tag_reads else AsyncMock(return_value=dict(tags or {})) + sb.terminate.aio = AsyncMock() + sb.tunnels.aio = AsyncMock(return_value={}) + sb.wait_until_ready.aio = AsyncMock() + sb._get_task_id.aio = AsyncMock(return_value="ta-1") + sb.exec.aio = AsyncMock(side_effect=RuntimeError("no exec here")) + return sb + + +def provider(cls): + p = cls(app_name="agent-env-test") + p._get_app = AsyncMock(return_value=MagicMock()) + p._get_client = AsyncMock(return_value=MagicMock()) + return p + + +@pytest.fixture +def modal_api(monkeypatch): + """Patch Modal's create and from_id to hand out the sandbox a test sets as ``modal_api.sb``.""" + api = MagicMock() + api.create = AsyncMock(side_effect=lambda *a, **kw: api.sb) + api.from_id = AsyncMock(side_effect=lambda *a, **kw: api.sb) + monkeypatch.setattr(modal.Sandbox, "create", MagicMock(aio=api.create)) + monkeypatch.setattr(modal.Sandbox, "from_id", MagicMock(aio=api.from_id)) + return api + + +async def created_vm(p): + return await p.create_vm(setup_for_gateway=False) + + +async def created_container(p): + return await p.create_container(image_name="img:latest", port=8000, env={}, expose_externally=False) + + +PROVIDERS = pytest.mark.parametrize("cls", [ModalVmSandboxProvider, ModalSandboxProvider], ids=["vm", "container"]) + + +@pytest.mark.parametrize("create", [(ModalVmSandboxProvider, created_vm), (ModalSandboxProvider, created_container)], + ids=["vm", "container"]) +@pytest.mark.asyncio +async def test_a_handle_this_process_created_terminates_with_no_tag_read(modal_api, create): + cls, make = create + modal_api.sb = modal_sb() + handle = await make(provider(cls)) + + await handle.terminate() + + modal_api.sb.terminate.aio.assert_awaited_once() + modal_api.sb.get_tags.aio.assert_not_called() + + +@PROVIDERS +@pytest.mark.asyncio +async def test_terminate_sandbox_reads_the_tags_of_a_sandbox_that_is_no_box_then_terminates_it(modal_api, cls): + modal_api.sb = modal_sb({PIPELINE_STEP_KEY: "t_s"}) + + await provider(cls).terminate_sandbox("sb-1") + + modal_api.sb.get_tags.aio.assert_awaited_once() + modal_api.sb.terminate.aio.assert_awaited_once() + + +@pytest.mark.parametrize("gone_at", ["from_id", "get_tags"]) +@PROVIDERS +@pytest.mark.asyncio +async def test_terminate_sandbox_counts_a_sandbox_modal_no_longer_knows_as_gone(modal_api, cls, gone_at): + modal_api.sb = modal_sb(tag_reads=[modal.exception.NotFoundError("Sandbox not found")]) + if gone_at == "from_id": + modal_api.from_id.side_effect = modal.exception.NotFoundError("Sandbox not found") + + await provider(cls).terminate_sandbox("sb-1") + + modal_api.sb.terminate.aio.assert_not_called() + + +@PROVIDERS +@pytest.mark.asyncio +async def test_a_tag_read_that_fails_then_succeeds_terminates(modal_api, cls): + modal_api.sb = modal_sb(tag_reads=[modal.exception.ConnectionError("reset"), {}]) + + await provider(cls).terminate_sandbox("sb-1") + + modal_api.sb.terminate.aio.assert_awaited_once() + + +@PROVIDERS +@pytest.mark.asyncio +async def test_a_tag_read_that_keeps_failing_raises_terminates_nothing_and_says_so(modal_api, cls, caplog): + modal_api.sb = modal_sb(tag_reads=modal.exception.ConnectionError("reset")) + + with caplog.at_level(logging.WARNING, logger="agent_env"), pytest.raises(modal.exception.ConnectionError): + await provider(cls).terminate_sandbox("sb-1") + + modal_api.sb.terminate.aio.assert_not_called() + [warning] = [r for r in caplog.records if getattr(r, "event", None) == SHARED_BOX_TAG_READ_FAILED_EVENT] + assert warning.modal_sandbox_id == "sb-1" + + +@pytest.mark.asyncio +async def test_a_created_shared_box_handle_refuses_to_terminate(modal_api): + modal_api.sb = modal_sb(BOX) + handle = await provider(ModalVmSandboxProvider).create_shared_box("box-1", setup_for_gateway=False) + + with pytest.raises(SharedBoxTerminateRefused) as refused: + await handle.terminate() + + assert (refused.value.sandbox_id, refused.value.box_id) == ("sb-1", "box-1") + modal_api.sb.terminate.aio.assert_not_called() + + +@PROVIDERS +@pytest.mark.asyncio +async def test_a_reattached_handle_to_a_shared_box_refuses_to_terminate(modal_api, cls): + modal_api.sb = modal_sb(BOX) + handle = await provider(cls).get_sandbox("sb-1") + + for _ in range(2): + with pytest.raises(SharedBoxTerminateRefused): + await handle.terminate() + + assert handle.shared_box_id == "box-1" + modal_api.sb.terminate.aio.assert_not_called() + + +@PROVIDERS +@pytest.mark.asyncio +async def test_terminate_sandbox_refuses_a_shared_box(modal_api, cls): + modal_api.sb = modal_sb(BOX) + + with pytest.raises(SharedBoxTerminateRefused): + await provider(cls).terminate_sandbox("sb-1") + + modal_api.sb.terminate.aio.assert_not_called() + + +@pytest.mark.asyncio +async def test_terminate_shared_box_ends_the_box_it_names(modal_api): + modal_api.sb = modal_sb(BOX) + + await provider(ModalVmSandboxProvider).terminate_shared_box("sb-1", "box-1") + + modal_api.sb.terminate.aio.assert_awaited_once() + + +@pytest.mark.parametrize("sb", [ + pytest.param(lambda: modal_sb({SHARED_BOX_TAG: "box-2"}), id="another box"), + pytest.param(lambda: modal_sb({PIPELINE_STEP_KEY: "t_s"}), id="no box"), +]) +@pytest.mark.asyncio +async def test_terminate_shared_box_refuses_a_sandbox_that_isnt_that_box(modal_api, sb): + modal_api.sb = sb() + + with pytest.raises(SharedBoxTerminateRefused): + await provider(ModalVmSandboxProvider).terminate_shared_box("sb-1", "box-1") + + modal_api.sb.terminate.aio.assert_not_called() + + +@pytest.mark.asyncio +async def test_terminate_shared_box_raises_when_the_tags_cant_be_read(modal_api): + modal_api.sb = modal_sb(tag_reads=modal.exception.ConnectionError("reset")) + + with pytest.raises(modal.exception.ConnectionError): + await provider(ModalVmSandboxProvider).terminate_shared_box("sb-1", "box-1") + + modal_api.sb.terminate.aio.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_shared_box_tags_the_vm_with_its_box_beside_its_attribution(modal_api): + modal_api.sb = modal_sb() + + handle = await provider(ModalVmSandboxProvider).create_shared_box( + "box-1", setup_for_gateway=False, attribution={PIPELINE_STEP_KEY: "t_s"}) + + assert modal_api.create.call_args.kwargs["tags"] == {PIPELINE_STEP_KEY: "t_s", SHARED_BOX_TAG: "box-1"} + assert handle.shared_box_id == "box-1" + + +@pytest.mark.parametrize("box_id, attribution", [ + pytest.param("box-1", {SHARED_BOX_TAG: "forged"}, id="attribution names the box tag"), + pytest.param("", {}, id="empty id"), + pytest.param("box 1", {}, id="id outside a tag's characters"), + pytest.param("b" * 64, {}, id="id longer than a tag"), +]) +@pytest.mark.asyncio +async def test_create_shared_box_refuses_before_creating(modal_api, box_id, attribution): + with pytest.raises(ValueError): + await provider(ModalVmSandboxProvider).create_shared_box(box_id, attribution=attribution) + + modal_api.create.assert_not_called() + + +@pytest.mark.asyncio +async def test_an_ordinary_vm_cant_be_given_the_box_tag_through_its_attribution(modal_api): + with pytest.raises(ValueError): + await provider(ModalVmSandboxProvider).create_vm(attribution={SHARED_BOX_TAG: "forged"}) + + modal_api.create.assert_not_called() + + +@pytest.mark.asyncio +async def test_providers_other_than_modal_vm_make_no_shared_box(): + with pytest.raises(NotImplementedError): + await ModalSandboxProvider().create_shared_box("box-1") + + +_ALLOWED = {"_terminate", "_terminate_created"} +_CREATE_CLEANUPS = {"create_container", "_created"} + + +def test_every_modal_terminate_goes_through_the_check_point_but_a_failed_creates_cleanup(): + """``terminate.aio(`` is called only in ``_terminate``, or where a create that failed ends its sandbox: the cancelled + create's cleanup, or an ``except`` in a create.""" + stray = [] + sources = Path(modal_sandbox.__file__).parent.glob("modal_*.py") + for source in sources: + tree = ast.parse(source.read_text()) + for function in (n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))): + in_handler = {id(c) for h in ast.walk(function) if isinstance(h, ast.ExceptHandler) for c in ast.walk(h)} + for call in (n for n in ast.walk(function) if isinstance(n, ast.Call)): + func = call.func + if not (isinstance(func, ast.Attribute) and func.attr == "aio" + and isinstance(func.value, ast.Attribute) and func.value.attr == "terminate"): + continue + if function.name in _ALLOWED or (function.name in _CREATE_CLEANUPS and id(call) in in_handler): + continue + stray.append(f"{source.name}:{call.lineno} in {function.name}") + assert stray == [] diff --git a/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py b/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py index 036351aa3..dafd91115 100644 --- a/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py @@ -404,6 +404,7 @@ async def test_terminate_sandbox_terminates_a_vm_by_id_and_counts_one_modal_no_l provider = ModalVmSandboxProvider() provider._get_client = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign] sb = MagicMock() + sb.get_tags.aio = AsyncMock(return_value={}) sb.terminate.aio = AsyncMock() with patch("agent_env.providers.sandbox_providers.modal_sandbox.modal.Sandbox.from_id") as from_id: from_id.aio = (AsyncMock(return_value=sb) if found diff --git a/tst/unit/task/teardown_test.py b/tst/unit/task/teardown_test.py index 6cc97c1a9..76aba5b56 100644 --- a/tst/unit/task/teardown_test.py +++ b/tst/unit/task/teardown_test.py @@ -312,6 +312,7 @@ async def terminate_sandbox(self, sandbox_id): def test_a_modal_sandbox_that_already_finished_is_counted_down_not_reported_up(monkeypatch): sb = MagicMock() sb.tunnels.aio = AsyncMock(side_effect=modal.exception.ConflictError("Sandbox has already finished with status terminated")) + sb.get_tags.aio = AsyncMock(return_value={}) sb.terminate.aio = AsyncMock() monkeypatch.setattr(modal.Sandbox, "from_id", MagicMock(aio=AsyncMock(return_value=sb))) monkeypatch.setattr(ModalSandboxProvider, "_get_client", AsyncMock(return_value=MagicMock())) From 35fbb0ce5ad1678e566fc828fd8bd5dee3f7207b Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Thu, 19 Nov 2026 18:47:32 -0500 Subject: [PATCH 3/3] fix(sandbox-providers): never reboot a shared box, and check every Modal SDK module for stray terminates reset_vm refuses a handle known to be a shared box, whose reboot would end every slot on it. The source check now covers every agent_env module that uses the Modal SDK and fails on a direct SandboxTerminate RPC anywhere. Co-Authored-By: Claude Opus 5.5 --- .../sandbox_providers/modal_vm_sandbox.py | 6 ++++- .../modal_shared_box_test.py | 22 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py b/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py index ec6df5558..8c88f1948 100644 --- a/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py +++ b/src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py @@ -38,6 +38,7 @@ SHARED_BOX_TAG, UNREAD, SharedBox, + SharedBoxTerminateRefused, _ModalProcessAdapter, _attribution_tags, _ended_output, @@ -433,7 +434,10 @@ async def terminate_shared_box(self, sandbox_id: str, box_id: str) -> None: await _terminate_by_id(await self._get_client(), sandbox_id, ending_box=box_id) async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: - """The launcher in the sandbox booting its disk afresh with ``env``, at the same URLs.""" + """The launcher in the sandbox booting its disk afresh with ``env``, at the same URLs. Refused for a shared box, + whose reboot would end every slot on it.""" + if isinstance(getattr(sandbox, "shared_box_id", None), str): + raise SharedBoxTerminateRefused(sandbox.sandbox_id, sandbox.shared_box_id) code, _, stderr = await sandbox._exec_with_stdin(shlex.join(RESET), _chunks(reset_input(env))) if code: raise RuntimeError(f"the launcher in Modal sandbox {sandbox.sandbox_id} didn't boot its disk afresh " diff --git a/tst/unit/providers/sandbox_providers/modal_shared_box_test.py b/tst/unit/providers/sandbox_providers/modal_shared_box_test.py index 24e1f8654..86864590f 100644 --- a/tst/unit/providers/sandbox_providers/modal_shared_box_test.py +++ b/tst/unit/providers/sandbox_providers/modal_shared_box_test.py @@ -4,6 +4,7 @@ from __future__ import annotations import ast +import re import logging from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -251,9 +252,14 @@ async def test_providers_other_than_modal_vm_make_no_shared_box(): def test_every_modal_terminate_goes_through_the_check_point_but_a_failed_creates_cleanup(): """``terminate.aio(`` is called only in ``_terminate``, or where a create that failed ends its sandbox: the cancelled - create's cleanup, or an ``except`` in a create.""" + create's cleanup, or an ``except`` in a create. Every module of agent_env that uses the Modal SDK is checked, and no + module calls Modal's terminate RPC directly.""" stray = [] - sources = Path(modal_sandbox.__file__).parent.glob("modal_*.py") + package = Path(modal_sandbox.__file__).parents[2] + sources = [p for p in package.rglob("*.py") if re.search(r"^\s*(import modal|from modal)", p.read_text(), re.M)] + assert Path(modal_sandbox.__file__) in sources + for source in package.rglob("*.py"): + assert "SandboxTerminate" not in source.read_text(), f"{source} calls Modal's terminate RPC directly" for source in sources: tree = ast.parse(source.read_text()) for function in (n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))): @@ -267,3 +273,15 @@ def test_every_modal_terminate_goes_through_the_check_point_but_a_failed_creates continue stray.append(f"{source.name}:{call.lineno} in {function.name}") assert stray == [] + + +@pytest.mark.asyncio +async def test_a_shared_box_is_never_rebooted_under_its_slots(modal_api): + modal_api.sb = modal_sb() + vm_provider = provider(ModalVmSandboxProvider) + box = await vm_provider.create_shared_box("box-1", setup_for_gateway=False) + box._exec_with_stdin = AsyncMock() + + with pytest.raises(SharedBoxTerminateRefused): + await vm_provider.reset_vm(box, image="disk", env={}) + box._exec_with_stdin.assert_not_called()