Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/agent_env/providers/env_providers/env_gateway_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,23 @@ def uses_database(self) -> bool:
return _ENV_NETWORK in self.networks


TEARDOWN_RETRIES = 2


async def tear_down_slot(sandbox: Sandbox, layout: StackLayout) -> None:
"""End one slot's stack on a VM other slots share, leaving the VM and every other slot running. Idempotent, so a
retry after a dropped connection is safe. Raises for the default stack, or for a sandbox that is not a VM, since
only a VM holds slots; raises too if the script fails, so the caller never counts a slot gone that isn't.

The caller must first cancel or await the slot's own deploy: a ``compose up`` still running can create containers
after the script has listed them, and the slot would be counted gone while it still holds its ports."""
if not layout.is_slot:
raise ValueError("tear_down_slot ends one slot of a shared VM; the default stack ends with its VM")
if sandbox.mode != SANDBOX_MODE_VM:
raise ValueError(f"slot {layout.compose_project} can only be on a VM, not a {sandbox.mode} sandbox")
await sandbox.exec_script(layout.teardown_script, max_retries=TEARDOWN_RETRIES)


@dataclass
class DeployedGateway:
"""Result of a gateway deployment."""
Expand Down
27 changes: 27 additions & 0 deletions src/agent_env/providers/env_providers/stack_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import posixpath
import re
import shlex
from dataclasses import dataclass

from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR
Expand Down Expand Up @@ -133,6 +134,32 @@ def from_record(cls, data: object) -> StackLayout:
raise ValueError(f"a recorded stack layout must be a slot's, got {data!r}")
return layout

@property
def teardown_script(self) -> str:
"""A root shell script that removes this slot's stack and nothing else on the VM: every container, volume and
network its compose project labels, then its dir and scratch files. Images stay, since every stack on the VM
shares them. Idempotent, and it needs no compose file, so it also clears a slot whose deploy failed before
writing one. Raises for the default stack, which has the VM to itself and goes with it, and for any layout
not made by ``for_slot``: a hand-built project such as ``s1-x`` would have its files caught by ``s1``'s glob."""
if not self.is_slot:
raise ValueError("only a slot's stack is torn down on its own; the default stack ends with its VM")
index = self.compose_project[1:]
if not (self.compose_project.startswith("s") and index.isascii() and index.isdigit()
and self == StackLayout.for_slot(int(index))):
raise ValueError(f"only a layout made by StackLayout.for_slot is torn down, so its dir and scratch glob "
f"can't reach another stack's files; got {self}")
label = shlex.quote(f"label=com.docker.compose.project={self.compose_project}")
return "\n".join([
"set -eu",
f"containers=$(docker ps -aq --filter {label})",
'[ -z "$containers" ] || docker rm -f -v $containers >/dev/null',
f"volumes=$(docker volume ls -q --filter {label})",
'[ -z "$volumes" ] || docker volume rm -f $volumes >/dev/null',
f"networks=$(docker network ls -q --filter {label})",
'[ -z "$networks" ] || docker network rm $networks >/dev/null',
f"rm -rf {shlex.quote(self.app_dir)} {shlex.quote(self.scratch_prefix)}*",
])

def scratch_path(self, name: str) -> str:
return f"{self.scratch_prefix}{name}"

Expand Down
1 change: 1 addition & 0 deletions src/agent_env/providers/sandbox_providers/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ with `exposed_ports=[port]` and returns the VM, and the deploy then loads and st
| `create_sandbox(*, image_name, port, env, cpu, memory, disk_size_gb, timeout, attribution, network_policy) -> Sandbox` | yes (a VM backend wraps `create_vm`) | an agent's sandbox (`provider.create_sandbox`); `create_container` of a container-only backend |
| `get_sandbox(sandbox_id) -> Sandbox` | yes | every step after the one that created the sandbox, and teardown |
| `terminate_sandbox(sandbox_id)` | no | teardown. The default calls `get_sandbox` then `terminate()`, and treats `SandboxNotFoundError` as done; override when you can terminate without a live handle |
| `create_shared_box(box_id, *, cpu, memory, timeout, exposed_ports, setup_for_gateway, attribution, network_policy) -> VmSandbox` | no | nothing yet: a VM for several tasks' slots. The default raises `NotImplementedError`; `modal_vm` tags the VM with `SHARED_BOX_TAG` at create, refuses to end it from its handle's `terminate()` or `terminate_sandbox`, raising `SharedBoxTerminateRefused`, and ends it only through `terminate_shared_box(sandbox_id, box_id)` |
| `create_container(...)` | inherited | containers that run an env or agent image. Override only when the platform boots images natively. Its `shm_mb` is the MiB the container's `/dev/shm` holds (`--shm-size` on the default VM path), passed only where `SHARED_MEMORY` is on |
| `reset_container(sandbox, *, image_name, port, env, shm_mb)` | inherited | a computer env's reset, which replaces the container `create_container` ran with a fresh one on the same sandbox and port, its `/dev/shm` as before. The default works on a VM backend; one whose sandbox is the container itself raises `NotImplementedError` |
| `native_platforms()` | no | a computer env's deploy and gate 3, which boot a build only on a provider that runs its arch without emulation. The default is `linux/amd64`; local Docker returns this machine's |
Expand Down
86 changes: 82 additions & 4 deletions src/agent_env/providers/sandbox_providers/modal_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import base64
import collections
import contextlib
import enum
import functools
import hashlib
import logging
Expand All @@ -30,6 +31,7 @@

if TYPE_CHECKING:
from agent_env.artifact.artifacts.docker_image import DockerImageArtifact
from agent_env.providers.sandbox_providers.modal_vm_sandbox import ModalVmSandbox

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -95,14 +97,86 @@ def _tunnel_url(t) -> str:
return f"https://{t.host}" + (f":{t.port}" if t.port != 443 else "")


async def _terminate_by_id(client, sandbox_id: str) -> None:
#: The Modal sandbox tag that marks a VM as a shared box, several tasks' slots on one VM; its value is the box's id.
SHARED_BOX_TAG = "agent_env_shared_box"
SHARED_BOX_TAG_READ_FAILED_EVENT = "agent_env.modal_shared_box_tag_read_failed"
_TAG_READ_ATTEMPTS, _TAG_READ_BACKOFF_SECONDS = 3, 1.0


class _Unread(enum.Enum):
UNREAD = "unread"


#: A handle's ``shared_box_id`` before its tags are read: one this process reattached to by id.
UNREAD = _Unread.UNREAD
SharedBox = Optional[str] | _Unread


class SharedBoxTerminateRefused(Exception):
"""A terminate that would have ended a shared box, or a ``terminate_shared_box`` naming a box the sandbox isn't;
the sandbox was left running."""

def __init__(self, sandbox_id: str, box_id: Optional[str], ending_box: Optional[str] = None):
self.sandbox_id, self.box_id = sandbox_id, box_id
if ending_box is None:
why = f"is shared box {box_id!r}, which only terminate_shared_box ends"
else:
why = f"is {f'shared box {box_id!r}' if box_id else 'no shared box'}, not shared box {ending_box!r}"
super().__init__(f"Modal sandbox {sandbox_id} {why}; it was not terminated")


async def _read_shared_box(sb: modal.Sandbox) -> Optional[str]:
"""The shared box ``sb`` is tagged as, if any. A read that keeps failing raises, so a terminate fails closed;
NotFoundError raises at once."""
attempt = 1
while True:
try:
return (await sb.get_tags.aio()).get(SHARED_BOX_TAG)
except modal.exception.NotFoundError:
raise
except Exception as e:
if attempt == _TAG_READ_ATTEMPTS:
logger.warning(f"Modal sandbox {sb.object_id} was not terminated: its tags, which say whether it's a "
f"shared box, couldn't be read: {fmt_exc(e)}",
extra={"event": SHARED_BOX_TAG_READ_FAILED_EVENT, "modal_sandbox_id": sb.object_id})
raise
await asyncio.sleep(_TAG_READ_BACKOFF_SECONDS * 2 ** (attempt - 1))
attempt += 1


async def _terminate(sb: modal.Sandbox, box: SharedBox, *, ending_box: Optional[str] = None) -> None:
"""Terminate ``sb`` when it is the shared box ``ending_box``, or, with none, when it is no shared box; else raise
SharedBoxTerminateRefused. Every Modal terminate goes through here but the cleanup of a create that failed, whose
sandbox holds no slots yet. ``box`` is what the caller knows of ``sb``; UNREAD reads its tags, and one gone already
counts as terminated."""
if box is UNREAD:
try:
box = await _read_shared_box(sb)
except modal.exception.NotFoundError:
logger.info("Modal sandbox %s was already gone at terminate", sb.object_id)
return
if box != ending_box:
raise SharedBoxTerminateRefused(sb.object_id, box, ending_box)
await sb.terminate.aio()


async def _terminate_by_id(client, sandbox_id: str, *, ending_box: Optional[str] = None) -> None:
try:
sb = await modal.Sandbox.from_id.aio(sandbox_id, client=client)
await sb.terminate.aio()
await _terminate(sb, UNREAD, ending_box=ending_box)
except modal.exception.NotFoundError:
logger.info("Modal sandbox %s was already gone at terminate", sandbox_id)


async def _terminate_handle(handle: ModalSandbox | ModalVmSandbox) -> None:
"""Terminate a handle's sandbox through ``_terminate``, keeping the shared box a refusal found on the handle."""
try:
await _terminate(handle._sb, handle.shared_box_id)
except SharedBoxTerminateRefused as refused:
handle.shared_box_id = refused.box_id
raise


def _modal_network_kwargs(policy: NetworkPolicy) -> dict:
"""``policy`` as Modal sandbox creation kwargs.

Expand Down Expand Up @@ -194,8 +268,9 @@ class ModalSandbox(Sandbox):
type = "modal"

def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str], i6pn_address: Optional[str] = None,
network_policy: NetworkPolicy | None = None):
network_policy: NetworkPolicy | None = None, shared_box_id: SharedBox = UNREAD):
self._sb = sb
self.shared_box_id = shared_box_id
self.sandbox_id = sb.object_id
self.tunnel_urls = tunnel_urls
self.i6pn_address = i6pn_address
Expand All @@ -210,7 +285,7 @@ def private_host(self) -> str | None:
return f"[{self.i6pn_address}]" if self.i6pn_address else None

async def terminate(self) -> None:
await self._sb.terminate.aio()
await _terminate_handle(self)

async def exit_status(self) -> Optional[int]:
return await self._sb.poll.aio()
Expand Down Expand Up @@ -511,6 +586,7 @@ async def create_rebuilding_once() -> modal.Sandbox:
tunnel_urls,
i6pn_address=i6pn_address,
network_policy=effective,
shared_box_id=None,
)
if vnc_port in tunnel_urls:
sandbox.vnc_url = f"{tunnel_urls[vnc_port]}/vnc.html"
Expand Down Expand Up @@ -589,6 +665,8 @@ def _attribution_tags(attribution: Attribution) -> dict[str, str]:
continue
if not _MODAL_TAG_KEY.fullmatch(key):
raise ValueError(f"attribution key {key!r} can't be a Modal tag: use 1-63 of a-z A-Z 0-9 . _ -")
if key == SHARED_BOX_TAG:
raise ValueError(f"attribution key {key!r} is the tag that marks a shared box; only create_shared_box sets it")
tags[key] = _modal_tag_value(value)
return tags

Expand Down
54 changes: 47 additions & 7 deletions src/agent_env/providers/sandbox_providers/modal_vm_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,19 @@
import modal

from agent_env.providers.sandbox_providers.modal_sandbox import (
_MODAL_TAG_KEY,
SHARED_BOX_TAG,
UNREAD,
SharedBox,
SharedBoxTerminateRefused,
_ModalProcessAdapter,
_attribution_tags,
_ended_output,
_last_lines,
_log_sandbox_started,
_resolve_app_base_name,
_terminate_by_id,
_terminate_handle,
_tunnel_url,
_modal_network_kwargs,
)
Expand Down Expand Up @@ -144,8 +150,9 @@ class ModalVmSandbox(VmSandbox):
_WFT_CHUNK_BYTES = 32 * 1024

def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str],
network_policy: NetworkPolicy | None = None):
network_policy: NetworkPolicy | None = None, shared_box_id: SharedBox = UNREAD):
self._sb = sb
self.shared_box_id = shared_box_id
self.sandbox_id = sb.object_id
self.tunnel_urls = tunnel_urls
self.vnc_url = None
Expand All @@ -155,7 +162,7 @@ def __init__(self, sb: modal.Sandbox, tunnel_urls: dict[int, str],
self._ended: Optional[tuple[list[str], list[str]]] = None

async def terminate(self) -> None:
await self._sb.terminate.aio()
await _terminate_handle(self)

async def exit_status(self) -> Optional[int]:
return await self._sb.poll.aio()
Expand Down Expand Up @@ -399,22 +406,55 @@ async def create_vm(
experimental_options={"enable_nested_virt": True}, secrets=[modal.Secret.from_dict(variables)],
idle_timeout=self._idle_timeout or _DISK_IDLE_TIMEOUT_S)

async def create_shared_box(
self,
box_id: str,
*,
cpu: float = 0.5,
memory: int = 8192,
timeout: int = 3600 * 2,
exposed_ports: Optional[list[int]] = None,
setup_for_gateway: bool = True,
attribution: Optional[Attribution] = None,
network_policy: Optional[NetworkPolicy] = None,
) -> ModalVmSandbox:
"""A VM running Docker, as ``create_vm`` makes, tagged as shared box ``box_id`` at create, since a later
``set_tags`` would replace its attribution tags. Its handle's terminate, and ``terminate_sandbox``, refuse to end
it: only ``terminate_shared_box`` does."""
if not _MODAL_TAG_KEY.fullmatch(box_id):
raise ValueError(f"shared box id {box_id!r} can't be a Modal tag: use 1-63 of a-z A-Z 0-9 . _ -")
return await self._created(_KEEPALIVE_CMD, _VM_IMAGE, cpu=cpu, memory=memory, timeout=timeout,
ports=list(exposed_ports or []), attribution=attribution,
network_policy=network_policy, setup_for_gateway=setup_for_gateway,
idle_timeout=self._idle_timeout, runtime="vm", shared_box_id=box_id)

async def terminate_shared_box(self, sandbox_id: str, box_id: str) -> None:
"""End shared box ``box_id``, the sandbox ``sandbox_id``, and every slot on it. A sandbox tagged as another box,
or as none, raises SharedBoxTerminateRefused, and one whose tags can't be read raises; neither is terminated."""
await _terminate_by_id(await self._get_client(), sandbox_id, ending_box=box_id)

async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None:
"""The launcher in the sandbox booting its disk afresh with ``env``, at the same URLs."""
"""The launcher in the sandbox booting its disk afresh with ``env``, at the same URLs. Refused for a shared box,
whose reboot would end every slot on it."""
if isinstance(getattr(sandbox, "shared_box_id", None), str):
raise SharedBoxTerminateRefused(sandbox.sandbox_id, sandbox.shared_box_id)
code, _, stderr = await sandbox._exec_with_stdin(shlex.join(RESET), _chunks(reset_input(env)))
if code:
raise RuntimeError(f"the launcher in Modal sandbox {sandbox.sandbox_id} didn't boot its disk afresh "
f"(exit {code}): {stderr.strip()[-500:]}")

async def _created(self, command: tuple[str, ...], image: modal.Image, *, cpu: float, memory: int, timeout: int,
ports: list[int], attribution: Optional[Attribution], network_policy: Optional[NetworkPolicy],
setup_for_gateway: bool, idle_timeout: Optional[int], **create) -> ModalVmSandbox:
setup_for_gateway: bool, idle_timeout: Optional[int], shared_box_id: Optional[str] = None,
**create) -> ModalVmSandbox:
"""A sandbox of ``image`` running ``command``, in the configured region, its ports' tunnels up, ended once idle
``idle_timeout`` seconds when given, set up for a gateway when ``setup_for_gateway``; one that fails before
it's returned is terminated."""
``idle_timeout`` seconds when given, set up for a gateway when ``setup_for_gateway``, tagged as shared box
``shared_box_id`` when given; one that fails before it's returned is terminated."""
attribution = dict(attribution or {})
app_tags = _attribution_tags({})
sandbox_tags = _attribution_tags(attribution)
if shared_box_id is not None:
sandbox_tags[SHARED_BOX_TAG] = shared_box_id
app_name = self._app_name
app = await self._get_app(app_name, app_tags)
client = await self._get_client()
Expand Down Expand Up @@ -449,7 +489,7 @@ async def _created(self, command: tuple[str, ...], image: modal.Image, *, cpu: f
for p in ports:
if p not in tunnel_urls:
raise RuntimeError(f"No Modal tunnel for exposed port {p}; got {sorted(tunnel_urls)}")
sandbox = ModalVmSandbox(sb, tunnel_urls, network_policy=effective)
sandbox = ModalVmSandbox(sb, tunnel_urls, network_policy=effective, shared_box_id=shared_box_id)
logger.info(f"Modal VM sandbox created: {sb.object_id} (tunnels={tunnel_urls})")
await _log_sandbox_started(
sb, app_name=app_name, sandbox_tags=sandbox_tags,
Expand Down
15 changes: 15 additions & 0 deletions src/agent_env/providers/sandbox_providers/sandbox_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ async def create_vm(
channel."""
raise NotImplementedError(f"{type(self).__name__} does not support create_vm")

async def create_shared_box(
self,
box_id: str,
*,
cpu: float = 1.0,
memory: int = 8192,
timeout: int = 3600 * 2,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Shared timeout repeats unnamed values. The new create_shared_box methods both use 3600 * 2 for their default timeout. The repository requires magic numbers to be stored in descriptively named class or instance variables. Define a shared named default and use it here and in ModalVmSandboxProvider.create_shared_box so the two defaults stay together.

Rule Used: Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. (source)

Learned From
scaleapi/scaleapi#126388

Prompt To Fix With AI
This is a comment left during a code review.
Path: src/agent_env/providers/sandbox_providers/sandbox_provider.py
Line: 106

Comment:
**Shared timeout repeats unnamed values.** The new `create_shared_box` methods both use `3600 * 2` for their default timeout. The repository requires magic numbers to be stored in descriptively named class or instance variables. Define a shared named default and use it here and in `ModalVmSandboxProvider.create_shared_box` so the two defaults stay together.

**Rule Used:** Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. ([source](https://app.greptile.com/-/switch-tenant?tenant=scale-ai&next=%2Fcustom-context%3Fmemory%3D002e0051-41ad-46c1-9098-47433c580150))

**Learned From**
[scaleapi/scaleapi#126388](https://github.com/scaleapi/scaleapi/pull/126388)

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Cursor Fix in Claude Code Fix in Codex

exposed_ports: Optional[list[int]] = None,
setup_for_gateway: bool = True,
attribution: Optional[Attribution] = None,
network_policy: Optional[NetworkPolicy] = None,
) -> VmSandbox:
"""A VM for several tasks' slots, shared box ``box_id``, which no single task's terminate ends."""
raise NotImplementedError(f"{type(self).__name__} does not support shared boxes")

async def create_container(
self,
*,
Expand Down
Loading
Loading