From 479aad8df222765fd659fad9a9a026f1c6d67992 Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Tue, 17 Nov 2026 19:01:46 -0500 Subject: [PATCH 1/3] feat(env-providers): give each gateway stack its own layout on a VM Sandbox stuffing will run several agent-env stacks ("slots") in one Modal VM. Today one stack per VM is assumed throughout: one /app compose dir, compose's implicit project, fixed host ports, fixed /tmp scratch paths, and a gateway-running check that matches any container named gateway. StackLayout (providers/env_providers/stack_layout.py) holds where one stack lives: its app dir, compose project, host port offset and scratch prefix. Every one of those call sites now asks the layout. The default layout reproduces today's commands byte for byte, and nothing passes another layout yet, so behavior is unchanged. StackLayout.for_slot(index) gives slot `index` the project s, /app/s, host ports shifted by (index + 1) * 1000 (container ports unchanged), its own scratch names and its own snapshot image tag. A slot checks its gateway with a project-scoped `compose ps`, and a slot port without a tunnel raises instead of returning no URL. A slot's project can never equal the default stack's implicit `app`. App dir and scratch prefix are checked against a shell-safe charset, since they reach shell scripts unquoted. Persisting the layout for reattach, slot teardown, sidecar port checks and exposing every slot's ports at VM create are left for the next PR. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/_deployment.py | 7 + src/agent_env/env/envs/mcp_server.py | 5 +- src/agent_env/env/envs/multi_env.py | 30 +- src/agent_env/env/envs/website.py | 11 +- src/agent_env/env/snapshot_store.py | 48 ++- .../providers/env_providers/constants.py | 3 +- .../env_providers/env_gateway_provider.py | 79 +++-- .../providers/env_providers/env_provider.py | 7 + .../providers/env_providers/stack_layout.py | 124 +++++++ tst/integration/env/gateway/gateway_test.py | 5 +- .../test_multi_env_snapshot_reload_name.py | 4 +- tst/unit/env/envs/test_stack_layout_of.py | 71 ++++ .../env_providers/stack_layout_test.py | 322 ++++++++++++++++++ 13 files changed, 648 insertions(+), 68 deletions(-) create mode 100644 src/agent_env/providers/env_providers/stack_layout.py create mode 100644 tst/unit/env/envs/test_stack_layout_of.py create mode 100644 tst/unit/providers/env_providers/stack_layout_test.py diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 426dfa45c..1f5e1ae87 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -17,6 +17,7 @@ from agent_env.env.env import DeployedSandboxEnv, gateway_url_of from agent_env.env.gateway.constants import DATA_PLANE_LOAD_TIMEOUT_S from agent_env.env.store import register_env_instance +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.store.base import ObjectNotFoundError from agent_env.utils.paths import validate_file_name @@ -322,6 +323,12 @@ def as_builtin(provider: Optional[EnvironmentProvider]) -> Optional[_SandboxEnvi return provider if isinstance(provider, _SandboxEnvironmentProvider) else None +def stack_layout_of(provider: Optional[EnvironmentProvider]) -> StackLayout: + """Where a built-in provider's stack lives in its sandbox; the default for any other provider, or none.""" + builtin = as_builtin(provider) + return builtin.stack_layout if builtin is not None else DEFAULT_STACK_LAYOUT + + def provider_options(env: Env, provider: EnvironmentProvider, **options) -> dict: """The options provider takes: every one for a deploy(**options), else those it names, one it doesn't being dropped at deploy()'s default; raises for a deploy refused (see ``deploy_refusal``).""" diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 44f5c02dd..c26a15de0 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -22,7 +22,7 @@ from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, - reset_server_data, + reset_server_data, stack_layout_of, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -260,8 +260,7 @@ async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: # Stage on the disk-backed app dir, not /tmp (tmpfs/RAM): a large # artifact (github ~8GB) OOM-kills the copy under concurrent loads. # Random suffix avoids collisions between concurrent loads. - from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR - stage_dir = f"{GATEWAY_APP_DIR}/_artifact_staging" + stage_dir = stack_layout_of(self._env_provider).artifact_staging_dir vm_temp_path = f"{stage_dir}/{self.environment_name}-{uuid.uuid4().hex[:8]}-{posixpath.basename(path)}" await self._sandbox.exec_script(f"mkdir -p {shlex.quote(stage_dir)}") await self._sandbox.load_object_file(file_artifact.object_url, vm_temp_path) diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index 3239c4ed3..48840f37e 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -43,7 +43,7 @@ def _snapshot_after_load_default() -> bool: from agent_env.env.gateway import GatewayMode from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + plugin_provider_like_a_builtin, provider_or_class, reset_server_data, stack_layout_of, ) from agent_env.env import store as env_store from agent_env.env.envs import _multi_computers @@ -52,7 +52,7 @@ def _snapshot_after_load_default() -> bool: from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.website import WebsiteEnv from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.env_state.local_postgres import INIT_SCRIPT_MOUNT_PATH from agent_env.store.base import NotFoundError from agent_env.store.ids import derive_id @@ -117,14 +117,14 @@ def browser_problems(envs: list[Env]) -> list[str]: _SERVICE_DB_IMAGE = re.compile(rf"^( {re.escape(DATABASE_SERVICE_NAME)}:\n image: ).*$", re.MULTILINE) -def snapshot_compose(compose: str, db_image: str) -> str: - """``compose``, the file a deploy wrote, with its servicedb started from ``db_image`` and without the schema init - script, whose work a snapshot already holds. Every other service stays as the deploy wrote it.""" +def snapshot_compose(compose: str, db_image: str, *, compose_file: str = DEFAULT_STACK_LAYOUT.compose_file) -> str: + """``compose``, the file a deploy wrote at ``compose_file``, with its servicedb started from ``db_image`` and without + the schema init script, whose work a snapshot already holds. Every other service stays as the deploy wrote it.""" if problem := image_reference_problem(db_image): raise ValueError(problem) swapped, count = _SERVICE_DB_IMAGE.subn(lambda match: match.group(1) + db_image, compose) if count != 1: - raise RuntimeError(f"{DOCKER_COMPOSE_PATH} names a {DATABASE_SERVICE_NAME} image {count} times, not once, so " + raise RuntimeError(f"{compose_file} names a {DATABASE_SERVICE_NAME} image {count} times, not once, so " "the snapshot can't replace it") return swapped.replace(f" volumes:\n - ./init-schemas.sql:{INIT_SCRIPT_MOUNT_PATH}\n", "") @@ -711,7 +711,7 @@ async def _resumable_environments(self, environment_universe_artifact, total: in )) if not recorded: return set() - if not await _check_changelog_empty(self._sandbox): + if not await _check_changelog_empty(self._sandbox, layout=stack_layout_of(self._env_provider)): logger.info( "Not resuming: the changelog is non-empty, so this env's data has been " "modified since it was loaded and every service must be re-seeded" @@ -880,7 +880,6 @@ async def _load_from_snapshot(self, snapshot) -> None: from agent_env.env.envs.service_db import DB_USER from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT from agent_env.providers import EnvironmentGatewayProvider - from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR from agent_env.providers.env_state import LocalPostgresStateProvider from agent_env.config import get_config, service_db_env_id @@ -888,6 +887,7 @@ async def _load_from_snapshot(self, snapshot) -> None: raise RuntimeError("Environment not deployed - call deploy() first") if self._sandbox.mode == SANDBOX_MODE_CONTAINER: raise NotImplementedError("Snapshot loading not supported on container mode") + layout = stack_layout_of(self._env_provider) db_image = Artifact.get(snapshot.db_image_artifact_id, snapshot.db_image_artifact_version) if not isinstance(db_image, DockerImageArtifact): raise RuntimeError(f"Snapshot artifact {snapshot.db_image_artifact_id} is not a DockerImageArtifact") @@ -896,19 +896,21 @@ async def _load_from_snapshot(self, snapshot) -> None: logger.info("Pointing docker-compose.yml's servicedb at the snapshot image...") # The write adds back the newline cat returns, so the file doesn't grow a blank line each reload. - deployed = (await self._sandbox.exec_script(f"cat {DOCKER_COMPOSE_PATH}")).removesuffix("\n") - compose = snapshot_compose(deployed, db_image.image_name) + deployed = (await self._sandbox.exec_script(f"cat {layout.compose_file}")).removesuffix("\n") + compose = snapshot_compose(deployed, db_image.image_name, compose_file=layout.compose_file) gw = EnvironmentGatewayProvider() gw._sandbox = self._sandbox + gw._stack_layout = layout # The state store is the deploy's local Postgres; the changelog triggers below run through it. gw._state_provider = LocalPostgresStateProvider(service_db_config=Env.get(service_db_env_id(get_config())).to_config()) gw._state_instance = LocalPostgresStateProvider.default_instance() - await gw._write_file(self._sandbox, DOCKER_COMPOSE_PATH, compose) + await gw._write_file(self._sandbox, layout.compose_file, compose) # Remove old servicedb container and its anonymous volume, then start fresh from snapshot image logger.info("Recreating servicedb container with snapshot image...") await self._sandbox.exec_script( - f"cd {GATEWAY_APP_DIR} && docker compose rm -sf -v {DATABASE_SERVICE_NAME} && docker compose up -d {DATABASE_SERVICE_NAME} 2>&1" + f"cd {layout.app_dir} && {layout.compose_command_from_app_dir} rm -sf -v {DATABASE_SERVICE_NAME} && " + f"{layout.compose_command_from_app_dir} up -d {DATABASE_SERVICE_NAME} 2>&1" ) # Wait for servicedb to be healthy @@ -916,7 +918,7 @@ async def _load_from_snapshot(self, snapshot) -> None: for i in range(60): try: await self._sandbox.exec_script( - f"docker compose -f {DOCKER_COMPOSE_PATH} exec -T {DATABASE_SERVICE_NAME} pg_isready -U {DB_USER}", + f"{layout.compose_command} exec -T {DATABASE_SERVICE_NAME} pg_isready -U {DB_USER}", ) logger.info(f"servicedb healthy after {i + 1}s") break @@ -938,7 +940,7 @@ async def _load_from_snapshot(self, snapshot) -> None: if name != DATABASE_SERVICE_NAME))) logger.info(f"Recreating services: {svc_list}") await gw._compose_up(self._sandbox, f"--force-recreate {svc_list}") - await gw._wait_for_gateway(self._sandbox, self._sandbox.host_port(AGENT_ENV_GATEWAY_MCP_PORT)) + await gw._wait_for_gateway(self._sandbox, self._sandbox.host_port(layout.host_port(AGENT_ENV_GATEWAY_MCP_PORT))) # Install changelog triggers for each service for name in environment_names: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index b7a01c7d2..0875528a8 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -45,7 +45,7 @@ from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, - reset_server_data, + reset_server_data, stack_layout_of, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -340,14 +340,14 @@ async def load_file_artifact_universe(self, file_artifact_universe: "Any", desti return await super().load_file_artifact_universe(file_artifact_universe, destination_path) async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: - from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX, DOCKER_COMPOSE_PATH + from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX validate_file_path(path) backend_service = f"{self.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}" # Random suffix so concurrent loads on a shared sandbox (parallel /load-universe, # MCP+website sharing a service_name, repeated loads) don't race on a shared path. vm_path = f"/tmp/_artifact_{self.environment_name}-{uuid.uuid4().hex[:8]}-{posixpath.basename(path)}" container_path = f"/tmp/data/{path}" - compose = f"docker compose -f {shlex.quote(DOCKER_COMPOSE_PATH)}" + compose = stack_layout_of(self._env_provider).compose_command await self._sandbox.load_object_file(file_artifact.object_url, vm_path) await self._sandbox.exec_script( @@ -358,11 +358,12 @@ async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: async def _remove_from_container(self, path: str) -> None: """Remove what a load staged at ``/tmp/data/``, best-effort: a load that finished stands either way.""" - from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX, DOCKER_COMPOSE_PATH + from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX backend_service = f"{self.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}" container_path = f"/tmp/data/{path}" + compose = stack_layout_of(self._env_provider).compose_command try: - await self._sandbox.exec_script(f"docker compose -f {shlex.quote(DOCKER_COMPOSE_PATH)} exec -T " + await self._sandbox.exec_script(f"{compose} exec -T " f"{shlex.quote(backend_service)} rm -rf {shlex.quote(container_path)}") except Exception as e: # noqa: BLE001 -- cleanup, not the load logger.warning(f"[{self.environment_name}] could not remove {container_path} after a load (ignored): {e}") diff --git a/src/agent_env/env/snapshot_store.py b/src/agent_env/env/snapshot_store.py index 279abdd76..53fbf1c78 100644 --- a/src/agent_env/env/snapshot_store.py +++ b/src/agent_env/env/snapshot_store.py @@ -10,6 +10,7 @@ from datetime import datetime, timezone from typing import Callable, Optional +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER from agent_env.config import get_config @@ -87,6 +88,7 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N """ from agent_env.artifact import DockerImageArtifact from agent_env.env.env import DeployedSandboxEnv, Env + from agent_env.env.envs._deployment import stack_layout_of from agent_env.env.envs.multi_env import MultiEnv, _gateway_state_provider from agent_env.env.store import get_env_instance_store @@ -135,16 +137,17 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N ) log("check_changelog", "Checking changelog...", 20) - is_clean = await _check_changelog_empty(multi_env._sandbox) + layout = stack_layout_of(multi_env._env_provider) + is_clean = await _check_changelog_empty(multi_env._sandbox, layout=layout) logger.info(f"Changelog empty: {is_clean} ({'clean' if is_clean else 'dirty'} snapshot)") log("check_changelog", f"Changelog: {'clean' if is_clean else 'dirty'}", 30) log("snapshot_db", "Exporting servicedb data...", 40) - tar_gz_object_url = await _snapshot_servicedb(multi_env._sandbox, env_id, universe_id, log) + tar_gz_object_url = await _snapshot_servicedb(multi_env._sandbox, env_id, universe_id, log, layout=layout) log("store_artifact", "Storing Docker image artifact...", 70) logger.info("Storing snapshot as DockerImageArtifact...") - image_tag = _image_tag(env_id, universe_id) + image_tag = _snapshot_image_tag(env_id, universe_id, layout) artifact = DockerImageArtifact.put_tar( id=derive_id(env_id, "env-snapshot"), description=( @@ -174,14 +177,19 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N return snapshot -async def _check_changelog_empty(sandbox) -> bool: +def _snapshot_image_tag(env_id: str, environment_universe_id: str, layout: StackLayout) -> str: + """The image a snapshot builds on the VM, named for the slot too so two slots snapshotting one env and universe don't collide.""" + tag = _image_tag(env_id, environment_universe_id) + return f"{tag}-{layout.compose_project}" if layout.is_slot else tag + + +async def _check_changelog_empty(sandbox, *, layout: StackLayout = DEFAULT_STACK_LAYOUT) -> bool: """Return True if the _changelog table is empty (no modifications since data load).""" from agent_env.env.envs.service_db import DB_NAME, DB_USER from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME - from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH cmd = ( - f"docker compose -f {DOCKER_COMPOSE_PATH} exec -T {DATABASE_SERVICE_NAME} " + f"{layout.compose_command} exec -T {DATABASE_SERVICE_NAME} " f'psql -U {DB_USER} -d {DB_NAME} -tAc "SELECT count(*) FROM public._changelog"' ) output = await sandbox.exec_script(cmd) @@ -189,16 +197,20 @@ async def _check_changelog_empty(sandbox) -> bool: return count == 0 -async def _snapshot_servicedb(sandbox, env_id: str, environment_universe_id: str, log: ProgressCallback) -> str: +async def _snapshot_servicedb( + sandbox, env_id: str, environment_universe_id: str, log: ProgressCallback, *, layout: StackLayout = DEFAULT_STACK_LAYOUT, +) -> str: """Export pgdata from sandbox servicedb, build Docker image on sandbox, upload it to the object store. Returns the object URL of the uploaded tar.gz. """ from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME - from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH + pgdata = layout.scratch_path("pgdata") + build_dir = layout.scratch_path("snapshot-build") + tarball = layout.scratch_path("snapshot-image.tar.gz") container_id_output = await sandbox.exec_script( - f"docker compose -f {DOCKER_COMPOSE_PATH} ps -a -q {DATABASE_SERVICE_NAME}" + f"{layout.compose_command} ps -a -q {DATABASE_SERVICE_NAME}" ) container_id = container_id_output.strip() if not container_id: @@ -206,34 +218,34 @@ async def _snapshot_servicedb(sandbox, env_id: str, environment_universe_id: str log("snapshot_db", "Copying pgdata from container...", 45) logger.info("Exporting pgdata from servicedb container...") - await sandbox.exec_script(f"docker cp {container_id}:/var/lib/postgresql/data /tmp/pgdata") + await sandbox.exec_script(f"docker cp {container_id}:/var/lib/postgresql/data {pgdata}") - image_tag = _image_tag(env_id, environment_universe_id) + image_tag = _snapshot_image_tag(env_id, environment_universe_id, layout) log("snapshot_db", f"Building Docker image on sandbox: {image_tag}...", 50) logger.info(f"Building Docker image on sandbox: {image_tag}...") await sandbox.exec_script( - "mkdir -p /tmp/snapshot-build && " - "mv /tmp/pgdata /tmp/snapshot-build/pgdata && " - "cat > /tmp/snapshot-build/Dockerfile << 'DEOF'\n" + f"mkdir -p {build_dir} && " + f"mv {pgdata} {build_dir}/pgdata && " + f"cat > {build_dir}/Dockerfile << 'DEOF'\n" "FROM public.ecr.aws/docker/library/postgres:16-alpine\n" "COPY pgdata /var/lib/postgresql/data\n" "RUN chown -R postgres:postgres /var/lib/postgresql/data\n" "DEOF" ) await sandbox.exec_script( - f"docker build --platform linux/amd64 -t {image_tag} /tmp/snapshot-build" + f"docker build --platform linux/amd64 -t {image_tag} {build_dir}" ) log("snapshot_db", "Saving Docker image...", 55) logger.info("Saving Docker image to tar.gz on sandbox...") - await sandbox.exec_script(f"docker save {image_tag} | gzip > /tmp/snapshot-image.tar.gz") + await sandbox.exec_script(f"docker save {image_tag} | gzip > {tarball}") object_store, object_url = _tarball_destination(env_id, environment_universe_id) log("snapshot_db", "Uploading Docker image...", 60) logger.info(f"Uploading Docker image to {object_url}...") - await upload_vm_file(sandbox, "/tmp/snapshot-image.tar.gz", object_store, object_url) + await upload_vm_file(sandbox, tarball, object_store, object_url) - await sandbox.exec_script(f"rm -rf /tmp/snapshot-build /tmp/snapshot-image.tar.gz && docker rmi {image_tag}") + await sandbox.exec_script(f"rm -rf {build_dir} {tarball} && docker rmi {image_tag}") return object_url diff --git a/src/agent_env/providers/env_providers/constants.py b/src/agent_env/providers/env_providers/constants.py index 7a6a00e97..301da78c9 100644 --- a/src/agent_env/providers/env_providers/constants.py +++ b/src/agent_env/providers/env_providers/constants.py @@ -1,4 +1,4 @@ -"""The gateway deployment's layout: its compose service name, VM app dir and compose path, and website names. +"""The gateway deployment's layout: its compose service name, VM app dir, and website names. Imports nothing from agent_env, so modules in the ``agent_env.env`` import chain can use these while ``agent_env.providers`` is still importing. @@ -15,4 +15,3 @@ AGENT_ENV_WEBSITE_HEALTH_PATH = "/api/health" AGENT_ENV_WEBSITE_BACKEND_SUFFIX = "website-backend" AGENT_ENV_WEBSITE_FRONTEND_SUFFIX = "website-frontend" -DOCKER_COMPOSE_PATH = f"{GATEWAY_APP_DIR}/docker-compose.yml" diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 2e993e8b8..1deb332fb 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -62,11 +62,10 @@ AGENT_ENV_WEBSITE_BACKEND_SUFFIX, AGENT_ENV_WEBSITE_FRONTEND_SUFFIX, AGENT_ENV_WEBSITE_HEALTH_PATH, - DOCKER_COMPOSE_PATH, - GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME, GATEWAY_SERVICE_NAMES, ) +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.env_providers.env_provider import ( DeployRefused, _SandboxEnvironmentProvider, @@ -293,7 +292,7 @@ async def store_exec(argv: list[str]) -> tuple[int, str, str]: async def store_exec(argv: list[str]) -> tuple[int, str, str]: return await self._sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, + "sudo", *self._stack_layout.compose_argv, "exec", "-T", store_name, *argv, ) @@ -823,6 +822,7 @@ async def _deploy_via_vm( website_configs = website_configs or [] website_images = website_images or [] sidecars = sidecars or [] + layout = self._stack_layout config = get_config() gateway_env = Env.get(gateway_env_id(config)) @@ -858,13 +858,26 @@ async def _deploy_via_vm( if memory_mb is not None: vm_kwargs["memory"] = memory_mb self._sandbox = await sandbox_provider.create_vm( - exposed_ports=[gateway_port, DB_WEB_PORT, DB_MCP_PORT] + [sc.host_port for sc in sidecars], + exposed_ports=[layout.host_port(p) for p in [gateway_port, DB_WEB_PORT, DB_MCP_PORT, *(sc.host_port for sc in sidecars)]], timeout=ttl_seconds, disk_size_gb=disk_size_gb, attribution=attribution, **vm_kwargs, ) sandbox = self._sandbox + + def publish(port: int) -> int: + return sandbox.host_port(layout.host_port(port)) + + def tunnel_url(port: int) -> str | None: + host_port = layout.host_port(port) + url = sandbox.tunnel_urls.get(host_port) + if url is None and layout.is_slot: + raise RuntimeError( + f"Sandbox {sandbox.sandbox_id} has no tunnel for slot port {host_port} (container port {port}); " + "a shared VM must be created exposing every slot's ports" + ) + return url await sandbox.load_docker_images(all_images) # Compose declares the local_state service by name, so "standing up" is just naming the host; @@ -898,7 +911,7 @@ async def _stand_up(_spec): sidecars=sidecars, state_provider=self._state_provider, state_instance=state_instance, - host_port=sandbox.host_port, + host_port=publish, host_ips=sandbox.host_ips, extra_hosts=sandbox.extra_hosts, mcp_server_name=mcp_server_name, @@ -909,16 +922,16 @@ async def _stand_up(_spec): # Write files to VM logger.info("Writing files to VM...") - await sandbox.exec_script(f"mkdir -p {GATEWAY_APP_DIR}", max_retries=2) + await sandbox.exec_script(f"mkdir -p {layout.app_dir}", max_retries=2) # Only local postgres needs its init script to be directly run by the gateway's servicedb # container; remote/external stores are already initialized upstream. if self._needs_local_postgres: init_script = local_store_spec.init_sql logger.info(f"Generated init-schemas.sql:\n{init_script}") - await self._write_file(sandbox, f"{GATEWAY_APP_DIR}/init-schemas.sql", init_script) + await self._write_file(sandbox, layout.init_schemas_file, init_script) - await self._write_file(sandbox, DOCKER_COMPOSE_PATH, compose_content) + await self._write_file(sandbox, layout.compose_file, compose_content) # Start MCP services with docker compose logger.info("Starting MCP services...") @@ -928,14 +941,14 @@ async def _stand_up(_spec): # Everything but the gateway, beside the children; then the gateway, which waits for every child it fronts. # Both run to the end before either's failure is raised, so no child is still starting when the deploy closes. up, started = await asyncio.gather(self._compose_up(sandbox, f"--scale {GATEWAY_SERVICE_NAME}=0"), - children(sandbox.tunnel_urls.get(gateway_port), control_token), + children(tunnel_url(gateway_port), control_token), return_exceptions=True) for result in (up, started): if isinstance(result, BaseException): raise result external = tuple(replace(child, mcp_url=sandbox.url_from_sandbox(child.mcp_url), base_url=sandbox.url_from_sandbox(child.base_url)) for child in started) - await self._write_file(sandbox, DOCKER_COMPOSE_PATH, compose(external_children=external)) + await self._write_file(sandbox, layout.compose_file, compose(external_children=external)) await self._compose_up(sandbox) # Wait for containers to start @@ -948,17 +961,16 @@ async def _stand_up(_spec): if exit_code != 0: raise RuntimeError(f"docker ps failed: {stderr}") - # Check gateway container is running - exit_code, stdout, stderr = await sandbox.exec_with_output( "sudo", "docker", "ps") - if GATEWAY_SERVICE_NAME not in stdout: + running, listing = await self._gateway_running(sandbox) + if not running: down = await self._down_services(sandbox) raise RuntimeError(f"Gateway container not running:\n{down}" if down else - f"Gateway container not running. docker ps: {stdout}") + f"Gateway container not running. docker ps: {listing}") # Wait for gateway to be fully ready (needs time to discover internal servers) # Host-side port: exec_with_output runs inside the VM on per-VM backends but on # the host for LocalSandbox, where the gateway is published elsewhere. - gateway_ready = await self._wait_for_gateway(sandbox, sandbox.host_port(gateway_port)) + gateway_ready = await self._wait_for_gateway(sandbox, publish(gateway_port)) # Print gateway logs for debugging logger.info("Gateway logs:") @@ -977,10 +989,10 @@ async def _stand_up(_spec): if not await self._log_service(sandbox, name): logger.warning(f"{name} container not found") - gateway_url = sandbox.tunnel_urls.get(gateway_port) - db_web_base_url = sandbox.tunnel_urls.get(DB_WEB_PORT) if PGWEB_SERVICE_NAME in sidecar_names else None + gateway_url = tunnel_url(gateway_port) + db_web_base_url = tunnel_url(DB_WEB_PORT) if PGWEB_SERVICE_NAME in sidecar_names else None db_web_url = f"{db_web_base_url}/" if db_web_base_url else None - db_mcp_base_url = sandbox.tunnel_urls.get(DB_MCP_PORT) if DB_MCP_SERVICE_NAME in sidecar_names else None + db_mcp_base_url = tunnel_url(DB_MCP_PORT) if DB_MCP_SERVICE_NAME in sidecar_names else None db_mcp_url = f"{db_mcp_base_url}/mcp" if db_mcp_base_url else None logger.info(f"pgweb URL: {db_web_url}") if db_mcp_url: @@ -1249,7 +1261,9 @@ async def _compose_up(self, sandbox: VmSandbox, args: str = "") -> None: """``docker compose up -d`` the stack, whose failure names each service that went down and ends with its output, which compose's own error leaves out.""" try: - await sandbox.exec_script(f"cd {GATEWAY_APP_DIR} && docker compose up -d {args}".rstrip(), max_retries=2) + layout = self._stack_layout + await sandbox.exec_script(f"cd {layout.app_dir} && {layout.compose_command_from_app_dir} up -d {args}".rstrip(), + max_retries=2) except RuntimeError as e: down = await self._down_services(sandbox) if down is None: @@ -1269,7 +1283,7 @@ async def _down_services(self, sandbox: VmSandbox) -> str | None: async def _read_down_services(self, sandbox: VmSandbox) -> str | None: try: exit_code, stdout, stderr = await sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, "ps", "-a", "--format", "json" + "sudo", *self._stack_layout.compose_argv, "ps", "-a", "--format", "json" ) if exit_code != 0: logger.warning(f"Can't list the stack's services: docker compose ps exited {exit_code}: {stderr.strip()[-500:]}") @@ -1322,7 +1336,7 @@ async def _get_container_id(self, sandbox: VmSandbox, compose_service: str) -> s exec transport failed (exit -1) is retried, then raised: it can't tell "no container" apart.""" for attempt in range(_TRAJECTORY_READ_ATTEMPTS): exit_code, stdout, stderr = await sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, "ps", "-a", "-q", compose_service + "sudo", *self._stack_layout.compose_argv, "ps", "-a", "-q", compose_service ) if exit_code != -1: break @@ -1332,6 +1346,22 @@ async def _get_container_id(self, sandbox: VmSandbox, compose_service: str) -> s container_id = stdout.strip() return container_id if container_id else None + async def _gateway_running(self, sandbox: VmSandbox) -> tuple[bool, str]: + """Whether this stack's gateway container is running, and the listing that says so. + + The default stack keeps today's plain ``docker ps`` check, which any gateway container on the VM satisfies. That is + safe because a VM runs either one default stack or only slot stacks, never both. A slot asks its own compose project + (``compose ps --status running -q``), since another slot's gateway would satisfy the plain check. + """ + layout = self._stack_layout + if not layout.is_slot: + _, stdout, _ = await sandbox.exec_with_output("sudo", "docker", "ps") + return GATEWAY_SERVICE_NAME in stdout, stdout + _, stdout, _ = await sandbox.exec_with_output( + "sudo", *layout.compose_argv, "ps", "--status", "running", "-q", GATEWAY_SERVICE_NAME + ) + return bool(stdout.strip()), stdout + async def _provision_sidecar(self, deploy: _ContainerDeploy, spec) -> Sandbox: logger.info(f"Provisioning {spec.name} from {spec.image}...") sb = await deploy.sandbox_provider.create_container( @@ -1469,12 +1499,17 @@ async def _deploy_gateway( attribution: Optional[Attribution] = None, children: "Callable[[Optional[str], Optional[str]], Awaitable[list[ExternalChild]]] | None" = None, website_browser: "MCPServerEnv | None | object" = _THE_RUNS, + stack_layout: StackLayout = DEFAULT_STACK_LAYOUT, ) -> DeployedGateway: """One gateway deploy on one (unchained) sandbox provider: a VM on one that creates VMs, else a container per server on its private network. ``website_browser`` runs among its MCP servers, on the network it shares with - the sites; by default the run's (``run_website_browser``) when there are websites, which only a VM runs.""" + the sites; by default the run's (``run_website_browser``) when there are websites, which only a VM runs. + + ``stack_layout`` places the stack in a VM; the provider keeps it for the commands it later runs on that stack. + """ from agent_env.providers.env_state import LocalPostgresStateProvider, build_state_provider + self._stack_layout = stack_layout # Every attempt starts from the caller's store: a failed attempt's close() cleared it. self._state_instance = state_instance self._state_provider = ( diff --git a/src/agent_env/providers/env_providers/env_provider.py b/src/agent_env/providers/env_providers/env_provider.py index a6f4d1913..c98c236f2 100644 --- a/src/agent_env/providers/env_providers/env_provider.py +++ b/src/agent_env/providers/env_providers/env_provider.py @@ -14,6 +14,7 @@ from agent_env.env.env import DeployedEnv from agent_env.env.gateway.constants import WELL_KNOWN_PATH from agent_env.plugins import _registration +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.sandbox_providers.sandbox import Sandbox from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER @@ -75,6 +76,12 @@ class _SandboxEnvironmentProvider(EnvironmentProvider): def __init__(self): self._container_sandboxes: list[Sandbox] = [] self._environment_sandboxes: dict[str, Sandbox] = {} + self._stack_layout: StackLayout = DEFAULT_STACK_LAYOUT + + @property + def stack_layout(self) -> StackLayout: + """Where the stack this provider deployed lives in its sandbox; the default until a deploy lays out a slot.""" + return self._stack_layout def environment_sandbox(self, environment_name: str) -> Sandbox | None: return self._environment_sandboxes.get(environment_name) diff --git a/src/agent_env/providers/env_providers/stack_layout.py b/src/agent_env/providers/env_providers/stack_layout.py new file mode 100644 index 000000000..caad5f158 --- /dev/null +++ b/src/agent_env/providers/env_providers/stack_layout.py @@ -0,0 +1,124 @@ +"""Where a gateway stack lives in its sandbox: the default single stack, or one slot of several sharing a VM. + +Imports nothing from agent_env but ``constants``, so modules in the ``agent_env.env`` import chain can use it while +``agent_env.providers`` is still importing. +""" + +from __future__ import annotations + +import posixpath +import re +from dataclasses import dataclass + +from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR + +SLOT_HOST_PORT_STRIDE = 1000 +MAX_PORT = 65535 +_COMPOSE_PROJECT_NAME = re.compile(r"[a-z0-9][a-z0-9_-]*") +_SHELL_SAFE_PATH = re.compile(r"/[A-Za-z0-9/_.-]*") +_DEFAULT_IMPLICIT_PROJECT = posixpath.basename(GATEWAY_APP_DIR) + + +def _check_shell_safe_path(field: str, path: str) -> None: + """Paths are interpolated unquoted into shell scripts, so only absolute paths of a plain charset pass.""" + if not _SHELL_SAFE_PATH.fullmatch(path) or ".." in path.split("/"): + raise ValueError(f"{field} {path!r} must be absolute, use only letters, digits, '/', '_', '.' or '-', and have no '..'") + + +@dataclass(frozen=True) +class StackLayout: + """Where one gateway stack lives in its sandbox: its compose dir and project, its block of host ports and its scratch files. + + The default is the one stack a sandbox has always held. A slot is one of several stacks sharing a VM, so each gets its + own dir, compose project, host ports and scratch names; container ports and service names are the same in every stack. + """ + + app_dir: str = GATEWAY_APP_DIR + compose_project: str | None = None + host_port_offset: int = 0 + scratch_prefix: str = "/tmp/" + + def __post_init__(self) -> None: + _check_shell_safe_path("app dir", self.app_dir) + _check_shell_safe_path("scratch prefix", self.scratch_prefix) + if self.compose_project is not None: + if not _COMPOSE_PROJECT_NAME.fullmatch(self.compose_project): + raise ValueError( + f"compose project {self.compose_project!r} must be lowercase letters, digits, '-' or '_', starting with a letter or digit" + ) + if self.compose_project == _DEFAULT_IMPLICIT_PROJECT: + raise ValueError( + f"compose project {self.compose_project!r} is the default stack's implicit project (the basename of {GATEWAY_APP_DIR})" + ) + if self.host_port_offset < 0 or self.host_port_offset % SLOT_HOST_PORT_STRIDE: + raise ValueError(f"host port offset {self.host_port_offset} must be a non-negative multiple of {SLOT_HOST_PORT_STRIDE}") + if self.compose_project is None and ( + self.app_dir != GATEWAY_APP_DIR or self.host_port_offset or self.scratch_prefix != "/tmp/" + ): + raise ValueError("a layout that differs from the default stack's must name its compose project") + + @property + def is_slot(self) -> bool: + """A slot names its compose project; only the default stack leaves it to compose.""" + return self.compose_project is not None + + @classmethod + def for_slot(cls, index: int) -> StackLayout: + """Slot ``index`` of a shared VM: compose project ``s{index}`` names its dir and scratch files, its host ports are the index's block. + + The project is the hub's slot index, so the two cannot disagree. It is unique only within one docker daemon, which + each Modal VM has to itself, so slots are for VMs, not local sandboxes sharing the host's daemon. Block 0 is the + default stack's: a VM holds the default stack or slots, never both, but a slot starting one stride up keeps a + mix-up from publishing on the default stack's ports. The gateway + (18765) publishes on ``18765 + (index + 1) * 1000``, so 45 is the highest usable index; ``host_port`` raises past it. + """ + if index < 0: + raise ValueError(f"slot index must be >= 0, got {index}") + project = f"s{index}" + return cls( + app_dir=f"{GATEWAY_APP_DIR}/{project}", + compose_project=project, + host_port_offset=(index + 1) * SLOT_HOST_PORT_STRIDE, + scratch_prefix=f"/tmp/{project}-", + ) + + @property + def compose_file(self) -> str: + return f"{self.app_dir}/docker-compose.yml" + + @property + def init_schemas_file(self) -> str: + return f"{self.app_dir}/init-schemas.sql" + + @property + def artifact_staging_dir(self) -> str: + return f"{self.app_dir}/_artifact_staging" + + @property + def compose_argv(self) -> tuple[str, ...]: + """``docker compose`` bound to this stack's file and, for a slot, its project.""" + project = ("-p", self.compose_project) if self.is_slot else () + return ("docker", "compose", "-f", self.compose_file, *project) + + @property + def compose_command(self) -> str: + return " ".join(self.compose_argv) + + @property + def compose_command_from_app_dir(self) -> str: + """``docker compose`` for a script that has changed into ``app_dir``, where compose finds this stack's file.""" + project = f" -p {self.compose_project}" if self.is_slot else "" + return f"docker compose{project}" + + def scratch_path(self, name: str) -> str: + return f"{self.scratch_prefix}{name}" + + def host_port(self, port: int) -> int: + """The host port this stack publishes ``port`` on; raises if the shift leaves the port range.""" + shifted = port + self.host_port_offset + if shifted > MAX_PORT: + raise ValueError(f"port {port} shifted by {self.host_port_offset} is past {MAX_PORT}") + return shifted + + +DEFAULT_STACK_LAYOUT = StackLayout() diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 2c1c779b5..c5dd18489 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -41,7 +41,8 @@ ) from agentenv_protocol.computer.card import COMPUTER_RECORDING from agentenv_protocol.types import WELL_KNOWN_PATH -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH, GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME +from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.integration.env.envs.playwright_mcp_local_test import PROCESSES from agent_env.env.snapshot_store import compute_env_fingerprint @@ -1380,7 +1381,7 @@ async def assert_website_tools_via_step(client, gateway_url, tools, tool_names): browser = protocol_v1.find_child(result.environment_card, PLAYWRIGHT_MCP_ENVIRONMENT_NAME) assert browser is not None, result.environment_card.get("children_environments") sandbox = await LocalSandboxProvider().get_sandbox(result.sandbox_id) - compose = yaml.safe_load(await sandbox.exec_script(f"cat {DOCKER_COMPOSE_PATH}")) + compose = yaml.safe_load(await sandbox.exec_script(f"cat {DEFAULT_STACK_LAYOUT.compose_file}")) proxied = dict(item.split("=", 1) for item in compose["services"][GATEWAY_SERVICE_NAME]["environment"]) assert (f"mcp-{PLAYWRIGHT_MCP_ENVIRONMENT_NAME}=http://{PLAYWRIGHT_MCP_ENVIRONMENT_NAME}:18765" in proxied["REST_PROXY_URLS"].split(",")), proxied["REST_PROXY_URLS"] diff --git a/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py b/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py index 0a306b116..2a39b7ab7 100644 --- a/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py +++ b/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py @@ -14,7 +14,7 @@ from agent_env.env.envs.service_db import ServiceDBConfig from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT from agent_env.providers import EnvironmentGatewayProvider, MCPServerConfig, WebsiteConfig -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.env_state import LocalPostgresStateProvider @@ -127,7 +127,7 @@ async def test_the_reload_writes_its_compose_as_one_argument(): assert "$(touch pwned) 'q' `x`" in _written(env) scripts = [call.args[0] for call in env._sandbox.exec_script.await_args_list] - assert any(argv[:2] == ["printf", "%s\\n"] and argv[3:] == [">", DOCKER_COMPOSE_PATH] + assert any(argv[:2] == ["printf", "%s\\n"] and argv[3:] == [">", DEFAULT_STACK_LAYOUT.compose_file] for argv in map(shlex.split, scripts)) assert not any("<<" in script for script in scripts) diff --git a/tst/unit/env/envs/test_stack_layout_of.py b/tst/unit/env/envs/test_stack_layout_of.py new file mode 100644 index 000000000..b642ae060 --- /dev/null +++ b/tst/unit/env/envs/test_stack_layout_of.py @@ -0,0 +1,71 @@ +"""An env reads its stack's layout from the built-in provider that deployed it, and a slot's artifact loads stage inside that slot.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from agent_env.env.envs._deployment import stack_layout_of +from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.env.envs.website import WebsiteEnv +from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider +from agent_env.providers.env_providers.env_provider import EnvironmentProvider +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout + + +class _PluginProvider(EnvironmentProvider): + type = "plugin_layout" + + async def deploy(self, env, sandbox_provider, **options): + raise NotImplementedError + + async def close(self): + pass + + +def _slot_provider(index: int) -> EnvironmentGatewayProvider: + provider = EnvironmentGatewayProvider() + provider._stack_layout = StackLayout.for_slot(index) + return provider + + +def _vm_env(provider) -> SimpleNamespace: + sandbox = MagicMock(mode="vm", load_object_file=AsyncMock(), docker_cp=AsyncMock(), exec_script=AsyncMock(return_value="")) + provider._get_container_id = AsyncMock(return_value="cid") + return SimpleNamespace(_sandbox=sandbox, _env_provider=provider, environment_name="crm") + + +def _scripts(env: SimpleNamespace) -> list[str]: + return [c.args[0] for c in env._sandbox.exec_script.await_args_list] + + +def test_a_builtin_provider_gives_the_layout_it_deployed(): + assert stack_layout_of(_slot_provider(2)) == StackLayout.for_slot(2) + assert stack_layout_of(EnvironmentGatewayProvider()) == DEFAULT_STACK_LAYOUT + + +@pytest.mark.parametrize("provider", [None, _PluginProvider()]) +def test_a_plugin_provider_or_none_gives_the_default_layout(provider): + assert stack_layout_of(provider) == DEFAULT_STACK_LAYOUT + + +@pytest.mark.asyncio +async def test_a_slot_mcp_server_stages_its_artifact_in_the_slot_dir(): + env = _vm_env(_slot_provider(1)) + await MCPServerEnv._copy_artifact_into_container(env, SimpleNamespace(object_url="s3://b/mail.json"), "mail.json") + + staged = env._sandbox.load_object_file.await_args.args[1] + assert staged.startswith("/app/s1/_artifact_staging/crm-") + assert "mkdir -p /app/s1/_artifact_staging" in _scripts(env) + + +@pytest.mark.asyncio +async def test_a_slot_website_copies_and_removes_its_artifact_through_the_slot_compose_project(): + env = _vm_env(_slot_provider(1)) + await WebsiteEnv._copy_artifact_into_container(env, SimpleNamespace(object_url="s3://b/shop.json"), "shop.json") + await WebsiteEnv._remove_from_container(env, "shop.json") + + compose_scripts = [s for s in _scripts(env) if s.startswith("docker compose")] + assert compose_scripts and all(s.startswith("docker compose -f /app/s1/docker-compose.yml -p s1 ") for s in compose_scripts) diff --git a/tst/unit/providers/env_providers/stack_layout_test.py b/tst/unit/providers/env_providers/stack_layout_test.py new file mode 100644 index 000000000..8e9b334a1 --- /dev/null +++ b/tst/unit/providers/env_providers/stack_layout_test.py @@ -0,0 +1,322 @@ +"""A gateway stack's layout in its sandbox: the default is the one stack a sandbox has always held, a slot is one of several.""" + +from __future__ import annotations + +import shlex +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agent_env.env import snapshot_store +from agent_env.env.envs.service_db import DB_MCP_PORT, DB_WEB_PORT, ServiceDBConfig +from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT +from agent_env.providers.env_providers.env_gateway_provider import ( + EnvironmentGatewayProvider, + MCPServerConfig, + SidecarConfig, +) +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout + +SIDECAR_HOST_PORT = 9100 + + +class _RecordingVm: + """A VM sandbox that records every command and answers the ones a gateway deploy waits on. + + ``running`` holds the compose projects whose gateway is up, keyed ``None`` for the default (unnamed) project; ``docker ps`` + lists a gateway container for each of them, as it would on a VM shared by several stacks. + """ + + type = "modal_vm" + mode = "vm" + sandbox_id = "vm-1" + host_ips: tuple[str, ...] = () + extra_hosts: tuple[str, ...] = () + ON_THIS_MACHINE = False + + def __init__(self, exposed_ports: list[int], running: set[str | None]): + self.exposed_ports = exposed_ports + self.tunnel_urls = {port: f"https://tunnel-{port}" for port in exposed_ports} + self.running = running + self.calls: list[tuple] = [] + + def host_port(self, port: int) -> int: + return port + + async def load_docker_images(self, images) -> None: + self.calls.append(("load", tuple(i.image_name for i in images))) + + async def exec_script(self, script: str, *, max_retries: int = 0) -> str: + self.calls.append(("script", script)) + return "cid-servicedb" if " ps -a -q " in script else "" + + async def exec_with_output(self, *args: str) -> tuple[int, str, str]: + self.calls.append(("exec", args)) + if args[0] == "curl": + return 0, "200", "" + if args == ("sudo", "docker", "ps"): + return 0, "\n".join(f"{p or 'app'}-gateway-1" for p in self.running), "" + if "--status" in args: + project = args[args.index("-p") + 1] if "-p" in args else None + return 0, "cid-gateway" if project in self.running else "", "" + if args[-2:-1] == ("-q",): + return 0, f"cid-{args[-1]}", "" + return 0, "", "" + + +def _image(name: str) -> MagicMock: + image = MagicMock() + image.image_name = name + image.load_problem.return_value = None + return image + + +async def _deploy(running: set[str | None] = frozenset({None}), **layout) -> tuple[_RecordingVm, object, EnvironmentGatewayProvider]: + """One VM gateway deploy (an MCP server, local Postgres with pgweb and db-mcp, one sidecar) with every I/O faked.""" + vms: list[_RecordingVm] = [] + + async def create_vm(*, exposed_ports, **_): + vms.append(_RecordingVm(exposed_ports, set(running))) + return vms[-1] + + sandbox_provider = MagicMock(ON_THIS_MACHINE=False, SUPPORTED_SANDBOX_MODES=("vm",)) + sandbox_provider.create_vm = create_vm + gateway_env = MagicMock() + gateway_env.docker_image_artifact = _image("agent-gateway") + db_env = MagicMock() + db_env.to_config.return_value = ServiceDBConfig(db_web_image="pgweb", db_mcp_image="db-mcp") + db_env.db_docker_image_artifact = _image("postgres") + db_env.db_web_docker_image_artifact = _image("pgweb") + db_env.db_mcp_docker_image_artifact = _image("db-mcp") + config = MagicMock(default_gateway_env_id="gw-id", default_service_db_env_id="db-id") + + gp = EnvironmentGatewayProvider() + gp._wait_for_tunnel = AsyncMock(return_value={"name": "crm"}) + gp._refuse_tool_name_clashes = AsyncMock() + with patch("agent_env.env.env.Env.get", side_effect=lambda env_id, *a, **k: gateway_env if env_id == "gw-id" else db_env), \ + patch("agent_env.config.get_config", return_value=config), \ + patch("agent_env.providers.env_providers.env_gateway_provider.asyncio.sleep", AsyncMock()): + result = await gp._deploy_gateway( + sandbox_provider, + state_instance=None, + mcp_servers=[MCPServerConfig(image="mcp-slack", environment_name="slack")], + mcp_server_images=[_image("mcp-slack")], + sidecars=[SidecarConfig(environment_name="probe", image_artifact=_image("probe"), container_port=80, host_port=SIDECAR_HOST_PORT)], + mcp_server_name="crm", + **layout, + ) + return vms[0], result, gp + + +def _scripts(vm: _RecordingVm) -> list[str]: + return [c[1] for c in vm.calls if c[0] == "script"] + + +def _written(script: str) -> tuple[str, str] | None: + """The path and content of a ``_write_file`` script (``printf '%s\\n' > ``), else None.""" + argv = shlex.split(script) + if argv[:2] == ["printf", "%s\\n"] and argv[3:4] == [">"]: + return argv[4], argv[2] + return None + + +def _execs(vm: _RecordingVm) -> list[tuple]: + return [c[1] for c in vm.calls if c[0] == "exec" and c[1][0] != "curl"] + + +@pytest.mark.asyncio +async def test_default_deploy_runs_todays_commands_on_todays_paths_and_ports(): + vm, result, _ = await _deploy() + + assert vm.exposed_ports == [AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT] + scripts = _scripts(vm) + assert scripts[0] == "mkdir -p /app" + assert _written(scripts[1])[0] == "/app/init-schemas.sql" + path, compose = _written(scripts[2]) + assert path == "/app/docker-compose.yml" + assert f'"{AGENT_ENV_GATEWAY_MCP_PORT}:{AGENT_ENV_GATEWAY_MCP_PORT}"' in compose + assert f'"{SIDECAR_HOST_PORT}:80"' in compose + assert scripts[3:] == ["cd /app && docker compose up -d"] + execs = _execs(vm) + assert ("sudo", "docker", "ps") in execs + for service in ("gateway", "pgweb", "db-mcp"): + assert ("sudo", "docker", "compose", "-f", "/app/docker-compose.yml", "ps", "-a", "-q", service) in execs + assert not any("-p" in e or "--status" in e for e in execs) + assert result.gateway_url == f"https://tunnel-{AGENT_ENV_GATEWAY_MCP_PORT}" + assert result.db_web_url == f"https://tunnel-{DB_WEB_PORT}/" + assert result.db_mcp_url == f"https://tunnel-{DB_MCP_PORT}/mcp" + + +@pytest.mark.asyncio +async def test_slot_deploy_uses_its_own_dir_project_and_host_ports(): + slot = StackLayout.for_slot(1) + vm, result, gp = await _deploy(running={None, "s1"}, stack_layout=slot) + + shifted = [p + 2000 for p in (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT)] + assert vm.exposed_ports == shifted + scripts = _scripts(vm) + assert scripts[0] == "mkdir -p /app/s1" + assert _written(scripts[1])[0] == "/app/s1/init-schemas.sql" + path, compose = _written(scripts[2]) + assert path == "/app/s1/docker-compose.yml" + assert f'"{AGENT_ENV_GATEWAY_MCP_PORT + 2000}:{AGENT_ENV_GATEWAY_MCP_PORT}"' in compose + assert f'"{DB_WEB_PORT + 2000}:{DB_WEB_PORT}"' in compose + assert f'"{SIDECAR_HOST_PORT + 2000}:80"' in compose + assert f"MCP_PORT={AGENT_ENV_GATEWAY_MCP_PORT}" in compose + assert scripts[3:] == ["cd /app/s1 && docker compose -p s1 up -d"] + compose_ps = ("sudo", "docker", "compose", "-f", "/app/s1/docker-compose.yml", "-p", "s1", "ps") + assert (*compose_ps, "--status", "running", "-q", "gateway") in _execs(vm) + assert (*compose_ps, "-a", "-q", "gateway") in _execs(vm) + assert ("sudo", "docker", "ps") not in _execs(vm) + assert result.gateway_url == f"https://tunnel-{AGENT_ENV_GATEWAY_MCP_PORT + 2000}" + assert result.db_web_url == f"https://tunnel-{DB_WEB_PORT + 2000}/" + assert result.db_mcp_url == f"https://tunnel-{DB_MCP_PORT + 2000}/mcp" + assert gp.stack_layout == slot + + +@pytest.mark.asyncio +async def test_slot_deploy_is_not_satisfied_by_another_stacks_gateway(): + with pytest.raises(RuntimeError, match="Gateway container not running"): + await _deploy(running={None, "s0"}, stack_layout=StackLayout.for_slot(1)) + + +@pytest.mark.asyncio +async def test_slot_deploy_on_a_vm_not_exposing_its_ports_names_the_missing_port(): + vm = _RecordingVm([AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT], {"s1"}) + with pytest.raises(RuntimeError, match=f"no tunnel for slot port {AGENT_ENV_GATEWAY_MCP_PORT + 2000}"): + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) + + +@pytest.mark.asyncio +async def test_slot_commands_after_deploy_stay_in_the_slot(): + vm, _, gp = await _deploy(running={"s1"}, stack_layout=StackLayout.for_slot(1)) + gp._sandbox = vm + vm.calls.clear() + + await gp.read_trajectory(vm) + await gp.install_changelog_triggers("slack") + await gp._down_services(vm) + + slot_compose = ("sudo", "docker", "compose", "-f", "/app/s1/docker-compose.yml", "-p", "s1") + assert _execs(vm)[0] == (*slot_compose, "ps", "-a", "-q", "gateway") + assert _execs(vm)[-2][: len(slot_compose) + 3] == (*slot_compose, "exec", "-T", "servicedb") + assert _execs(vm)[-1] == (*slot_compose, "ps", "-a", "--format", "json") + + +@pytest.mark.asyncio +async def test_slot_snapshot_uses_its_own_compose_and_scratch_files(): + vm = _RecordingVm([], set()) + object_store = MagicMock() + object_store.signed_put_url.return_value = "https://put" + config = MagicMock(get_object_store_for=lambda _env_id: object_store, get_artifact_key_prefix=lambda: "") + with patch.object(snapshot_store, "get_config", return_value=config): + await snapshot_store._snapshot_servicedb(vm, "env1", "uni1", lambda *a: None, layout=StackLayout.for_slot(0)) + + scripts = _scripts(vm) + assert scripts[0] == "docker compose -f /app/s0/docker-compose.yml -p s0 ps -a -q servicedb" + assert all("/tmp/pgdata" not in s and "/tmp/snapshot-" not in s for s in scripts) + assert "/tmp/s0-pgdata" in scripts[1] and "/tmp/s0-snapshot-build" in scripts[2] + assert scripts[-1].startswith("rm -rf /tmp/s0-snapshot-build /tmp/s0-snapshot-image.tar.gz") + + +async def _snapshot_image_commands(layout: StackLayout) -> list[str]: + vm = _RecordingVm([], set()) + object_store = MagicMock() + object_store.signed_put_url.return_value = "https://put" + config = MagicMock(get_object_store_for=lambda _env_id: object_store, get_artifact_key_prefix=lambda: "") + with patch.object(snapshot_store, "get_config", return_value=config): + await snapshot_store._snapshot_servicedb(vm, "env1", "uni1", lambda *a: None, layout=layout) + return [s for s in _scripts(vm) if s.startswith(("docker build", "docker save")) or "docker rmi" in s] + + +@pytest.mark.asyncio +async def test_default_snapshot_image_tag_is_unchanged(): + build, save, rmi = await _snapshot_image_commands(DEFAULT_STACK_LAYOUT) + assert build == "docker build --platform linux/amd64 -t env-snapshot-env1-uni1 /tmp/snapshot-build" + assert save.startswith("docker save env-snapshot-env1-uni1 |") + assert rmi.endswith("docker rmi env-snapshot-env1-uni1") + + +@pytest.mark.asyncio +async def test_two_slots_snapshotting_one_env_and_universe_build_different_images(): + s0 = await _snapshot_image_commands(StackLayout.for_slot(0)) + s1 = await _snapshot_image_commands(StackLayout.for_slot(1)) + assert all("env-snapshot-env1-uni1-s0" in c for c in s0) + assert all("env-snapshot-env1-uni1-s1" in c for c in s1) + + +def test_default_layout_is_the_single_stack_paths_and_ports(): + assert DEFAULT_STACK_LAYOUT.compose_command == "docker compose -f /app/docker-compose.yml" + assert DEFAULT_STACK_LAYOUT.compose_command_from_app_dir == "docker compose" + assert DEFAULT_STACK_LAYOUT.artifact_staging_dir == "/app/_artifact_staging" + assert DEFAULT_STACK_LAYOUT.scratch_path("pgdata") == "/tmp/pgdata" + assert DEFAULT_STACK_LAYOUT.host_port(AGENT_ENV_GATEWAY_MCP_PORT) == AGENT_ENV_GATEWAY_MCP_PORT + + +def test_slots_get_disjoint_host_port_blocks_from_the_default_and_each_other(): + ports = (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT) + layouts = [DEFAULT_STACK_LAYOUT, *(StackLayout.for_slot(i) for i in range(8))] + published = [layout.host_port(p) for layout in layouts for p in ports] + assert len(set(published)) == len(published) + + +def test_for_slot_derives_its_project_dir_scratch_and_ports_from_the_index(): + slot = StackLayout.for_slot(3) + assert slot.compose_project == "s3" + assert slot.compose_command == "docker compose -f /app/s3/docker-compose.yml -p s3" + assert slot.artifact_staging_dir == "/app/s3/_artifact_staging" + assert slot.scratch_path("pgdata") == "/tmp/s3-pgdata" + assert slot.host_port(AGENT_ENV_GATEWAY_MCP_PORT) == AGENT_ENV_GATEWAY_MCP_PORT + 4000 + + +def test_for_slot_rejects_a_negative_index(): + with pytest.raises(ValueError): + StackLayout.for_slot(-1) + + +def test_slot_45_is_the_highest_whose_gateway_port_fits(): + assert StackLayout.for_slot(45).host_port(AGENT_ENV_GATEWAY_MCP_PORT) <= 65535 + with pytest.raises(ValueError): + StackLayout.for_slot(46).host_port(AGENT_ENV_GATEWAY_MCP_PORT) + + +@pytest.mark.parametrize("project", ["", "S0", "-s0", "s 0", "s/0", "app"]) +def test_layout_rejects_a_bad_project_or_the_default_stacks_implicit_one(project): + with pytest.raises(ValueError): + StackLayout(app_dir="/app/x", compose_project=project) + + +@pytest.mark.parametrize("app_dir", ["app", "/app/../etc", "/app/x;rm -rf /", "/app/$(id)", "/app/a b", "/app/'x'"]) +def test_layout_rejects_an_app_dir_unsafe_in_a_shell_script(app_dir): + with pytest.raises(ValueError): + StackLayout(app_dir=app_dir) + + +@pytest.mark.parametrize("scratch_prefix", ["tmp/", "/tmp/$x-", "/tmp/a b-", "/tmp/x;", "/tmp/../"]) +def test_layout_rejects_a_scratch_prefix_unsafe_in_a_shell_script(scratch_prefix): + with pytest.raises(ValueError): + StackLayout(scratch_prefix=scratch_prefix) + + +@pytest.mark.parametrize("offset", [-1000, 1, 999]) +def test_layout_rejects_an_offset_off_the_slot_stride(offset): + with pytest.raises(ValueError): + StackLayout(host_port_offset=offset) + + +def test_host_port_rejects_a_shift_past_the_port_range(): + with pytest.raises(ValueError): + StackLayout.for_slot(60).host_port(AGENT_ENV_GATEWAY_MCP_PORT) + + +@pytest.mark.parametrize("changes", [ + {"app_dir": "/app/s0"}, {"host_port_offset": 1000}, {"scratch_prefix": "/tmp/s0-"}, +]) +def test_a_layout_that_differs_from_the_default_must_name_its_project(changes): + with pytest.raises(ValueError, match="must name its compose project"): + StackLayout(**changes) + + +def test_only_a_named_project_makes_a_slot(): + assert not DEFAULT_STACK_LAYOUT.is_slot + assert StackLayout.for_slot(0).is_slot From 7a911c8ad0b00e559b466cce84220d5928d4a2c8 Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Wed, 18 Nov 2026 16:09:10 -0500 Subject: [PATCH 2/3] feat(env-providers): refuse a slot deploy unless its sandbox's walls are verified A slot stack may only go into an existing sandbox whose `walls_verified` is true. That is a read-only property on Sandbox, false for every sandbox today, so no slot can be deployed until the framework sets up and checks the walls between slots. The check runs before the provider records the layout, so a refused deploy leaves the provider on its previous layout. The default layout is now also a class attribute, so a provider subclass that skips `super().__init__` keeps working. Co-Authored-By: Claude Opus 5.5 --- .../env_providers/env_gateway_provider.py | 5 +- .../providers/env_providers/env_provider.py | 3 +- .../providers/sandbox_providers/sandbox.py | 6 ++ .../env_providers/stack_layout_test.py | 66 ++++++++++++++++--- 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 1deb332fb..27b093c9e 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -858,7 +858,7 @@ async def _deploy_via_vm( if memory_mb is not None: vm_kwargs["memory"] = memory_mb self._sandbox = await sandbox_provider.create_vm( - exposed_ports=[layout.host_port(p) for p in [gateway_port, DB_WEB_PORT, DB_MCP_PORT, *(sc.host_port for sc in sidecars)]], + exposed_ports=[gateway_port, DB_WEB_PORT, DB_MCP_PORT] + [sc.host_port for sc in sidecars], timeout=ttl_seconds, disk_size_gb=disk_size_gb, attribution=attribution, @@ -1509,6 +1509,9 @@ async def _deploy_gateway( """ from agent_env.providers.env_state import LocalPostgresStateProvider, build_state_provider + if stack_layout.is_slot and (existing_sandbox is None or not existing_sandbox.walls_verified): + raise ValueError(f"Can't deploy into slot {stack_layout.compose_project}: a slot needs an existing sandbox " + f"whose walls between slots are verified") self._stack_layout = stack_layout # Every attempt starts from the caller's store: a failed attempt's close() cleared it. self._state_instance = state_instance diff --git a/src/agent_env/providers/env_providers/env_provider.py b/src/agent_env/providers/env_providers/env_provider.py index c98c236f2..3b71b1f0b 100644 --- a/src/agent_env/providers/env_providers/env_provider.py +++ b/src/agent_env/providers/env_providers/env_provider.py @@ -73,10 +73,11 @@ class _SandboxEnvironmentProvider(EnvironmentProvider): record_class: ClassVar[type[DeployedEnv]] = DeployedEnv # the class its records load back as + _stack_layout: StackLayout = DEFAULT_STACK_LAYOUT + def __init__(self): self._container_sandboxes: list[Sandbox] = [] self._environment_sandboxes: dict[str, Sandbox] = {} - self._stack_layout: StackLayout = DEFAULT_STACK_LAYOUT @property def stack_layout(self) -> StackLayout: diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index 6b46f29e1..b784d21d8 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -174,6 +174,12 @@ class Sandbox(ABC): # The host other containers on its provider's private network reach it at, ready for a URL; None when it's on none. private_host: str | None = None + @property + def walls_verified(self) -> bool: + """Whether the framework itself set up and tested the walls between slots on this sandbox, so stacks of + different runs may share it. False for every sandbox today: nothing sets up those walls yet.""" + return False + def url_from_sandbox(self, url: str) -> str: """``url``, of a service this process reaches, as this sandbox reaches it.""" return url diff --git a/tst/unit/providers/env_providers/stack_layout_test.py b/tst/unit/providers/env_providers/stack_layout_test.py index 8e9b334a1..ff0dd394c 100644 --- a/tst/unit/providers/env_providers/stack_layout_test.py +++ b/tst/unit/providers/env_providers/stack_layout_test.py @@ -34,8 +34,9 @@ class _RecordingVm: extra_hosts: tuple[str, ...] = () ON_THIS_MACHINE = False - def __init__(self, exposed_ports: list[int], running: set[str | None]): + def __init__(self, exposed_ports: list[int], running: set[str | None], walls_verified: bool = False): self.exposed_ports = exposed_ports + self.walls_verified = walls_verified self.tunnel_urls = {port: f"https://tunnel-{port}" for port in exposed_ports} self.running = running self.calls: list[tuple] = [] @@ -105,7 +106,7 @@ async def create_vm(*, exposed_ports, **_): mcp_server_name="crm", **layout, ) - return vms[0], result, gp + return (vms[0] if vms else layout["existing_sandbox"]), result, gp def _scripts(vm: _RecordingVm) -> list[str]: @@ -147,13 +148,19 @@ async def test_default_deploy_runs_todays_commands_on_todays_paths_and_ports(): assert result.db_mcp_url == f"https://tunnel-{DB_MCP_PORT}/mcp" +SLOT_1_PORTS = [p + 2000 for p in (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT)] + + +def _walled_vm(running: set[str | None], ports: list[int] = SLOT_1_PORTS) -> _RecordingVm: + return _RecordingVm(ports, running, walls_verified=True) + + @pytest.mark.asyncio async def test_slot_deploy_uses_its_own_dir_project_and_host_ports(): slot = StackLayout.for_slot(1) - vm, result, gp = await _deploy(running={None, "s1"}, stack_layout=slot) + vm = _walled_vm({None, "s1"}) + _, result, gp = await _deploy(stack_layout=slot, existing_sandbox=vm) - shifted = [p + 2000 for p in (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT)] - assert vm.exposed_ports == shifted scripts = _scripts(vm) assert scripts[0] == "mkdir -p /app/s1" assert _written(scripts[1])[0] == "/app/s1/init-schemas.sql" @@ -174,22 +181,65 @@ async def test_slot_deploy_uses_its_own_dir_project_and_host_ports(): assert gp.stack_layout == slot +@pytest.mark.asyncio +async def test_a_slot_is_never_deployed_on_a_new_vm(): + with pytest.raises(ValueError, match="walls between slots are verified"): + await _deploy(stack_layout=StackLayout.for_slot(1)) + + +@pytest.mark.asyncio +async def test_a_slot_is_never_deployed_on_a_vm_whose_walls_are_not_verified(): + vm = _RecordingVm(SLOT_1_PORTS, {"s1"}) + with pytest.raises(ValueError, match="walls between slots are verified"): + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) + assert vm.calls == [] + + +def test_no_sandbox_has_verified_walls_today(): + import importlib + import pkgutil + + import agent_env.providers.sandbox_providers as package + from agent_env.providers.sandbox_providers.sandbox import Sandbox + + for module in pkgutil.walk_packages(package.__path__, f"{package.__name__}.", onerror=lambda _: None): + try: + importlib.import_module(module.name) + except ImportError: + continue + subclasses, pending = [], [Sandbox] + while pending: + subclasses += (children := pending.pop().__subclasses__()) + pending += children + assert subclasses + assert [c.__name__ for c in subclasses if c.walls_verified is not Sandbox.walls_verified] == [] + + +def test_a_provider_subclass_that_skips_init_keeps_the_default_layout(): + class Skips(EnvironmentGatewayProvider): + def __init__(self): + pass + + assert Skips().stack_layout == DEFAULT_STACK_LAYOUT + + @pytest.mark.asyncio async def test_slot_deploy_is_not_satisfied_by_another_stacks_gateway(): with pytest.raises(RuntimeError, match="Gateway container not running"): - await _deploy(running={None, "s0"}, stack_layout=StackLayout.for_slot(1)) + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=_walled_vm({None, "s0"})) @pytest.mark.asyncio async def test_slot_deploy_on_a_vm_not_exposing_its_ports_names_the_missing_port(): - vm = _RecordingVm([AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT], {"s1"}) + vm = _walled_vm({"s1"}, [AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT]) with pytest.raises(RuntimeError, match=f"no tunnel for slot port {AGENT_ENV_GATEWAY_MCP_PORT + 2000}"): await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) @pytest.mark.asyncio async def test_slot_commands_after_deploy_stay_in_the_slot(): - vm, _, gp = await _deploy(running={"s1"}, stack_layout=StackLayout.for_slot(1)) + vm = _walled_vm({"s1"}) + _, _, gp = await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) gp._sandbox = vm vm.calls.clear() From e5464a54b4462e39a9e937534b6ae5ab734df26f Mon Sep 17 00:00:00 2001 From: Man Yin Kwok Date: Thu, 19 Nov 2026 17:06:52 -0500 Subject: [PATCH 3/3] feat(env-providers): keep a slot's stack layout on its deploy record A reload rebuilt the provider with the default layout, so a slot's later commands would have gone to /app/docker-compose.yml on a VM other runs share. The record now keeps the slot's layout and a reattach takes it back; a damaged recorded layout raises instead of falling back to the default stack. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/env.py | 6 ++- .../env_providers/env_gateway_provider.py | 5 ++- .../providers/env_providers/stack_layout.py | 23 +++++++++++ .../env_gateway_provider_test.py | 40 ++++++++++++++++++- .../env_providers/stack_layout_test.py | 25 ++++++++++++ 5 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index bea7e06f4..377bede5e 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -238,6 +238,9 @@ class DeployedGatewayEnv(DeployedSandboxEnv): # Each child env on a sandbox of its own, which the gateway fronts (a multi env's computers): its own deploy record, # by its environment name. child_envs: dict[str, dict] = field(default_factory=dict) + # Where the gateway's stack lives when it is one slot of a VM other runs share (StackLayout.to_record); None for + # the default stack, which has the VM to itself. + stack_layout: Optional[dict] = None @classmethod def _fields_from(cls, data: dict) -> dict: @@ -252,6 +255,7 @@ def _fields_from(cls, data: dict) -> dict: "vnc_url": data.get("vnc_url"), "env_state_instance_ids": data.get("env_state_instance_ids") or [], "child_envs": data.get("child_envs") or {}, + "stack_layout": data.get("stack_layout"), } @@ -584,4 +588,4 @@ def _record_class(data: dict) -> type[DeployedEnv]: # A record carrying any of these is a gateway's. Not gateway_mode: an older kernel writes its default into any record it round-trips. _GATEWAY_ONLY_FIELDS = ("gateway_url", "db_web_url", "db_mcp_url", "website_frontend_urls", "vnc_url", "env_state_instance_ids", - "child_envs") + "child_envs", "stack_layout") diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 27b093c9e..bafe2b816 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -708,6 +708,7 @@ async def deploy( gateway_mode=gateway_mode.value, env_state_instance_ids=deployed.env_state_instance_ids, control_token=deployed.control_token, + stack_layout=self._stack_layout.to_record(), ) async def create_gateway( @@ -1571,9 +1572,11 @@ def _sandbox_ids(self, always_name_gateway: bool = False) -> dict[str, str | dic return sandbox_ids async def _reattach(self, env: Env, deployed: DeployedEnv) -> Sandbox: - """A record this provider wrote, reattached: a MultiEnv's through its own rebuild, any other env's as the base reattaches it.""" + """A record this provider wrote, reattached: a MultiEnv's through its own rebuild, any other env's as the base reattaches it. + Either way the provider takes back the stack layout the record kept, so its commands reach the same stack.""" from agent_env.env.envs.multi_env import MultiEnv + self._stack_layout = StackLayout.from_record(getattr(deployed, "stack_layout", None)) if isinstance(env, MultiEnv): return await self._reattach_multi_env(deployed) return await super()._reattach(env, deployed) diff --git a/src/agent_env/providers/env_providers/stack_layout.py b/src/agent_env/providers/env_providers/stack_layout.py index caad5f158..f3277ec36 100644 --- a/src/agent_env/providers/env_providers/stack_layout.py +++ b/src/agent_env/providers/env_providers/stack_layout.py @@ -110,6 +110,29 @@ def compose_command_from_app_dir(self) -> str: project = f" -p {self.compose_project}" if self.is_slot else "" return f"docker compose{project}" + def to_record(self) -> dict | None: + """What a deploy record keeps of this layout, so a reload finds the same stack; None for the default stack.""" + if not self.is_slot: + return None + return {"app_dir": self.app_dir, "compose_project": self.compose_project, + "host_port_offset": self.host_port_offset, "scratch_prefix": self.scratch_prefix} + + @classmethod + def from_record(cls, data: object) -> StackLayout: + """The layout a deploy record kept; the default stack for a record that kept none. Raises for one that isn't a + valid slot layout, so a damaged record never sends a slot's commands to the default stack.""" + if data is None: + return DEFAULT_STACK_LAYOUT + fields = ("app_dir", "compose_project", "host_port_offset", "scratch_prefix") + if not isinstance(data, dict) or set(data) != set(fields): + raise ValueError(f"a recorded stack layout must have exactly the fields {fields}, got {data!r}") + if not isinstance(data["host_port_offset"], int) or isinstance(data["host_port_offset"], bool): + raise ValueError(f"a recorded stack layout's host_port_offset must be an int, got {data['host_port_offset']!r}") + layout = cls(**data) + if not layout.is_slot: + raise ValueError(f"a recorded stack layout must be a slot's, got {data!r}") + return layout + def scratch_path(self, name: str) -> str: return f"{self.scratch_prefix}{name}" diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index ab9588196..d8df4242a 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -26,7 +26,8 @@ from pytest_socket import enable_socket from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.env.env import DeployedGatewayEnv, _mcp_url +from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, _mcp_url +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.service_db import SERVICE_DB_PORT @@ -3111,3 +3112,40 @@ def test_a_site_image_without_curl_passes_its_health_probe_once_its_health_path_ def test_a_site_answering_only_errors_fails_its_health_probe(tmp_path): assert _run_probe(tmp_path, {"/.well-known/agent-env.json": 500, "/api/health": 503}) != 0 + + +@pytest.mark.asyncio +async def test_a_reattach_takes_back_the_slot_layout_its_record_kept(): + sandboxes = {i: _FakeSandbox(i, 18765, f"https://{i}") for i in ("gw", "srv", "db", "pg")} + record = dataclasses.replace(_multi_record([]), stack_layout=StackLayout.for_slot(2).to_record()) + gp = EnvironmentGatewayProvider() + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(side_effect=sandboxes.get))): + await gp._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), record) + assert gp.stack_layout == StackLayout.for_slot(2) + + +@pytest.mark.asyncio +async def test_a_reattach_of_a_record_with_no_layout_uses_the_default_stack(): + sandboxes = {i: _FakeSandbox(i, 18765, f"https://{i}") for i in ("gw", "srv", "db", "pg")} + gp = EnvironmentGatewayProvider() + gp._stack_layout = StackLayout.for_slot(1) + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(side_effect=sandboxes.get))): + await gp._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), _multi_record([])) + assert gp.stack_layout is DEFAULT_STACK_LAYOUT + + +@pytest.mark.asyncio +async def test_a_reattach_of_a_damaged_layout_raises_before_touching_any_sandbox(): + get_sandbox = AsyncMock() + record = dataclasses.replace(_multi_record([]), stack_layout={"compose_project": "s1"}) + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=get_sandbox)), \ + pytest.raises(ValueError): + await EnvironmentGatewayProvider()._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), record) + get_sandbox.assert_not_awaited() + + +def test_a_gateway_record_keeps_its_slot_layout_through_storage_and_an_older_record_has_none(): + record = dataclasses.replace(_multi_record([]), stack_layout=StackLayout.for_slot(4).to_record()) + assert DeployedEnv.from_dict(dataclasses.asdict(record)).stack_layout == StackLayout.for_slot(4).to_record() + older = {k: v for k, v in dataclasses.asdict(_multi_record([])).items() if k != "stack_layout"} + assert DeployedEnv.from_dict(older).stack_layout is None diff --git a/tst/unit/providers/env_providers/stack_layout_test.py b/tst/unit/providers/env_providers/stack_layout_test.py index ff0dd394c..9a6ccf1f0 100644 --- a/tst/unit/providers/env_providers/stack_layout_test.py +++ b/tst/unit/providers/env_providers/stack_layout_test.py @@ -370,3 +370,28 @@ def test_a_layout_that_differs_from_the_default_must_name_its_project(changes): def test_only_a_named_project_makes_a_slot(): assert not DEFAULT_STACK_LAYOUT.is_slot assert StackLayout.for_slot(0).is_slot + + +@pytest.mark.parametrize("index", [0, 3, 45]) +def test_a_slot_layout_comes_back_the_same_from_its_record(index): + layout = StackLayout.for_slot(index) + assert StackLayout.from_record(layout.to_record()) == layout + + +def test_the_default_stack_records_nothing_and_a_record_without_a_layout_is_the_default(): + assert DEFAULT_STACK_LAYOUT.to_record() is None + assert StackLayout.from_record(None) is DEFAULT_STACK_LAYOUT + + +@pytest.mark.parametrize("data", [ + "s1", + {"app_dir": "/app/s1", "compose_project": "s1", "host_port_offset": 2000}, + {**StackLayout.for_slot(1).to_record(), "extra": 1}, + {**StackLayout.for_slot(1).to_record(), "host_port_offset": True}, + {**StackLayout.for_slot(1).to_record(), "host_port_offset": "2000"}, + {**StackLayout.for_slot(1).to_record(), "app_dir": "/app/../etc"}, + {"app_dir": "/app", "compose_project": None, "host_port_offset": 0, "scratch_prefix": "/tmp/"}, +]) +def test_a_damaged_recorded_layout_is_refused_rather_than_read_as_the_default_stack(data): + with pytest.raises((ValueError, TypeError)): + StackLayout.from_record(data)