diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index bea7e06f4..377bede5e 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -238,6 +238,9 @@ class DeployedGatewayEnv(DeployedSandboxEnv): # Each child env on a sandbox of its own, which the gateway fronts (a multi env's computers): its own deploy record, # by its environment name. child_envs: dict[str, dict] = field(default_factory=dict) + # Where the gateway's stack lives when it is one slot of a VM other runs share (StackLayout.to_record); None for + # the default stack, which has the VM to itself. + stack_layout: Optional[dict] = None @classmethod def _fields_from(cls, data: dict) -> dict: @@ -252,6 +255,7 @@ def _fields_from(cls, data: dict) -> dict: "vnc_url": data.get("vnc_url"), "env_state_instance_ids": data.get("env_state_instance_ids") or [], "child_envs": data.get("child_envs") or {}, + "stack_layout": data.get("stack_layout"), } @@ -584,4 +588,4 @@ def _record_class(data: dict) -> type[DeployedEnv]: # A record carrying any of these is a gateway's. Not gateway_mode: an older kernel writes its default into any record it round-trips. _GATEWAY_ONLY_FIELDS = ("gateway_url", "db_web_url", "db_mcp_url", "website_frontend_urls", "vnc_url", "env_state_instance_ids", - "child_envs") + "child_envs", "stack_layout") diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 426dfa45c..1f5e1ae87 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -17,6 +17,7 @@ from agent_env.env.env import DeployedSandboxEnv, gateway_url_of from agent_env.env.gateway.constants import DATA_PLANE_LOAD_TIMEOUT_S from agent_env.env.store import register_env_instance +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.store.base import ObjectNotFoundError from agent_env.utils.paths import validate_file_name @@ -322,6 +323,12 @@ def as_builtin(provider: Optional[EnvironmentProvider]) -> Optional[_SandboxEnvi return provider if isinstance(provider, _SandboxEnvironmentProvider) else None +def stack_layout_of(provider: Optional[EnvironmentProvider]) -> StackLayout: + """Where a built-in provider's stack lives in its sandbox; the default for any other provider, or none.""" + builtin = as_builtin(provider) + return builtin.stack_layout if builtin is not None else DEFAULT_STACK_LAYOUT + + def provider_options(env: Env, provider: EnvironmentProvider, **options) -> dict: """The options provider takes: every one for a deploy(**options), else those it names, one it doesn't being dropped at deploy()'s default; raises for a deploy refused (see ``deploy_refusal``).""" diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 44f5c02dd..c26a15de0 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -22,7 +22,7 @@ from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, - reset_server_data, + reset_server_data, stack_layout_of, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -260,8 +260,7 @@ async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: # Stage on the disk-backed app dir, not /tmp (tmpfs/RAM): a large # artifact (github ~8GB) OOM-kills the copy under concurrent loads. # Random suffix avoids collisions between concurrent loads. - from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR - stage_dir = f"{GATEWAY_APP_DIR}/_artifact_staging" + stage_dir = stack_layout_of(self._env_provider).artifact_staging_dir vm_temp_path = f"{stage_dir}/{self.environment_name}-{uuid.uuid4().hex[:8]}-{posixpath.basename(path)}" await self._sandbox.exec_script(f"mkdir -p {shlex.quote(stage_dir)}") await self._sandbox.load_object_file(file_artifact.object_url, vm_temp_path) diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index 3239c4ed3..48840f37e 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -43,7 +43,7 @@ def _snapshot_after_load_default() -> bool: from agent_env.env.gateway import GatewayMode from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + plugin_provider_like_a_builtin, provider_or_class, reset_server_data, stack_layout_of, ) from agent_env.env import store as env_store from agent_env.env.envs import _multi_computers @@ -52,7 +52,7 @@ def _snapshot_after_load_default() -> bool: from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.website import WebsiteEnv from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.env_state.local_postgres import INIT_SCRIPT_MOUNT_PATH from agent_env.store.base import NotFoundError from agent_env.store.ids import derive_id @@ -117,14 +117,14 @@ def browser_problems(envs: list[Env]) -> list[str]: _SERVICE_DB_IMAGE = re.compile(rf"^( {re.escape(DATABASE_SERVICE_NAME)}:\n image: ).*$", re.MULTILINE) -def snapshot_compose(compose: str, db_image: str) -> str: - """``compose``, the file a deploy wrote, with its servicedb started from ``db_image`` and without the schema init - script, whose work a snapshot already holds. Every other service stays as the deploy wrote it.""" +def snapshot_compose(compose: str, db_image: str, *, compose_file: str = DEFAULT_STACK_LAYOUT.compose_file) -> str: + """``compose``, the file a deploy wrote at ``compose_file``, with its servicedb started from ``db_image`` and without + the schema init script, whose work a snapshot already holds. Every other service stays as the deploy wrote it.""" if problem := image_reference_problem(db_image): raise ValueError(problem) swapped, count = _SERVICE_DB_IMAGE.subn(lambda match: match.group(1) + db_image, compose) if count != 1: - raise RuntimeError(f"{DOCKER_COMPOSE_PATH} names a {DATABASE_SERVICE_NAME} image {count} times, not once, so " + raise RuntimeError(f"{compose_file} names a {DATABASE_SERVICE_NAME} image {count} times, not once, so " "the snapshot can't replace it") return swapped.replace(f" volumes:\n - ./init-schemas.sql:{INIT_SCRIPT_MOUNT_PATH}\n", "") @@ -711,7 +711,7 @@ async def _resumable_environments(self, environment_universe_artifact, total: in )) if not recorded: return set() - if not await _check_changelog_empty(self._sandbox): + if not await _check_changelog_empty(self._sandbox, layout=stack_layout_of(self._env_provider)): logger.info( "Not resuming: the changelog is non-empty, so this env's data has been " "modified since it was loaded and every service must be re-seeded" @@ -880,7 +880,6 @@ async def _load_from_snapshot(self, snapshot) -> None: from agent_env.env.envs.service_db import DB_USER from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT from agent_env.providers import EnvironmentGatewayProvider - from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR from agent_env.providers.env_state import LocalPostgresStateProvider from agent_env.config import get_config, service_db_env_id @@ -888,6 +887,7 @@ async def _load_from_snapshot(self, snapshot) -> None: raise RuntimeError("Environment not deployed - call deploy() first") if self._sandbox.mode == SANDBOX_MODE_CONTAINER: raise NotImplementedError("Snapshot loading not supported on container mode") + layout = stack_layout_of(self._env_provider) db_image = Artifact.get(snapshot.db_image_artifact_id, snapshot.db_image_artifact_version) if not isinstance(db_image, DockerImageArtifact): raise RuntimeError(f"Snapshot artifact {snapshot.db_image_artifact_id} is not a DockerImageArtifact") @@ -896,19 +896,21 @@ async def _load_from_snapshot(self, snapshot) -> None: logger.info("Pointing docker-compose.yml's servicedb at the snapshot image...") # The write adds back the newline cat returns, so the file doesn't grow a blank line each reload. - deployed = (await self._sandbox.exec_script(f"cat {DOCKER_COMPOSE_PATH}")).removesuffix("\n") - compose = snapshot_compose(deployed, db_image.image_name) + deployed = (await self._sandbox.exec_script(f"cat {layout.compose_file}")).removesuffix("\n") + compose = snapshot_compose(deployed, db_image.image_name, compose_file=layout.compose_file) gw = EnvironmentGatewayProvider() gw._sandbox = self._sandbox + gw._stack_layout = layout # The state store is the deploy's local Postgres; the changelog triggers below run through it. gw._state_provider = LocalPostgresStateProvider(service_db_config=Env.get(service_db_env_id(get_config())).to_config()) gw._state_instance = LocalPostgresStateProvider.default_instance() - await gw._write_file(self._sandbox, DOCKER_COMPOSE_PATH, compose) + await gw._write_file(self._sandbox, layout.compose_file, compose) # Remove old servicedb container and its anonymous volume, then start fresh from snapshot image logger.info("Recreating servicedb container with snapshot image...") await self._sandbox.exec_script( - f"cd {GATEWAY_APP_DIR} && docker compose rm -sf -v {DATABASE_SERVICE_NAME} && docker compose up -d {DATABASE_SERVICE_NAME} 2>&1" + f"cd {layout.app_dir} && {layout.compose_command_from_app_dir} rm -sf -v {DATABASE_SERVICE_NAME} && " + f"{layout.compose_command_from_app_dir} up -d {DATABASE_SERVICE_NAME} 2>&1" ) # Wait for servicedb to be healthy @@ -916,7 +918,7 @@ async def _load_from_snapshot(self, snapshot) -> None: for i in range(60): try: await self._sandbox.exec_script( - f"docker compose -f {DOCKER_COMPOSE_PATH} exec -T {DATABASE_SERVICE_NAME} pg_isready -U {DB_USER}", + f"{layout.compose_command} exec -T {DATABASE_SERVICE_NAME} pg_isready -U {DB_USER}", ) logger.info(f"servicedb healthy after {i + 1}s") break @@ -938,7 +940,7 @@ async def _load_from_snapshot(self, snapshot) -> None: if name != DATABASE_SERVICE_NAME))) logger.info(f"Recreating services: {svc_list}") await gw._compose_up(self._sandbox, f"--force-recreate {svc_list}") - await gw._wait_for_gateway(self._sandbox, self._sandbox.host_port(AGENT_ENV_GATEWAY_MCP_PORT)) + await gw._wait_for_gateway(self._sandbox, self._sandbox.host_port(layout.host_port(AGENT_ENV_GATEWAY_MCP_PORT))) # Install changelog triggers for each service for name in environment_names: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index b7a01c7d2..0875528a8 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -45,7 +45,7 @@ from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, - reset_server_data, + reset_server_data, stack_layout_of, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -340,14 +340,14 @@ async def load_file_artifact_universe(self, file_artifact_universe: "Any", desti return await super().load_file_artifact_universe(file_artifact_universe, destination_path) async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: - from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX, DOCKER_COMPOSE_PATH + from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX validate_file_path(path) backend_service = f"{self.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}" # Random suffix so concurrent loads on a shared sandbox (parallel /load-universe, # MCP+website sharing a service_name, repeated loads) don't race on a shared path. vm_path = f"/tmp/_artifact_{self.environment_name}-{uuid.uuid4().hex[:8]}-{posixpath.basename(path)}" container_path = f"/tmp/data/{path}" - compose = f"docker compose -f {shlex.quote(DOCKER_COMPOSE_PATH)}" + compose = stack_layout_of(self._env_provider).compose_command await self._sandbox.load_object_file(file_artifact.object_url, vm_path) await self._sandbox.exec_script( @@ -358,11 +358,12 @@ async def _copy_artifact_into_container(self, file_artifact, path: str) -> str: async def _remove_from_container(self, path: str) -> None: """Remove what a load staged at ``/tmp/data/``, best-effort: a load that finished stands either way.""" - from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX, DOCKER_COMPOSE_PATH + from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_SUFFIX backend_service = f"{self.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}" container_path = f"/tmp/data/{path}" + compose = stack_layout_of(self._env_provider).compose_command try: - await self._sandbox.exec_script(f"docker compose -f {shlex.quote(DOCKER_COMPOSE_PATH)} exec -T " + await self._sandbox.exec_script(f"{compose} exec -T " f"{shlex.quote(backend_service)} rm -rf {shlex.quote(container_path)}") except Exception as e: # noqa: BLE001 -- cleanup, not the load logger.warning(f"[{self.environment_name}] could not remove {container_path} after a load (ignored): {e}") diff --git a/src/agent_env/env/snapshot_store.py b/src/agent_env/env/snapshot_store.py index 279abdd76..53fbf1c78 100644 --- a/src/agent_env/env/snapshot_store.py +++ b/src/agent_env/env/snapshot_store.py @@ -10,6 +10,7 @@ from datetime import datetime, timezone from typing import Callable, Optional +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER from agent_env.config import get_config @@ -87,6 +88,7 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N """ from agent_env.artifact import DockerImageArtifact from agent_env.env.env import DeployedSandboxEnv, Env + from agent_env.env.envs._deployment import stack_layout_of from agent_env.env.envs.multi_env import MultiEnv, _gateway_state_provider from agent_env.env.store import get_env_instance_store @@ -135,16 +137,17 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N ) log("check_changelog", "Checking changelog...", 20) - is_clean = await _check_changelog_empty(multi_env._sandbox) + layout = stack_layout_of(multi_env._env_provider) + is_clean = await _check_changelog_empty(multi_env._sandbox, layout=layout) logger.info(f"Changelog empty: {is_clean} ({'clean' if is_clean else 'dirty'} snapshot)") log("check_changelog", f"Changelog: {'clean' if is_clean else 'dirty'}", 30) log("snapshot_db", "Exporting servicedb data...", 40) - tar_gz_object_url = await _snapshot_servicedb(multi_env._sandbox, env_id, universe_id, log) + tar_gz_object_url = await _snapshot_servicedb(multi_env._sandbox, env_id, universe_id, log, layout=layout) log("store_artifact", "Storing Docker image artifact...", 70) logger.info("Storing snapshot as DockerImageArtifact...") - image_tag = _image_tag(env_id, universe_id) + image_tag = _snapshot_image_tag(env_id, universe_id, layout) artifact = DockerImageArtifact.put_tar( id=derive_id(env_id, "env-snapshot"), description=( @@ -174,14 +177,19 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N return snapshot -async def _check_changelog_empty(sandbox) -> bool: +def _snapshot_image_tag(env_id: str, environment_universe_id: str, layout: StackLayout) -> str: + """The image a snapshot builds on the VM, named for the slot too so two slots snapshotting one env and universe don't collide.""" + tag = _image_tag(env_id, environment_universe_id) + return f"{tag}-{layout.compose_project}" if layout.is_slot else tag + + +async def _check_changelog_empty(sandbox, *, layout: StackLayout = DEFAULT_STACK_LAYOUT) -> bool: """Return True if the _changelog table is empty (no modifications since data load).""" from agent_env.env.envs.service_db import DB_NAME, DB_USER from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME - from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH cmd = ( - f"docker compose -f {DOCKER_COMPOSE_PATH} exec -T {DATABASE_SERVICE_NAME} " + f"{layout.compose_command} exec -T {DATABASE_SERVICE_NAME} " f'psql -U {DB_USER} -d {DB_NAME} -tAc "SELECT count(*) FROM public._changelog"' ) output = await sandbox.exec_script(cmd) @@ -189,16 +197,20 @@ async def _check_changelog_empty(sandbox) -> bool: return count == 0 -async def _snapshot_servicedb(sandbox, env_id: str, environment_universe_id: str, log: ProgressCallback) -> str: +async def _snapshot_servicedb( + sandbox, env_id: str, environment_universe_id: str, log: ProgressCallback, *, layout: StackLayout = DEFAULT_STACK_LAYOUT, +) -> str: """Export pgdata from sandbox servicedb, build Docker image on sandbox, upload it to the object store. Returns the object URL of the uploaded tar.gz. """ from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME - from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH + pgdata = layout.scratch_path("pgdata") + build_dir = layout.scratch_path("snapshot-build") + tarball = layout.scratch_path("snapshot-image.tar.gz") container_id_output = await sandbox.exec_script( - f"docker compose -f {DOCKER_COMPOSE_PATH} ps -a -q {DATABASE_SERVICE_NAME}" + f"{layout.compose_command} ps -a -q {DATABASE_SERVICE_NAME}" ) container_id = container_id_output.strip() if not container_id: @@ -206,34 +218,34 @@ async def _snapshot_servicedb(sandbox, env_id: str, environment_universe_id: str log("snapshot_db", "Copying pgdata from container...", 45) logger.info("Exporting pgdata from servicedb container...") - await sandbox.exec_script(f"docker cp {container_id}:/var/lib/postgresql/data /tmp/pgdata") + await sandbox.exec_script(f"docker cp {container_id}:/var/lib/postgresql/data {pgdata}") - image_tag = _image_tag(env_id, environment_universe_id) + image_tag = _snapshot_image_tag(env_id, environment_universe_id, layout) log("snapshot_db", f"Building Docker image on sandbox: {image_tag}...", 50) logger.info(f"Building Docker image on sandbox: {image_tag}...") await sandbox.exec_script( - "mkdir -p /tmp/snapshot-build && " - "mv /tmp/pgdata /tmp/snapshot-build/pgdata && " - "cat > /tmp/snapshot-build/Dockerfile << 'DEOF'\n" + f"mkdir -p {build_dir} && " + f"mv {pgdata} {build_dir}/pgdata && " + f"cat > {build_dir}/Dockerfile << 'DEOF'\n" "FROM public.ecr.aws/docker/library/postgres:16-alpine\n" "COPY pgdata /var/lib/postgresql/data\n" "RUN chown -R postgres:postgres /var/lib/postgresql/data\n" "DEOF" ) await sandbox.exec_script( - f"docker build --platform linux/amd64 -t {image_tag} /tmp/snapshot-build" + f"docker build --platform linux/amd64 -t {image_tag} {build_dir}" ) log("snapshot_db", "Saving Docker image...", 55) logger.info("Saving Docker image to tar.gz on sandbox...") - await sandbox.exec_script(f"docker save {image_tag} | gzip > /tmp/snapshot-image.tar.gz") + await sandbox.exec_script(f"docker save {image_tag} | gzip > {tarball}") object_store, object_url = _tarball_destination(env_id, environment_universe_id) log("snapshot_db", "Uploading Docker image...", 60) logger.info(f"Uploading Docker image to {object_url}...") - await upload_vm_file(sandbox, "/tmp/snapshot-image.tar.gz", object_store, object_url) + await upload_vm_file(sandbox, tarball, object_store, object_url) - await sandbox.exec_script(f"rm -rf /tmp/snapshot-build /tmp/snapshot-image.tar.gz && docker rmi {image_tag}") + await sandbox.exec_script(f"rm -rf {build_dir} {tarball} && docker rmi {image_tag}") return object_url diff --git a/src/agent_env/providers/env_providers/constants.py b/src/agent_env/providers/env_providers/constants.py index 7a6a00e97..301da78c9 100644 --- a/src/agent_env/providers/env_providers/constants.py +++ b/src/agent_env/providers/env_providers/constants.py @@ -1,4 +1,4 @@ -"""The gateway deployment's layout: its compose service name, VM app dir and compose path, and website names. +"""The gateway deployment's layout: its compose service name, VM app dir, and website names. Imports nothing from agent_env, so modules in the ``agent_env.env`` import chain can use these while ``agent_env.providers`` is still importing. @@ -15,4 +15,3 @@ AGENT_ENV_WEBSITE_HEALTH_PATH = "/api/health" AGENT_ENV_WEBSITE_BACKEND_SUFFIX = "website-backend" AGENT_ENV_WEBSITE_FRONTEND_SUFFIX = "website-frontend" -DOCKER_COMPOSE_PATH = f"{GATEWAY_APP_DIR}/docker-compose.yml" diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 2e993e8b8..bafe2b816 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -62,11 +62,10 @@ AGENT_ENV_WEBSITE_BACKEND_SUFFIX, AGENT_ENV_WEBSITE_FRONTEND_SUFFIX, AGENT_ENV_WEBSITE_HEALTH_PATH, - DOCKER_COMPOSE_PATH, - GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME, GATEWAY_SERVICE_NAMES, ) +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.env_providers.env_provider import ( DeployRefused, _SandboxEnvironmentProvider, @@ -293,7 +292,7 @@ async def store_exec(argv: list[str]) -> tuple[int, str, str]: async def store_exec(argv: list[str]) -> tuple[int, str, str]: return await self._sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, + "sudo", *self._stack_layout.compose_argv, "exec", "-T", store_name, *argv, ) @@ -709,6 +708,7 @@ async def deploy( gateway_mode=gateway_mode.value, env_state_instance_ids=deployed.env_state_instance_ids, control_token=deployed.control_token, + stack_layout=self._stack_layout.to_record(), ) async def create_gateway( @@ -823,6 +823,7 @@ async def _deploy_via_vm( website_configs = website_configs or [] website_images = website_images or [] sidecars = sidecars or [] + layout = self._stack_layout config = get_config() gateway_env = Env.get(gateway_env_id(config)) @@ -865,6 +866,19 @@ async def _deploy_via_vm( **vm_kwargs, ) sandbox = self._sandbox + + def publish(port: int) -> int: + return sandbox.host_port(layout.host_port(port)) + + def tunnel_url(port: int) -> str | None: + host_port = layout.host_port(port) + url = sandbox.tunnel_urls.get(host_port) + if url is None and layout.is_slot: + raise RuntimeError( + f"Sandbox {sandbox.sandbox_id} has no tunnel for slot port {host_port} (container port {port}); " + "a shared VM must be created exposing every slot's ports" + ) + return url await sandbox.load_docker_images(all_images) # Compose declares the local_state service by name, so "standing up" is just naming the host; @@ -898,7 +912,7 @@ async def _stand_up(_spec): sidecars=sidecars, state_provider=self._state_provider, state_instance=state_instance, - host_port=sandbox.host_port, + host_port=publish, host_ips=sandbox.host_ips, extra_hosts=sandbox.extra_hosts, mcp_server_name=mcp_server_name, @@ -909,16 +923,16 @@ async def _stand_up(_spec): # Write files to VM logger.info("Writing files to VM...") - await sandbox.exec_script(f"mkdir -p {GATEWAY_APP_DIR}", max_retries=2) + await sandbox.exec_script(f"mkdir -p {layout.app_dir}", max_retries=2) # Only local postgres needs its init script to be directly run by the gateway's servicedb # container; remote/external stores are already initialized upstream. if self._needs_local_postgres: init_script = local_store_spec.init_sql logger.info(f"Generated init-schemas.sql:\n{init_script}") - await self._write_file(sandbox, f"{GATEWAY_APP_DIR}/init-schemas.sql", init_script) + await self._write_file(sandbox, layout.init_schemas_file, init_script) - await self._write_file(sandbox, DOCKER_COMPOSE_PATH, compose_content) + await self._write_file(sandbox, layout.compose_file, compose_content) # Start MCP services with docker compose logger.info("Starting MCP services...") @@ -928,14 +942,14 @@ async def _stand_up(_spec): # Everything but the gateway, beside the children; then the gateway, which waits for every child it fronts. # Both run to the end before either's failure is raised, so no child is still starting when the deploy closes. up, started = await asyncio.gather(self._compose_up(sandbox, f"--scale {GATEWAY_SERVICE_NAME}=0"), - children(sandbox.tunnel_urls.get(gateway_port), control_token), + children(tunnel_url(gateway_port), control_token), return_exceptions=True) for result in (up, started): if isinstance(result, BaseException): raise result external = tuple(replace(child, mcp_url=sandbox.url_from_sandbox(child.mcp_url), base_url=sandbox.url_from_sandbox(child.base_url)) for child in started) - await self._write_file(sandbox, DOCKER_COMPOSE_PATH, compose(external_children=external)) + await self._write_file(sandbox, layout.compose_file, compose(external_children=external)) await self._compose_up(sandbox) # Wait for containers to start @@ -948,17 +962,16 @@ async def _stand_up(_spec): if exit_code != 0: raise RuntimeError(f"docker ps failed: {stderr}") - # Check gateway container is running - exit_code, stdout, stderr = await sandbox.exec_with_output( "sudo", "docker", "ps") - if GATEWAY_SERVICE_NAME not in stdout: + running, listing = await self._gateway_running(sandbox) + if not running: down = await self._down_services(sandbox) raise RuntimeError(f"Gateway container not running:\n{down}" if down else - f"Gateway container not running. docker ps: {stdout}") + f"Gateway container not running. docker ps: {listing}") # Wait for gateway to be fully ready (needs time to discover internal servers) # Host-side port: exec_with_output runs inside the VM on per-VM backends but on # the host for LocalSandbox, where the gateway is published elsewhere. - gateway_ready = await self._wait_for_gateway(sandbox, sandbox.host_port(gateway_port)) + gateway_ready = await self._wait_for_gateway(sandbox, publish(gateway_port)) # Print gateway logs for debugging logger.info("Gateway logs:") @@ -977,10 +990,10 @@ async def _stand_up(_spec): if not await self._log_service(sandbox, name): logger.warning(f"{name} container not found") - gateway_url = sandbox.tunnel_urls.get(gateway_port) - db_web_base_url = sandbox.tunnel_urls.get(DB_WEB_PORT) if PGWEB_SERVICE_NAME in sidecar_names else None + gateway_url = tunnel_url(gateway_port) + db_web_base_url = tunnel_url(DB_WEB_PORT) if PGWEB_SERVICE_NAME in sidecar_names else None db_web_url = f"{db_web_base_url}/" if db_web_base_url else None - db_mcp_base_url = sandbox.tunnel_urls.get(DB_MCP_PORT) if DB_MCP_SERVICE_NAME in sidecar_names else None + db_mcp_base_url = tunnel_url(DB_MCP_PORT) if DB_MCP_SERVICE_NAME in sidecar_names else None db_mcp_url = f"{db_mcp_base_url}/mcp" if db_mcp_base_url else None logger.info(f"pgweb URL: {db_web_url}") if db_mcp_url: @@ -1249,7 +1262,9 @@ async def _compose_up(self, sandbox: VmSandbox, args: str = "") -> None: """``docker compose up -d`` the stack, whose failure names each service that went down and ends with its output, which compose's own error leaves out.""" try: - await sandbox.exec_script(f"cd {GATEWAY_APP_DIR} && docker compose up -d {args}".rstrip(), max_retries=2) + layout = self._stack_layout + await sandbox.exec_script(f"cd {layout.app_dir} && {layout.compose_command_from_app_dir} up -d {args}".rstrip(), + max_retries=2) except RuntimeError as e: down = await self._down_services(sandbox) if down is None: @@ -1269,7 +1284,7 @@ async def _down_services(self, sandbox: VmSandbox) -> str | None: async def _read_down_services(self, sandbox: VmSandbox) -> str | None: try: exit_code, stdout, stderr = await sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, "ps", "-a", "--format", "json" + "sudo", *self._stack_layout.compose_argv, "ps", "-a", "--format", "json" ) if exit_code != 0: logger.warning(f"Can't list the stack's services: docker compose ps exited {exit_code}: {stderr.strip()[-500:]}") @@ -1322,7 +1337,7 @@ async def _get_container_id(self, sandbox: VmSandbox, compose_service: str) -> s exec transport failed (exit -1) is retried, then raised: it can't tell "no container" apart.""" for attempt in range(_TRAJECTORY_READ_ATTEMPTS): exit_code, stdout, stderr = await sandbox.exec_with_output( - "sudo", "docker", "compose", "-f", DOCKER_COMPOSE_PATH, "ps", "-a", "-q", compose_service + "sudo", *self._stack_layout.compose_argv, "ps", "-a", "-q", compose_service ) if exit_code != -1: break @@ -1332,6 +1347,22 @@ async def _get_container_id(self, sandbox: VmSandbox, compose_service: str) -> s container_id = stdout.strip() return container_id if container_id else None + async def _gateway_running(self, sandbox: VmSandbox) -> tuple[bool, str]: + """Whether this stack's gateway container is running, and the listing that says so. + + The default stack keeps today's plain ``docker ps`` check, which any gateway container on the VM satisfies. That is + safe because a VM runs either one default stack or only slot stacks, never both. A slot asks its own compose project + (``compose ps --status running -q``), since another slot's gateway would satisfy the plain check. + """ + layout = self._stack_layout + if not layout.is_slot: + _, stdout, _ = await sandbox.exec_with_output("sudo", "docker", "ps") + return GATEWAY_SERVICE_NAME in stdout, stdout + _, stdout, _ = await sandbox.exec_with_output( + "sudo", *layout.compose_argv, "ps", "--status", "running", "-q", GATEWAY_SERVICE_NAME + ) + return bool(stdout.strip()), stdout + async def _provision_sidecar(self, deploy: _ContainerDeploy, spec) -> Sandbox: logger.info(f"Provisioning {spec.name} from {spec.image}...") sb = await deploy.sandbox_provider.create_container( @@ -1469,12 +1500,20 @@ async def _deploy_gateway( attribution: Optional[Attribution] = None, children: "Callable[[Optional[str], Optional[str]], Awaitable[list[ExternalChild]]] | None" = None, website_browser: "MCPServerEnv | None | object" = _THE_RUNS, + stack_layout: StackLayout = DEFAULT_STACK_LAYOUT, ) -> DeployedGateway: """One gateway deploy on one (unchained) sandbox provider: a VM on one that creates VMs, else a container per server on its private network. ``website_browser`` runs among its MCP servers, on the network it shares with - the sites; by default the run's (``run_website_browser``) when there are websites, which only a VM runs.""" + the sites; by default the run's (``run_website_browser``) when there are websites, which only a VM runs. + + ``stack_layout`` places the stack in a VM; the provider keeps it for the commands it later runs on that stack. + """ from agent_env.providers.env_state import LocalPostgresStateProvider, build_state_provider + if stack_layout.is_slot and (existing_sandbox is None or not existing_sandbox.walls_verified): + raise ValueError(f"Can't deploy into slot {stack_layout.compose_project}: a slot needs an existing sandbox " + f"whose walls between slots are verified") + self._stack_layout = stack_layout # Every attempt starts from the caller's store: a failed attempt's close() cleared it. self._state_instance = state_instance self._state_provider = ( @@ -1533,9 +1572,11 @@ def _sandbox_ids(self, always_name_gateway: bool = False) -> dict[str, str | dic return sandbox_ids async def _reattach(self, env: Env, deployed: DeployedEnv) -> Sandbox: - """A record this provider wrote, reattached: a MultiEnv's through its own rebuild, any other env's as the base reattaches it.""" + """A record this provider wrote, reattached: a MultiEnv's through its own rebuild, any other env's as the base reattaches it. + Either way the provider takes back the stack layout the record kept, so its commands reach the same stack.""" from agent_env.env.envs.multi_env import MultiEnv + self._stack_layout = StackLayout.from_record(getattr(deployed, "stack_layout", None)) if isinstance(env, MultiEnv): return await self._reattach_multi_env(deployed) return await super()._reattach(env, deployed) diff --git a/src/agent_env/providers/env_providers/env_provider.py b/src/agent_env/providers/env_providers/env_provider.py index a6f4d1913..3b71b1f0b 100644 --- a/src/agent_env/providers/env_providers/env_provider.py +++ b/src/agent_env/providers/env_providers/env_provider.py @@ -14,6 +14,7 @@ from agent_env.env.env import DeployedEnv from agent_env.env.gateway.constants import WELL_KNOWN_PATH from agent_env.plugins import _registration +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.providers.sandbox_providers.sandbox import Sandbox from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER @@ -72,10 +73,17 @@ class _SandboxEnvironmentProvider(EnvironmentProvider): record_class: ClassVar[type[DeployedEnv]] = DeployedEnv # the class its records load back as + _stack_layout: StackLayout = DEFAULT_STACK_LAYOUT + def __init__(self): self._container_sandboxes: list[Sandbox] = [] self._environment_sandboxes: dict[str, Sandbox] = {} + @property + def stack_layout(self) -> StackLayout: + """Where the stack this provider deployed lives in its sandbox; the default until a deploy lays out a slot.""" + return self._stack_layout + def environment_sandbox(self, environment_name: str) -> Sandbox | None: return self._environment_sandboxes.get(environment_name) diff --git a/src/agent_env/providers/env_providers/stack_layout.py b/src/agent_env/providers/env_providers/stack_layout.py new file mode 100644 index 000000000..f3277ec36 --- /dev/null +++ b/src/agent_env/providers/env_providers/stack_layout.py @@ -0,0 +1,147 @@ +"""Where a gateway stack lives in its sandbox: the default single stack, or one slot of several sharing a VM. + +Imports nothing from agent_env but ``constants``, so modules in the ``agent_env.env`` import chain can use it while +``agent_env.providers`` is still importing. +""" + +from __future__ import annotations + +import posixpath +import re +from dataclasses import dataclass + +from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR + +SLOT_HOST_PORT_STRIDE = 1000 +MAX_PORT = 65535 +_COMPOSE_PROJECT_NAME = re.compile(r"[a-z0-9][a-z0-9_-]*") +_SHELL_SAFE_PATH = re.compile(r"/[A-Za-z0-9/_.-]*") +_DEFAULT_IMPLICIT_PROJECT = posixpath.basename(GATEWAY_APP_DIR) + + +def _check_shell_safe_path(field: str, path: str) -> None: + """Paths are interpolated unquoted into shell scripts, so only absolute paths of a plain charset pass.""" + if not _SHELL_SAFE_PATH.fullmatch(path) or ".." in path.split("/"): + raise ValueError(f"{field} {path!r} must be absolute, use only letters, digits, '/', '_', '.' or '-', and have no '..'") + + +@dataclass(frozen=True) +class StackLayout: + """Where one gateway stack lives in its sandbox: its compose dir and project, its block of host ports and its scratch files. + + The default is the one stack a sandbox has always held. A slot is one of several stacks sharing a VM, so each gets its + own dir, compose project, host ports and scratch names; container ports and service names are the same in every stack. + """ + + app_dir: str = GATEWAY_APP_DIR + compose_project: str | None = None + host_port_offset: int = 0 + scratch_prefix: str = "/tmp/" + + def __post_init__(self) -> None: + _check_shell_safe_path("app dir", self.app_dir) + _check_shell_safe_path("scratch prefix", self.scratch_prefix) + if self.compose_project is not None: + if not _COMPOSE_PROJECT_NAME.fullmatch(self.compose_project): + raise ValueError( + f"compose project {self.compose_project!r} must be lowercase letters, digits, '-' or '_', starting with a letter or digit" + ) + if self.compose_project == _DEFAULT_IMPLICIT_PROJECT: + raise ValueError( + f"compose project {self.compose_project!r} is the default stack's implicit project (the basename of {GATEWAY_APP_DIR})" + ) + if self.host_port_offset < 0 or self.host_port_offset % SLOT_HOST_PORT_STRIDE: + raise ValueError(f"host port offset {self.host_port_offset} must be a non-negative multiple of {SLOT_HOST_PORT_STRIDE}") + if self.compose_project is None and ( + self.app_dir != GATEWAY_APP_DIR or self.host_port_offset or self.scratch_prefix != "/tmp/" + ): + raise ValueError("a layout that differs from the default stack's must name its compose project") + + @property + def is_slot(self) -> bool: + """A slot names its compose project; only the default stack leaves it to compose.""" + return self.compose_project is not None + + @classmethod + def for_slot(cls, index: int) -> StackLayout: + """Slot ``index`` of a shared VM: compose project ``s{index}`` names its dir and scratch files, its host ports are the index's block. + + The project is the hub's slot index, so the two cannot disagree. It is unique only within one docker daemon, which + each Modal VM has to itself, so slots are for VMs, not local sandboxes sharing the host's daemon. Block 0 is the + default stack's: a VM holds the default stack or slots, never both, but a slot starting one stride up keeps a + mix-up from publishing on the default stack's ports. The gateway + (18765) publishes on ``18765 + (index + 1) * 1000``, so 45 is the highest usable index; ``host_port`` raises past it. + """ + if index < 0: + raise ValueError(f"slot index must be >= 0, got {index}") + project = f"s{index}" + return cls( + app_dir=f"{GATEWAY_APP_DIR}/{project}", + compose_project=project, + host_port_offset=(index + 1) * SLOT_HOST_PORT_STRIDE, + scratch_prefix=f"/tmp/{project}-", + ) + + @property + def compose_file(self) -> str: + return f"{self.app_dir}/docker-compose.yml" + + @property + def init_schemas_file(self) -> str: + return f"{self.app_dir}/init-schemas.sql" + + @property + def artifact_staging_dir(self) -> str: + return f"{self.app_dir}/_artifact_staging" + + @property + def compose_argv(self) -> tuple[str, ...]: + """``docker compose`` bound to this stack's file and, for a slot, its project.""" + project = ("-p", self.compose_project) if self.is_slot else () + return ("docker", "compose", "-f", self.compose_file, *project) + + @property + def compose_command(self) -> str: + return " ".join(self.compose_argv) + + @property + def compose_command_from_app_dir(self) -> str: + """``docker compose`` for a script that has changed into ``app_dir``, where compose finds this stack's file.""" + project = f" -p {self.compose_project}" if self.is_slot else "" + return f"docker compose{project}" + + def to_record(self) -> dict | None: + """What a deploy record keeps of this layout, so a reload finds the same stack; None for the default stack.""" + if not self.is_slot: + return None + return {"app_dir": self.app_dir, "compose_project": self.compose_project, + "host_port_offset": self.host_port_offset, "scratch_prefix": self.scratch_prefix} + + @classmethod + def from_record(cls, data: object) -> StackLayout: + """The layout a deploy record kept; the default stack for a record that kept none. Raises for one that isn't a + valid slot layout, so a damaged record never sends a slot's commands to the default stack.""" + if data is None: + return DEFAULT_STACK_LAYOUT + fields = ("app_dir", "compose_project", "host_port_offset", "scratch_prefix") + if not isinstance(data, dict) or set(data) != set(fields): + raise ValueError(f"a recorded stack layout must have exactly the fields {fields}, got {data!r}") + if not isinstance(data["host_port_offset"], int) or isinstance(data["host_port_offset"], bool): + raise ValueError(f"a recorded stack layout's host_port_offset must be an int, got {data['host_port_offset']!r}") + layout = cls(**data) + if not layout.is_slot: + raise ValueError(f"a recorded stack layout must be a slot's, got {data!r}") + return layout + + def scratch_path(self, name: str) -> str: + return f"{self.scratch_prefix}{name}" + + def host_port(self, port: int) -> int: + """The host port this stack publishes ``port`` on; raises if the shift leaves the port range.""" + shifted = port + self.host_port_offset + if shifted > MAX_PORT: + raise ValueError(f"port {port} shifted by {self.host_port_offset} is past {MAX_PORT}") + return shifted + + +DEFAULT_STACK_LAYOUT = StackLayout() diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index 6b46f29e1..b784d21d8 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -174,6 +174,12 @@ class Sandbox(ABC): # The host other containers on its provider's private network reach it at, ready for a URL; None when it's on none. private_host: str | None = None + @property + def walls_verified(self) -> bool: + """Whether the framework itself set up and tested the walls between slots on this sandbox, so stacks of + different runs may share it. False for every sandbox today: nothing sets up those walls yet.""" + return False + def url_from_sandbox(self, url: str) -> str: """``url``, of a service this process reaches, as this sandbox reaches it.""" return url diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 2c1c779b5..c5dd18489 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -41,7 +41,8 @@ ) from agentenv_protocol.computer.card import COMPUTER_RECORDING from agentenv_protocol.types import WELL_KNOWN_PATH -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH, GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME +from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.integration.env.envs.playwright_mcp_local_test import PROCESSES from agent_env.env.snapshot_store import compute_env_fingerprint @@ -1380,7 +1381,7 @@ async def assert_website_tools_via_step(client, gateway_url, tools, tool_names): browser = protocol_v1.find_child(result.environment_card, PLAYWRIGHT_MCP_ENVIRONMENT_NAME) assert browser is not None, result.environment_card.get("children_environments") sandbox = await LocalSandboxProvider().get_sandbox(result.sandbox_id) - compose = yaml.safe_load(await sandbox.exec_script(f"cat {DOCKER_COMPOSE_PATH}")) + compose = yaml.safe_load(await sandbox.exec_script(f"cat {DEFAULT_STACK_LAYOUT.compose_file}")) proxied = dict(item.split("=", 1) for item in compose["services"][GATEWAY_SERVICE_NAME]["environment"]) assert (f"mcp-{PLAYWRIGHT_MCP_ENVIRONMENT_NAME}=http://{PLAYWRIGHT_MCP_ENVIRONMENT_NAME}:18765" in proxied["REST_PROXY_URLS"].split(",")), proxied["REST_PROXY_URLS"] diff --git a/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py b/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py index 0a306b116..2a39b7ab7 100644 --- a/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py +++ b/tst/unit/env/envs/test_multi_env_snapshot_reload_name.py @@ -14,7 +14,7 @@ from agent_env.env.envs.service_db import ServiceDBConfig from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT from agent_env.providers import EnvironmentGatewayProvider, MCPServerConfig, WebsiteConfig -from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT from agent_env.providers.env_state import LocalPostgresStateProvider @@ -127,7 +127,7 @@ async def test_the_reload_writes_its_compose_as_one_argument(): assert "$(touch pwned) 'q' `x`" in _written(env) scripts = [call.args[0] for call in env._sandbox.exec_script.await_args_list] - assert any(argv[:2] == ["printf", "%s\\n"] and argv[3:] == [">", DOCKER_COMPOSE_PATH] + assert any(argv[:2] == ["printf", "%s\\n"] and argv[3:] == [">", DEFAULT_STACK_LAYOUT.compose_file] for argv in map(shlex.split, scripts)) assert not any("<<" in script for script in scripts) diff --git a/tst/unit/env/envs/test_stack_layout_of.py b/tst/unit/env/envs/test_stack_layout_of.py new file mode 100644 index 000000000..b642ae060 --- /dev/null +++ b/tst/unit/env/envs/test_stack_layout_of.py @@ -0,0 +1,71 @@ +"""An env reads its stack's layout from the built-in provider that deployed it, and a slot's artifact loads stage inside that slot.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from agent_env.env.envs._deployment import stack_layout_of +from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.env.envs.website import WebsiteEnv +from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider +from agent_env.providers.env_providers.env_provider import EnvironmentProvider +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout + + +class _PluginProvider(EnvironmentProvider): + type = "plugin_layout" + + async def deploy(self, env, sandbox_provider, **options): + raise NotImplementedError + + async def close(self): + pass + + +def _slot_provider(index: int) -> EnvironmentGatewayProvider: + provider = EnvironmentGatewayProvider() + provider._stack_layout = StackLayout.for_slot(index) + return provider + + +def _vm_env(provider) -> SimpleNamespace: + sandbox = MagicMock(mode="vm", load_object_file=AsyncMock(), docker_cp=AsyncMock(), exec_script=AsyncMock(return_value="")) + provider._get_container_id = AsyncMock(return_value="cid") + return SimpleNamespace(_sandbox=sandbox, _env_provider=provider, environment_name="crm") + + +def _scripts(env: SimpleNamespace) -> list[str]: + return [c.args[0] for c in env._sandbox.exec_script.await_args_list] + + +def test_a_builtin_provider_gives_the_layout_it_deployed(): + assert stack_layout_of(_slot_provider(2)) == StackLayout.for_slot(2) + assert stack_layout_of(EnvironmentGatewayProvider()) == DEFAULT_STACK_LAYOUT + + +@pytest.mark.parametrize("provider", [None, _PluginProvider()]) +def test_a_plugin_provider_or_none_gives_the_default_layout(provider): + assert stack_layout_of(provider) == DEFAULT_STACK_LAYOUT + + +@pytest.mark.asyncio +async def test_a_slot_mcp_server_stages_its_artifact_in_the_slot_dir(): + env = _vm_env(_slot_provider(1)) + await MCPServerEnv._copy_artifact_into_container(env, SimpleNamespace(object_url="s3://b/mail.json"), "mail.json") + + staged = env._sandbox.load_object_file.await_args.args[1] + assert staged.startswith("/app/s1/_artifact_staging/crm-") + assert "mkdir -p /app/s1/_artifact_staging" in _scripts(env) + + +@pytest.mark.asyncio +async def test_a_slot_website_copies_and_removes_its_artifact_through_the_slot_compose_project(): + env = _vm_env(_slot_provider(1)) + await WebsiteEnv._copy_artifact_into_container(env, SimpleNamespace(object_url="s3://b/shop.json"), "shop.json") + await WebsiteEnv._remove_from_container(env, "shop.json") + + compose_scripts = [s for s in _scripts(env) if s.startswith("docker compose")] + assert compose_scripts and all(s.startswith("docker compose -f /app/s1/docker-compose.yml -p s1 ") for s in compose_scripts) diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index ab9588196..d8df4242a 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -26,7 +26,8 @@ from pytest_socket import enable_socket from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.env.env import DeployedGatewayEnv, _mcp_url +from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, _mcp_url +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.service_db import SERVICE_DB_PORT @@ -3111,3 +3112,40 @@ def test_a_site_image_without_curl_passes_its_health_probe_once_its_health_path_ def test_a_site_answering_only_errors_fails_its_health_probe(tmp_path): assert _run_probe(tmp_path, {"/.well-known/agent-env.json": 500, "/api/health": 503}) != 0 + + +@pytest.mark.asyncio +async def test_a_reattach_takes_back_the_slot_layout_its_record_kept(): + sandboxes = {i: _FakeSandbox(i, 18765, f"https://{i}") for i in ("gw", "srv", "db", "pg")} + record = dataclasses.replace(_multi_record([]), stack_layout=StackLayout.for_slot(2).to_record()) + gp = EnvironmentGatewayProvider() + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(side_effect=sandboxes.get))): + await gp._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), record) + assert gp.stack_layout == StackLayout.for_slot(2) + + +@pytest.mark.asyncio +async def test_a_reattach_of_a_record_with_no_layout_uses_the_default_stack(): + sandboxes = {i: _FakeSandbox(i, 18765, f"https://{i}") for i in ("gw", "srv", "db", "pg")} + gp = EnvironmentGatewayProvider() + gp._stack_layout = StackLayout.for_slot(1) + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(side_effect=sandboxes.get))): + await gp._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), _multi_record([])) + assert gp.stack_layout is DEFAULT_STACK_LAYOUT + + +@pytest.mark.asyncio +async def test_a_reattach_of_a_damaged_layout_raises_before_touching_any_sandbox(): + get_sandbox = AsyncMock() + record = dataclasses.replace(_multi_record([]), stack_layout={"compose_project": "s1"}) + with patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=get_sandbox)), \ + pytest.raises(ValueError): + await EnvironmentGatewayProvider()._reattach(MultiEnv(id="crm-suite", version=2, mcp_server_envs=[_mcp("slack")]), record) + get_sandbox.assert_not_awaited() + + +def test_a_gateway_record_keeps_its_slot_layout_through_storage_and_an_older_record_has_none(): + record = dataclasses.replace(_multi_record([]), stack_layout=StackLayout.for_slot(4).to_record()) + assert DeployedEnv.from_dict(dataclasses.asdict(record)).stack_layout == StackLayout.for_slot(4).to_record() + older = {k: v for k, v in dataclasses.asdict(_multi_record([])).items() if k != "stack_layout"} + assert DeployedEnv.from_dict(older).stack_layout is None diff --git a/tst/unit/providers/env_providers/stack_layout_test.py b/tst/unit/providers/env_providers/stack_layout_test.py new file mode 100644 index 000000000..9a6ccf1f0 --- /dev/null +++ b/tst/unit/providers/env_providers/stack_layout_test.py @@ -0,0 +1,397 @@ +"""A gateway stack's layout in its sandbox: the default is the one stack a sandbox has always held, a slot is one of several.""" + +from __future__ import annotations + +import shlex +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agent_env.env import snapshot_store +from agent_env.env.envs.service_db import DB_MCP_PORT, DB_WEB_PORT, ServiceDBConfig +from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT +from agent_env.providers.env_providers.env_gateway_provider import ( + EnvironmentGatewayProvider, + MCPServerConfig, + SidecarConfig, +) +from agent_env.providers.env_providers.stack_layout import DEFAULT_STACK_LAYOUT, StackLayout + +SIDECAR_HOST_PORT = 9100 + + +class _RecordingVm: + """A VM sandbox that records every command and answers the ones a gateway deploy waits on. + + ``running`` holds the compose projects whose gateway is up, keyed ``None`` for the default (unnamed) project; ``docker ps`` + lists a gateway container for each of them, as it would on a VM shared by several stacks. + """ + + type = "modal_vm" + mode = "vm" + sandbox_id = "vm-1" + host_ips: tuple[str, ...] = () + extra_hosts: tuple[str, ...] = () + ON_THIS_MACHINE = False + + def __init__(self, exposed_ports: list[int], running: set[str | None], walls_verified: bool = False): + self.exposed_ports = exposed_ports + self.walls_verified = walls_verified + self.tunnel_urls = {port: f"https://tunnel-{port}" for port in exposed_ports} + self.running = running + self.calls: list[tuple] = [] + + def host_port(self, port: int) -> int: + return port + + async def load_docker_images(self, images) -> None: + self.calls.append(("load", tuple(i.image_name for i in images))) + + async def exec_script(self, script: str, *, max_retries: int = 0) -> str: + self.calls.append(("script", script)) + return "cid-servicedb" if " ps -a -q " in script else "" + + async def exec_with_output(self, *args: str) -> tuple[int, str, str]: + self.calls.append(("exec", args)) + if args[0] == "curl": + return 0, "200", "" + if args == ("sudo", "docker", "ps"): + return 0, "\n".join(f"{p or 'app'}-gateway-1" for p in self.running), "" + if "--status" in args: + project = args[args.index("-p") + 1] if "-p" in args else None + return 0, "cid-gateway" if project in self.running else "", "" + if args[-2:-1] == ("-q",): + return 0, f"cid-{args[-1]}", "" + return 0, "", "" + + +def _image(name: str) -> MagicMock: + image = MagicMock() + image.image_name = name + image.load_problem.return_value = None + return image + + +async def _deploy(running: set[str | None] = frozenset({None}), **layout) -> tuple[_RecordingVm, object, EnvironmentGatewayProvider]: + """One VM gateway deploy (an MCP server, local Postgres with pgweb and db-mcp, one sidecar) with every I/O faked.""" + vms: list[_RecordingVm] = [] + + async def create_vm(*, exposed_ports, **_): + vms.append(_RecordingVm(exposed_ports, set(running))) + return vms[-1] + + sandbox_provider = MagicMock(ON_THIS_MACHINE=False, SUPPORTED_SANDBOX_MODES=("vm",)) + sandbox_provider.create_vm = create_vm + gateway_env = MagicMock() + gateway_env.docker_image_artifact = _image("agent-gateway") + db_env = MagicMock() + db_env.to_config.return_value = ServiceDBConfig(db_web_image="pgweb", db_mcp_image="db-mcp") + db_env.db_docker_image_artifact = _image("postgres") + db_env.db_web_docker_image_artifact = _image("pgweb") + db_env.db_mcp_docker_image_artifact = _image("db-mcp") + config = MagicMock(default_gateway_env_id="gw-id", default_service_db_env_id="db-id") + + gp = EnvironmentGatewayProvider() + gp._wait_for_tunnel = AsyncMock(return_value={"name": "crm"}) + gp._refuse_tool_name_clashes = AsyncMock() + with patch("agent_env.env.env.Env.get", side_effect=lambda env_id, *a, **k: gateway_env if env_id == "gw-id" else db_env), \ + patch("agent_env.config.get_config", return_value=config), \ + patch("agent_env.providers.env_providers.env_gateway_provider.asyncio.sleep", AsyncMock()): + result = await gp._deploy_gateway( + sandbox_provider, + state_instance=None, + mcp_servers=[MCPServerConfig(image="mcp-slack", environment_name="slack")], + mcp_server_images=[_image("mcp-slack")], + sidecars=[SidecarConfig(environment_name="probe", image_artifact=_image("probe"), container_port=80, host_port=SIDECAR_HOST_PORT)], + mcp_server_name="crm", + **layout, + ) + return (vms[0] if vms else layout["existing_sandbox"]), result, gp + + +def _scripts(vm: _RecordingVm) -> list[str]: + return [c[1] for c in vm.calls if c[0] == "script"] + + +def _written(script: str) -> tuple[str, str] | None: + """The path and content of a ``_write_file`` script (``printf '%s\\n' > ``), else None.""" + argv = shlex.split(script) + if argv[:2] == ["printf", "%s\\n"] and argv[3:4] == [">"]: + return argv[4], argv[2] + return None + + +def _execs(vm: _RecordingVm) -> list[tuple]: + return [c[1] for c in vm.calls if c[0] == "exec" and c[1][0] != "curl"] + + +@pytest.mark.asyncio +async def test_default_deploy_runs_todays_commands_on_todays_paths_and_ports(): + vm, result, _ = await _deploy() + + assert vm.exposed_ports == [AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT] + scripts = _scripts(vm) + assert scripts[0] == "mkdir -p /app" + assert _written(scripts[1])[0] == "/app/init-schemas.sql" + path, compose = _written(scripts[2]) + assert path == "/app/docker-compose.yml" + assert f'"{AGENT_ENV_GATEWAY_MCP_PORT}:{AGENT_ENV_GATEWAY_MCP_PORT}"' in compose + assert f'"{SIDECAR_HOST_PORT}:80"' in compose + assert scripts[3:] == ["cd /app && docker compose up -d"] + execs = _execs(vm) + assert ("sudo", "docker", "ps") in execs + for service in ("gateway", "pgweb", "db-mcp"): + assert ("sudo", "docker", "compose", "-f", "/app/docker-compose.yml", "ps", "-a", "-q", service) in execs + assert not any("-p" in e or "--status" in e for e in execs) + assert result.gateway_url == f"https://tunnel-{AGENT_ENV_GATEWAY_MCP_PORT}" + assert result.db_web_url == f"https://tunnel-{DB_WEB_PORT}/" + assert result.db_mcp_url == f"https://tunnel-{DB_MCP_PORT}/mcp" + + +SLOT_1_PORTS = [p + 2000 for p in (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT, SIDECAR_HOST_PORT)] + + +def _walled_vm(running: set[str | None], ports: list[int] = SLOT_1_PORTS) -> _RecordingVm: + return _RecordingVm(ports, running, walls_verified=True) + + +@pytest.mark.asyncio +async def test_slot_deploy_uses_its_own_dir_project_and_host_ports(): + slot = StackLayout.for_slot(1) + vm = _walled_vm({None, "s1"}) + _, result, gp = await _deploy(stack_layout=slot, existing_sandbox=vm) + + scripts = _scripts(vm) + assert scripts[0] == "mkdir -p /app/s1" + assert _written(scripts[1])[0] == "/app/s1/init-schemas.sql" + path, compose = _written(scripts[2]) + assert path == "/app/s1/docker-compose.yml" + assert f'"{AGENT_ENV_GATEWAY_MCP_PORT + 2000}:{AGENT_ENV_GATEWAY_MCP_PORT}"' in compose + assert f'"{DB_WEB_PORT + 2000}:{DB_WEB_PORT}"' in compose + assert f'"{SIDECAR_HOST_PORT + 2000}:80"' in compose + assert f"MCP_PORT={AGENT_ENV_GATEWAY_MCP_PORT}" in compose + assert scripts[3:] == ["cd /app/s1 && docker compose -p s1 up -d"] + compose_ps = ("sudo", "docker", "compose", "-f", "/app/s1/docker-compose.yml", "-p", "s1", "ps") + assert (*compose_ps, "--status", "running", "-q", "gateway") in _execs(vm) + assert (*compose_ps, "-a", "-q", "gateway") in _execs(vm) + assert ("sudo", "docker", "ps") not in _execs(vm) + assert result.gateway_url == f"https://tunnel-{AGENT_ENV_GATEWAY_MCP_PORT + 2000}" + assert result.db_web_url == f"https://tunnel-{DB_WEB_PORT + 2000}/" + assert result.db_mcp_url == f"https://tunnel-{DB_MCP_PORT + 2000}/mcp" + assert gp.stack_layout == slot + + +@pytest.mark.asyncio +async def test_a_slot_is_never_deployed_on_a_new_vm(): + with pytest.raises(ValueError, match="walls between slots are verified"): + await _deploy(stack_layout=StackLayout.for_slot(1)) + + +@pytest.mark.asyncio +async def test_a_slot_is_never_deployed_on_a_vm_whose_walls_are_not_verified(): + vm = _RecordingVm(SLOT_1_PORTS, {"s1"}) + with pytest.raises(ValueError, match="walls between slots are verified"): + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) + assert vm.calls == [] + + +def test_no_sandbox_has_verified_walls_today(): + import importlib + import pkgutil + + import agent_env.providers.sandbox_providers as package + from agent_env.providers.sandbox_providers.sandbox import Sandbox + + for module in pkgutil.walk_packages(package.__path__, f"{package.__name__}.", onerror=lambda _: None): + try: + importlib.import_module(module.name) + except ImportError: + continue + subclasses, pending = [], [Sandbox] + while pending: + subclasses += (children := pending.pop().__subclasses__()) + pending += children + assert subclasses + assert [c.__name__ for c in subclasses if c.walls_verified is not Sandbox.walls_verified] == [] + + +def test_a_provider_subclass_that_skips_init_keeps_the_default_layout(): + class Skips(EnvironmentGatewayProvider): + def __init__(self): + pass + + assert Skips().stack_layout == DEFAULT_STACK_LAYOUT + + +@pytest.mark.asyncio +async def test_slot_deploy_is_not_satisfied_by_another_stacks_gateway(): + with pytest.raises(RuntimeError, match="Gateway container not running"): + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=_walled_vm({None, "s0"})) + + +@pytest.mark.asyncio +async def test_slot_deploy_on_a_vm_not_exposing_its_ports_names_the_missing_port(): + vm = _walled_vm({"s1"}, [AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT]) + with pytest.raises(RuntimeError, match=f"no tunnel for slot port {AGENT_ENV_GATEWAY_MCP_PORT + 2000}"): + await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) + + +@pytest.mark.asyncio +async def test_slot_commands_after_deploy_stay_in_the_slot(): + vm = _walled_vm({"s1"}) + _, _, gp = await _deploy(stack_layout=StackLayout.for_slot(1), existing_sandbox=vm) + gp._sandbox = vm + vm.calls.clear() + + await gp.read_trajectory(vm) + await gp.install_changelog_triggers("slack") + await gp._down_services(vm) + + slot_compose = ("sudo", "docker", "compose", "-f", "/app/s1/docker-compose.yml", "-p", "s1") + assert _execs(vm)[0] == (*slot_compose, "ps", "-a", "-q", "gateway") + assert _execs(vm)[-2][: len(slot_compose) + 3] == (*slot_compose, "exec", "-T", "servicedb") + assert _execs(vm)[-1] == (*slot_compose, "ps", "-a", "--format", "json") + + +@pytest.mark.asyncio +async def test_slot_snapshot_uses_its_own_compose_and_scratch_files(): + vm = _RecordingVm([], set()) + object_store = MagicMock() + object_store.signed_put_url.return_value = "https://put" + config = MagicMock(get_object_store_for=lambda _env_id: object_store, get_artifact_key_prefix=lambda: "") + with patch.object(snapshot_store, "get_config", return_value=config): + await snapshot_store._snapshot_servicedb(vm, "env1", "uni1", lambda *a: None, layout=StackLayout.for_slot(0)) + + scripts = _scripts(vm) + assert scripts[0] == "docker compose -f /app/s0/docker-compose.yml -p s0 ps -a -q servicedb" + assert all("/tmp/pgdata" not in s and "/tmp/snapshot-" not in s for s in scripts) + assert "/tmp/s0-pgdata" in scripts[1] and "/tmp/s0-snapshot-build" in scripts[2] + assert scripts[-1].startswith("rm -rf /tmp/s0-snapshot-build /tmp/s0-snapshot-image.tar.gz") + + +async def _snapshot_image_commands(layout: StackLayout) -> list[str]: + vm = _RecordingVm([], set()) + object_store = MagicMock() + object_store.signed_put_url.return_value = "https://put" + config = MagicMock(get_object_store_for=lambda _env_id: object_store, get_artifact_key_prefix=lambda: "") + with patch.object(snapshot_store, "get_config", return_value=config): + await snapshot_store._snapshot_servicedb(vm, "env1", "uni1", lambda *a: None, layout=layout) + return [s for s in _scripts(vm) if s.startswith(("docker build", "docker save")) or "docker rmi" in s] + + +@pytest.mark.asyncio +async def test_default_snapshot_image_tag_is_unchanged(): + build, save, rmi = await _snapshot_image_commands(DEFAULT_STACK_LAYOUT) + assert build == "docker build --platform linux/amd64 -t env-snapshot-env1-uni1 /tmp/snapshot-build" + assert save.startswith("docker save env-snapshot-env1-uni1 |") + assert rmi.endswith("docker rmi env-snapshot-env1-uni1") + + +@pytest.mark.asyncio +async def test_two_slots_snapshotting_one_env_and_universe_build_different_images(): + s0 = await _snapshot_image_commands(StackLayout.for_slot(0)) + s1 = await _snapshot_image_commands(StackLayout.for_slot(1)) + assert all("env-snapshot-env1-uni1-s0" in c for c in s0) + assert all("env-snapshot-env1-uni1-s1" in c for c in s1) + + +def test_default_layout_is_the_single_stack_paths_and_ports(): + assert DEFAULT_STACK_LAYOUT.compose_command == "docker compose -f /app/docker-compose.yml" + assert DEFAULT_STACK_LAYOUT.compose_command_from_app_dir == "docker compose" + assert DEFAULT_STACK_LAYOUT.artifact_staging_dir == "/app/_artifact_staging" + assert DEFAULT_STACK_LAYOUT.scratch_path("pgdata") == "/tmp/pgdata" + assert DEFAULT_STACK_LAYOUT.host_port(AGENT_ENV_GATEWAY_MCP_PORT) == AGENT_ENV_GATEWAY_MCP_PORT + + +def test_slots_get_disjoint_host_port_blocks_from_the_default_and_each_other(): + ports = (AGENT_ENV_GATEWAY_MCP_PORT, DB_WEB_PORT, DB_MCP_PORT) + layouts = [DEFAULT_STACK_LAYOUT, *(StackLayout.for_slot(i) for i in range(8))] + published = [layout.host_port(p) for layout in layouts for p in ports] + assert len(set(published)) == len(published) + + +def test_for_slot_derives_its_project_dir_scratch_and_ports_from_the_index(): + slot = StackLayout.for_slot(3) + assert slot.compose_project == "s3" + assert slot.compose_command == "docker compose -f /app/s3/docker-compose.yml -p s3" + assert slot.artifact_staging_dir == "/app/s3/_artifact_staging" + assert slot.scratch_path("pgdata") == "/tmp/s3-pgdata" + assert slot.host_port(AGENT_ENV_GATEWAY_MCP_PORT) == AGENT_ENV_GATEWAY_MCP_PORT + 4000 + + +def test_for_slot_rejects_a_negative_index(): + with pytest.raises(ValueError): + StackLayout.for_slot(-1) + + +def test_slot_45_is_the_highest_whose_gateway_port_fits(): + assert StackLayout.for_slot(45).host_port(AGENT_ENV_GATEWAY_MCP_PORT) <= 65535 + with pytest.raises(ValueError): + StackLayout.for_slot(46).host_port(AGENT_ENV_GATEWAY_MCP_PORT) + + +@pytest.mark.parametrize("project", ["", "S0", "-s0", "s 0", "s/0", "app"]) +def test_layout_rejects_a_bad_project_or_the_default_stacks_implicit_one(project): + with pytest.raises(ValueError): + StackLayout(app_dir="/app/x", compose_project=project) + + +@pytest.mark.parametrize("app_dir", ["app", "/app/../etc", "/app/x;rm -rf /", "/app/$(id)", "/app/a b", "/app/'x'"]) +def test_layout_rejects_an_app_dir_unsafe_in_a_shell_script(app_dir): + with pytest.raises(ValueError): + StackLayout(app_dir=app_dir) + + +@pytest.mark.parametrize("scratch_prefix", ["tmp/", "/tmp/$x-", "/tmp/a b-", "/tmp/x;", "/tmp/../"]) +def test_layout_rejects_a_scratch_prefix_unsafe_in_a_shell_script(scratch_prefix): + with pytest.raises(ValueError): + StackLayout(scratch_prefix=scratch_prefix) + + +@pytest.mark.parametrize("offset", [-1000, 1, 999]) +def test_layout_rejects_an_offset_off_the_slot_stride(offset): + with pytest.raises(ValueError): + StackLayout(host_port_offset=offset) + + +def test_host_port_rejects_a_shift_past_the_port_range(): + with pytest.raises(ValueError): + StackLayout.for_slot(60).host_port(AGENT_ENV_GATEWAY_MCP_PORT) + + +@pytest.mark.parametrize("changes", [ + {"app_dir": "/app/s0"}, {"host_port_offset": 1000}, {"scratch_prefix": "/tmp/s0-"}, +]) +def test_a_layout_that_differs_from_the_default_must_name_its_project(changes): + with pytest.raises(ValueError, match="must name its compose project"): + StackLayout(**changes) + + +def test_only_a_named_project_makes_a_slot(): + assert not DEFAULT_STACK_LAYOUT.is_slot + assert StackLayout.for_slot(0).is_slot + + +@pytest.mark.parametrize("index", [0, 3, 45]) +def test_a_slot_layout_comes_back_the_same_from_its_record(index): + layout = StackLayout.for_slot(index) + assert StackLayout.from_record(layout.to_record()) == layout + + +def test_the_default_stack_records_nothing_and_a_record_without_a_layout_is_the_default(): + assert DEFAULT_STACK_LAYOUT.to_record() is None + assert StackLayout.from_record(None) is DEFAULT_STACK_LAYOUT + + +@pytest.mark.parametrize("data", [ + "s1", + {"app_dir": "/app/s1", "compose_project": "s1", "host_port_offset": 2000}, + {**StackLayout.for_slot(1).to_record(), "extra": 1}, + {**StackLayout.for_slot(1).to_record(), "host_port_offset": True}, + {**StackLayout.for_slot(1).to_record(), "host_port_offset": "2000"}, + {**StackLayout.for_slot(1).to_record(), "app_dir": "/app/../etc"}, + {"app_dir": "/app", "compose_project": None, "host_port_offset": 0, "scratch_prefix": "/tmp/"}, +]) +def test_a_damaged_recorded_layout_is_refused_rather_than_read_as_the_default_stack(data): + with pytest.raises((ValueError, TypeError)): + StackLayout.from_record(data)