From ea10c7de0b1d8844b09579f9770ecaa05abd2376 Mon Sep 17 00:00:00 2001 From: Haoqing Wang <78337154+hqhq1025@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:02:13 +0000 Subject: [PATCH] fix(local): isolate native disk state and preserve startup ownership --- src/agent_env/bundle/run.py | 6 +- src/agent_env/computers/README.md | 4 + .../sandbox_providers/local_docker/disk.py | 6 +- .../local_docker/provider.py | 71 ++++--- tst/unit/bundle/run_test.py | 45 ++++- tst/unit/env/envs/test_disk_recipe.py | 180 +++++++++++++++++- .../sandbox_providers/local_disk_test.py | 26 +++ 7 files changed, 301 insertions(+), 37 deletions(-) create mode 100644 tst/unit/providers/sandbox_providers/local_disk_test.py diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 15707cbe6..6483fef3f 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -296,16 +296,18 @@ def prepare_bundle( played: bool = False, agent: str | None = None, solver_endpoint: bool = False, + emulate: bool = True, ) -> PreparedBundle: """Do what ``run_bundle`` with these arguments does before its first task runs, and stop there. ``played`` leaves out what each task's only prompted agent would deploy, for a caller that plays that agent itself (``EpisodeSpec``), so the agent needn't exist. With ``solver_endpoint`` the caller gives the solver's deploys their - model endpoint in each run's ``step_params``, so the config needn't set one. + model endpoint in each run's ``step_params``, so the config needn't set one. ``emulate`` controls foreign-arch + builds as it does for ``run_bundle``. It doesn't turn namespace routing on: the caller keeps it on for as long as it runs the tasks (``agent_env.store.routing.namespace_routing``), which suits a service that runs nothing else. Raises what ``run_bundle`` raises before its first task runs, except for a running event loop, which it doesn't need.""" - composing = _composing(sandbox) + composing = _composing(sandbox, emulate) return _prepare(root, tasks, evals, sandbox, id_root, agent, played, _progress(on_progress), _hint(id_root), composing, solver_endpoint=solver_endpoint) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 3c8387bec..3132de0d1 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -193,6 +193,10 @@ are, under a tag of its recipe, so another machine composing on it pulls it rath no ISO. A compose installs a base afresh once fewer than 7 days of its evaluation are left, from the ISO Microsoft's link serves then. Name it as `windows-11`, or by its id, `@local/agentenv-framework/computers/windows-11`. +Docker opens `/dev/kvm` for the QEMU launcher. The user running agent-env needs access to Docker, +but does not need read or write permission on that device. If Docker cannot pass the device to the launcher, +the container fails to start and the compose reports the error. + A Windows computer env is composed on the same machine as a thin disk (`env/envs/computer/utils/thin_disk.py`, with what's Windows' own in `windows_disk.py`): the base booted on an overlay kept in a folder of its own, each layer installed over ssh as `agentadmin` by a PowerShell script composing writes, as its `[install.windows]` says (its files diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py index 3450daaac..b8f4b318f 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/disk.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -32,9 +32,9 @@ def kvm_platforms() -> frozenset[str]: - """This machine's platform when its Docker can pass KVM to a container: /dev/kvm here, readable and writable; and - windows/amd64 on an amd64 machine, whose launcher boots a Windows guest too.""" - if not os.access(KVM, os.R_OK | os.W_OK): + """Disk platforms when this machine has a KVM device. Docker opens it for the launcher, so the client's user + need not have device access. A daemon that cannot pass the device fails when it starts the launcher.""" + if not Path(KVM).is_char_device(): return frozenset() here = host_platform() return frozenset({here, "windows/amd64"} if here == "linux/amd64" else {here}) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index f8b20301e..631a2fdc3 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -41,7 +41,7 @@ from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, forward, reset_input, unresettable from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker -from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_for +from agent_env.providers.sandbox_providers.local_docker.disk import KVM, kvm_platforms, launcher_args, machine_for from agent_env.providers.sandbox_providers.local_docker.relay import relay_image from agent_env.providers.sandbox_providers.sandbox import ( SANDBOX_LABEL, @@ -116,6 +116,8 @@ def trust_env(directory: str) -> dict[str, str]: # Marker dropped in the work dir when this sandbox runs a container, so a later get_sandbox() # (post-run teardown reconstructs the sandbox from disk) knows it owns that container. _CONTAINER_MODE_MARKER = ".agent-container-mode" +# A CLI failure can leave the daemon still creating this container; absence alone does not confirm its removal. +_CONTAINER_START_MARKER = ".container-start-pending" # The sandbox's container port -> host port map, so a sandbox rebuilt from its id publishes and finds the same ports. _PORT_MAP_FILE = ".port-map.json" # The internal network a container with no network runs on, so a sandbox rebuilt from its id removes it and its relay. @@ -364,9 +366,8 @@ async def terminate(self) -> None: this, local runs leak their containers/compose stacks, which squat host ports and block the next deploy. Prod backends override terminate() to tear the whole VM down. Processes its commands left running on this machine are stopped first (on Linux; see ``_marked_pids``), - then, once none is left to run from it, what its steps staged in /tmp is removed. With no Docker installed, - nothing of it can have been started; with Docker unreachable, the teardown fails only for a sandbox whose folder - records Docker objects of its own. + then, once none is left to run from it, what its steps staged in /tmp is removed. Unavailable Docker fails + teardown for a sandbox whose folder records Docker objects of its own, even if its CLI is now missing. """ failures: list[Exception] = [] try: @@ -377,19 +378,18 @@ async def terminate(self) -> None: failures.append(error) try: code, _, stderr = await self.exec_with_output("docker", "version", "--format", "{{.Server.Version}}") - except FileNotFoundError: - code, stderr = None, "" + except FileNotFoundError as error: + code, stderr = None, str(error) if code == 0: await self._remove_docker_objects(failures) - elif code is not None: + elif self._made_docker_objects: reason = stderr.strip()[-_ERROR_TAIL_CHARS:] - if self._made_docker_objects: - failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, " - f"compose stack or network: {reason}")) - else: - logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. " - "Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)", - reason, self.sandbox_id, SANDBOX_LABEL, self.sandbox_id) + failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, " + f"compose stack or network: {reason}")) + elif code is not None: + logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. " + "Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)", + stderr.strip()[-_ERROR_TAIL_CHARS:], self.sandbox_id, SANDBOX_LABEL, self.sandbox_id) if failures: raise RuntimeError("Local sandbox teardown incomplete: " + "; ".join(str(error) for error in failures)) from failures[0] @@ -419,7 +419,7 @@ async def _remove_docker_objects(self, failures: list[Exception]) -> None: failures.append(error) async def _remove_own_container(self) -> None: - """Remove this sandbox's named container, or verify that it is absent.""" + """Remove its named container; an unconfirmed start may still create one after an empty lookup.""" name = f"name=^{re.escape(self.container_name)}$" code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", name) if code != 0: @@ -427,6 +427,10 @@ async def _remove_own_container(self) -> None: f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}") if containers := stdout.split(): await self.exec_script(f"docker rm -f {shlex.join(containers)} >/dev/null") + (self._work_dir / _CONTAINER_START_MARKER).unlink(missing_ok=True) + elif (self._work_dir / _CONTAINER_START_MARKER).exists(): + raise RuntimeError(f"Container creation for sandbox {self.sandbox_id} is not confirmed; " + f"retain {self.work_dir} and retry teardown") async def _remove_labeled(self) -> None: """Remove labeled containers and networks; images can still be shared and are best-effort.""" @@ -806,11 +810,14 @@ async def create_vm( (sandbox.work_dir / _VM_SIZE_FILE).write_text(json.dumps({"cpu": cpu, "memory": memory})) try: await self._boot_disk(sandbox, image=image, env=env or {}) - (sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name) - except BaseException: - with contextlib.suppress(Exception): + except BaseException as error: + try: await sandbox.terminate() await asyncio.to_thread(remove_local_work_dir, sandbox.sandbox_id) + except Exception as cleanup_error: + note = f"Cleanup of sandbox {sandbox.sandbox_id} incomplete; retained {sandbox.work_dir}: {cleanup_error}" + error.add_note(note) + logger.warning("%s", note) raise return sandbox @@ -834,15 +841,29 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) for arg in ("-p", spec)] grants = await asyncio.to_thread(local_grants) - running = asyncio.ensure_future(asyncio.to_thread( - docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", - *publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), - *(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_QUERY_SECONDS)) + state = sandbox.work_dir / "vm-state" + state.mkdir(mode=0o700) + # The launcher owns its files as this user, so teardown needs no privileged recursive cleanup. + state_args = ("--mount", f"type=bind,src={state},dst=/var/lib/agentenv-vm", + "--user", f"{os.getuid()}:{os.getgid()}", "--group-add", str(os.stat(KVM).st_gid)) + (sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name) + pending = sandbox.work_dir / _CONTAINER_START_MARKER + pending.write_text(sandbox.container_name) + + def start() -> None: + docker("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + *publish, *machine.args, *state_args, *launcher_args(ports, env, size["cpu"], size["memory"]), + *(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_START_SECONDS) + pending.unlink(missing_ok=True) + + running = asyncio.ensure_future(asyncio.to_thread(start)) try: await asyncio.shield(running) - except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it - with contextlib.suppress(Exception): + except asyncio.CancelledError as cancelled: + try: await running + except Exception as error: + cancelled.add_note(f"Docker launcher failed during cancellation: {type(error).__name__}") raise async def create_container( @@ -1107,6 +1128,8 @@ def _forward_grants(container: str, port: int) -> None: _DOCKER_QUERY_SECONDS = 30 +# Starting a disk container also mounts its image; give creation its own bounded budget. +_DOCKER_START_SECONDS = 120 #: How long a reset may take: longer than the minute the launcher's reset waits for the computer to start again. _RESET_SECONDS = 90 #: Where a disk's launcher finds the local transfer CA's trust files. diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py index de2abdc4d..b76e27610 100644 --- a/tst/unit/bundle/run_test.py +++ b/tst/unit/bundle/run_test.py @@ -267,17 +267,54 @@ def test_the_model_and_sandbox_reach_each_run_and_one_invocation_shares_a_run_gr ] -def test_preparing_for_a_caller_that_plays_the_agent_needs_only_that_agent_to_exist_no_more(bundle_dir): +@pytest.mark.parametrize("options, expected_emulation", [({}, True), ({"emulate": False}, False)], + ids=["default", "native-only"]) +def test_preparing_passes_composition_options_and_keeps_selection_without_running_tasks( + bundle_dir, monkeypatch, options, expected_emulation): + compositions, progress = [], [] + write = run_module.materialize + provider = object() + providers = [] + + def selected_provider(name): + providers.append(name) + return provider + + def record_composition(plan, **hooks): + compositions.append((hooks["composing"], hooks["sandbox"])) + return write(plan, **hooks) + + monkeypatch.setattr(run_module, "build_sandbox_provider", selected_provider) + monkeypatch.setattr(run_module, "materialize", record_composition) + with namespace_routing(): + prepared = prepare_bundle( + bundle_dir, tasks=["unnamed"], evals=["smoke"], sandbox="local", + id_root="@local/prepared", on_progress=progress.append, **options) + + [(composing, sandbox)] = compositions + assert composing.emulate is expected_emulation + assert composing.forms_of == "deploys" + assert composing.sandbox_provider is None and composing.deploys_on is provider + assert sandbox == "local" and providers == ["local"] + assert [(entry.name, task.id, task.version) for entry, task in prepared.tasks] == [ + ("a", "@local/prepared/a", 1), ("b", "@local/prepared/b", 1), + ("unnamed", "@local/prepared/unnamed", 1)] + assert progress + assert _Scored.seen == [] + + +@pytest.mark.parametrize("options", [{}, {"emulate": False}], ids=["default", "native-only"]) +def test_preparing_for_a_caller_that_plays_the_agent_needs_only_that_agent_to_exist_no_more(bundle_dir, options): layout(bundle_dir, {"tasks/play.json": json.dumps(PLAYED)}) helper = {"id": "helper", "type": "deploy_agent", "agent_name": "helper", "a2a_agent_id": "nobody-else"} with namespace_routing(): with pytest.raises(BundleError, match="'nobody'"): - prepare_bundle(bundle_dir, tasks=["play"]) - prepared = prepare_bundle(bundle_dir, tasks=["play"], played=True) + prepare_bundle(bundle_dir, tasks=["play"], **options) + prepared = prepare_bundle(bundle_dir, tasks=["play"], played=True, **options) layout(bundle_dir, {"tasks/play.json": json.dumps([*PLAYED, helper])}) with pytest.raises(BundleError) as refused: - prepare_bundle(bundle_dir, tasks=["play"], played=True) + prepare_bundle(bundle_dir, tasks=["play"], played=True, **options) [(entry, task)] = prepared.tasks assert (entry.name, [step.type for step in task.steps]) == ( diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index 6d500448c..f5cf404d3 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -533,9 +533,12 @@ def _seed_environment(folder: Path, exported: dict[str, str] | None = None) -> t @pytest.fixture -def launched(monkeypatch): +def launched(monkeypatch, tmp_path): """The ``docker`` commands the local provider runs for a disk, with the launcher image it builds faked.""" ran = [] + device = tmp_path / "kvm" + device.touch() + monkeypatch.setattr(local_sandbox, "KVM", str(device)) monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(local_sandbox, "machine_for", AsyncMock(return_value=local_disk.Machine("agentenv-vm:0123"))) @@ -560,13 +563,54 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r first, reset = launched assert first == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", - "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, - 2, 4096), "agentenv-vm:0123") + "-p", "127.0.0.1:20001:18765", + "--mount", f"type=bind,src={sandbox.work_dir}/vm-state,dst=/var/lib/agentenv-vm", + "--user", f"{os.getuid()}:{os.getgid()}", "--group-add", str(os.getgid()), + *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, 2, 4096), "agentenv-vm:0123") assert reset == ("exec", "-i", sandbox.container_name, *RESET, "VM_ENV_ENVIRONMENT_NAME=YWdhaW4=\n", local_sandbox._RESET_SECONDS) assert local_sandbox._RESET_SECONDS > 60 LocalSandbox.pull_images.assert_awaited_once_with(["localhost:5000/disk@sha256:" + "a" * 64]) assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container + state = sandbox.work_dir / "vm-state" + assert state.is_dir() and state.stat().st_mode & 0o777 == 0o700 + + +@pytest.mark.asyncio +async def test_disks_keep_state_in_separate_sandbox_work_folders(launched): + provider = LocalSandboxProvider() + first = await provider.create_vm(image="disk", exposed_ports=[18765], env={}) + second = await provider.create_vm(image="disk", exposed_ports=[18765], env={}) + + assert first.work_dir != second.work_dir + for sandbox, command in zip((first, second), launched): + state = sandbox.work_dir / "vm-state" + assert str(state) in command[command.index("--mount") + 1] + assert state.stat().st_uid == os.getuid() + (state / "boot").mkdir() + (state / "boot" / "disk.qcow2").write_bytes(b"owned overlay") + local_sandbox.remove_local_work_dir(first.sandbox_id) + + assert not first.work_dir.exists() + assert (second.work_dir / "vm-state" / "boot" / "disk.qcow2").read_bytes() == b"owned overlay" + + +@pytest.mark.asyncio +async def test_a_disk_refuses_an_existing_state_symlink_before_starting(launched, tmp_path): + sandbox = LocalSandbox() + (sandbox.work_dir / local_sandbox._VM_SIZE_FILE).write_text('{"cpu":2,"memory":4096}') + other = tmp_path / "external" + other.mkdir() + (other / "keep").write_text("unrelated") + (sandbox.work_dir / "vm-state").symlink_to(other, target_is_directory=True) + + with pytest.raises(FileExistsError): + await LocalSandboxProvider()._boot_disk(sandbox, image="disk", env={}) + + assert launched == [] + assert not (sandbox.work_dir / local_sandbox._CONTAINER_MODE_MARKER).exists() + local_sandbox.remove_local_work_dir(sandbox.sandbox_id) + assert (other / "keep").read_text() == "unrelated" @pytest.mark.asyncio @@ -583,7 +627,10 @@ async def test_a_disk_gets_this_processs_grant_port_and_the_trust_and_a_reconnec run, forward = launched hosts = [arg for entry in local_sandbox._EXTRA_HOSTS for arg in ("--add-host", entry)] assert run == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", - "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {}, 2, 4096), *hosts, + "-p", "127.0.0.1:20001:18765", + "--mount", f"type=bind,src={sandbox.work_dir}/vm-state,dst=/var/lib/agentenv-vm", + "--user", f"{os.getuid()}:{os.getgid()}", "--group-add", str(os.getgid()), + *local_disk.launcher_args([18765], {}, 2, 4096), *hosts, "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust", "agentenv-vm:0123") assert forward == ("exec", "-d", sandbox.container_name, *disk_launcher.forward(52345)) @@ -642,6 +689,131 @@ def docker(*args, **options): assert order == ["run", "terminate"] +@pytest.mark.asyncio +async def test_a_disk_start_can_outlast_a_query_and_records_ownership_before_launching(launched, monkeypatch): + def docker(*args, timeout=None): + name = args[args.index("--name") + 1] + folder = LocalSandbox.find_work_dir(name.removeprefix("agent-")) + assert (folder / local_sandbox._CONTAINER_MODE_MARKER).read_text() == name + assert (folder / local_sandbox._CONTAINER_START_MARKER).exists() + if timeout < 45: + raise subprocess.TimeoutExpired(["docker", "run"], timeout) + return "" + + monkeypatch.setattr(local_sandbox, "docker", docker) + + sandbox = await LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={}) + + assert sandbox.owns_container + assert not (sandbox.work_dir / local_sandbox._CONTAINER_START_MARKER).exists() + assert local_sandbox._DOCKER_QUERY_SECONDS < 45 + + +@pytest.fixture +def uncertain_disk(launched, monkeypatch): + state = SimpleNamespace(visible=False, daemon="up", remove_error=None, folder=None, sandbox_id=None, + owned_at_start=False, lookups=[], removals=[], entered=threading.Event(), release=None) + + def docker(*args, timeout=None): + name = args[args.index("--name") + 1] + state.sandbox_id = name.removeprefix("agent-") + state.folder = LocalSandbox.find_work_dir(state.sandbox_id) + (state.folder / "vm-state" / "retained").write_text("pending disk state") + state.owned_at_start = (state.folder / local_sandbox._CONTAINER_MODE_MARKER).exists() + state.entered.set() + if state.release is not None: + assert state.release.wait(5) + raise subprocess.TimeoutExpired(["docker", "run"], timeout) + + async def exec_with_output(sandbox, *args): + if args[:2] == ("docker", "version"): + if state.daemon == "missing": + raise FileNotFoundError(2, "No such file or directory", "docker") + return (0, "29.1.4", "") if state.daemon == "up" else (1, "", "Cannot connect to the Docker daemon") + assert args[:2] == ("docker", "ps") + state.lookups.append(sandbox.container_name) + return 0, sandbox.container_name if state.visible else "", "" + + async def exec_script(sandbox, script, **kwargs): + state.removals.append(script) + if state.remove_error is not None: + raise state.remove_error + state.visible = False + return "" + + monkeypatch.setattr(local_sandbox, "docker", docker) + monkeypatch.setattr(local_sandbox, "_stop_marked", AsyncMock(return_value=True)) + monkeypatch.setattr(local_sandbox, "_remove_staged", lambda sandbox_id: None) + monkeypatch.setattr(LocalSandbox, "exec_with_output", exec_with_output) + monkeypatch.setattr(LocalSandbox, "exec_script", exec_script) + return state + + +@pytest.mark.asyncio +async def test_a_timed_out_create_keeps_ownership_until_a_late_container_is_removed(uncertain_disk): + provider = LocalSandboxProvider() + with pytest.raises(subprocess.TimeoutExpired) as failure: + await provider.create_vm(image="disk", exposed_ports=[18765], env={}) + + assert uncertain_disk.owned_at_start + assert uncertain_disk.folder.is_dir() + assert (uncertain_disk.folder / "vm-state" / "retained").read_text() == "pending disk state" + assert len(uncertain_disk.lookups) == 1 and uncertain_disk.removals == [] + assert any("creation" in note and "not confirmed" in note for note in failure.value.__notes__) + rebuilt = await provider.get_sandbox(uncertain_disk.sandbox_id) + assert rebuilt.owns_container + with pytest.raises(RuntimeError, match="creation.*not confirmed"): + await provider.terminate_sandbox(uncertain_disk.sandbox_id) + + uncertain_disk.visible = True + await provider.terminate_sandbox(uncertain_disk.sandbox_id) + + assert uncertain_disk.removals == [f"docker rm -f {rebuilt.container_name} >/dev/null"] + assert not (uncertain_disk.folder / local_sandbox._CONTAINER_START_MARKER).exists() + assert local_sandbox.remove_local_work_dir(uncertain_disk.sandbox_id) == uncertain_disk.folder + assert not uncertain_disk.folder.exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cleanup", ["unreachable", "missing-cli", "remove-failed"]) +async def test_a_start_failure_keeps_its_error_and_folder_when_cleanup_fails(uncertain_disk, cleanup, caplog): + if cleanup == "remove-failed": + uncertain_disk.visible = True + uncertain_disk.remove_error = RuntimeError("container removal failed") + else: + uncertain_disk.daemon = "missing" if cleanup == "missing-cli" else "down" + + with pytest.raises(subprocess.TimeoutExpired) as failure: + await LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={}) + + assert uncertain_disk.folder.is_dir() + assert (uncertain_disk.folder / "vm-state" / "retained").read_text() == "pending disk state" + assert (uncertain_disk.folder / local_sandbox._CONTAINER_START_MARKER).exists() + assert any("Cleanup of sandbox" in note for note in failure.value.__notes__) + assert "Cleanup of sandbox" in caplog.text + assert ("container removal failed" if cleanup == "remove-failed" else "Docker can't be reached") in caplog.text + + +@pytest.mark.asyncio +async def test_cancellation_during_a_start_that_times_out_preserves_the_unknown_creation(uncertain_disk): + uncertain_disk.release = threading.Event() + booting = asyncio.create_task(LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={})) + try: + assert await asyncio.to_thread(uncertain_disk.entered.wait, 5) + booting.cancel() + finally: + uncertain_disk.release.set() + + with pytest.raises(asyncio.CancelledError) as failure: + await booting + + assert uncertain_disk.folder.is_dir() + assert (uncertain_disk.folder / "vm-state" / "retained").read_text() == "pending disk state" + assert (uncertain_disk.folder / local_sandbox._CONTAINER_START_MARKER).exists() + assert any("TimeoutExpired" in note for note in failure.value.__notes__) + assert any("Cleanup of sandbox" in note for note in failure.value.__notes__) + + @pytest.mark.asyncio async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypatch): monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) diff --git a/tst/unit/providers/sandbox_providers/local_disk_test.py b/tst/unit/providers/sandbox_providers/local_disk_test.py new file mode 100644 index 000000000..fd18d3034 --- /dev/null +++ b/tst/unit/providers/sandbox_providers/local_disk_test.py @@ -0,0 +1,26 @@ +from pathlib import Path + +import pytest + +from agent_env.providers.sandbox_providers.local_docker import disk +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider + + +@pytest.mark.parametrize("platform,expected", [ + ("linux/amd64", frozenset({"linux/amd64", "windows/amd64"})), + ("linux/arm64", frozenset({"linux/arm64"})), +]) +def test_docker_disk_platforms_do_not_require_client_device_permissions(monkeypatch, platform, expected): + monkeypatch.setattr(Path, "is_char_device", lambda path: str(path) == disk.KVM) + monkeypatch.setattr(disk.os, "access", lambda *args: False) + monkeypatch.setattr(disk, "host_platform", lambda: platform) + assert LocalSandboxProvider.disk_image_platforms() == expected + + +@pytest.mark.parametrize("exists", [False, True]) +def test_missing_or_regular_kvm_path_does_not_advertise_disks(monkeypatch, tmp_path, exists): + device = tmp_path / "kvm" + if exists: + device.touch() + monkeypatch.setattr(disk, "KVM", str(device)) + assert LocalSandboxProvider.disk_image_platforms() == frozenset()