From fa436191e28aa43867d28c05775cf27596ebd49f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 12:51:12 -0800 Subject: [PATCH 1/4] feat(task): agent-env task teardown finishes the teardown a run left, which the run names A run whose teardown a second signal stopped, or that couldn't tear a sandbox down, listed what was still up and left it there: no command removed it, and a cancelled run's record didn't hold what a step the signal cut short had deployed. Such a run now keeps on its instance the context its teardown ended with (record_teardown), and agent-env run and task run --keep name agent-env task teardown --instance , which tears down what the instance records still up as the run's own teardown does, skipping what it took down, and exits 1 while any is still up. An instance recorded running is left alone unless --force, as for a run killed outright. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/run.py | 13 +++++- src/agent_env/cli/run.py | 3 ++ src/agent_env/cli/task/__init__.py | 2 + src/agent_env/cli/task/run.py | 14 +++++- src/agent_env/cli/task/teardown.py | 41 ++++++++++++++++++ src/agent_env/cli/teardown_output.py | 8 ++++ src/agent_env/task/store.py | 25 +++++++++++ tst/unit/bundle/run_teardown_test.py | 65 +++++++++++++++++++++++++++- 8 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 src/agent_env/cli/task/teardown.py diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 016cb6550..06a1419ef 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -30,7 +30,7 @@ from agent_env.providers import build_sandbox_provider from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing, run_scope -from agent_env.task.store import record_task_cancelled +from agent_env.task.store import record_task_cancelled, record_teardown from agent_env.task.interrupts import Interrupts from agent_env.utils.progress import run_progress from agent_env.task.task import Task, recorded_scores @@ -139,6 +139,7 @@ def teardown(self) -> BundleRun: with Interrupts() as interrupts: reports = interrupts.run(interrupts.stopping(teardown_run(run.context) for run in kept)) result = self._with_reports(kept, reports) + _record_left_up([replace(run, torn_down=report) for run, report in zip(kept, reports)]) if interrupts.count: raise RunInterrupted(result, interrupts.signum) return result @@ -270,6 +271,7 @@ def run_bundle( solver_params(plan, prepared.materialization), keep, say, interrupts, source)) _mark_cancelled(runs, interrupts.reason) + _record_left_up(runs) result = _bundle_run(plan, prepared.materialization, runs, _skipped(plan, entries, every=not tasks and not evals)) if not interrupts.count: @@ -382,6 +384,15 @@ def _mark_cancelled(runs: tuple[TaskRun, ...], reason: str) -> None: record_task_cancelled(run.instance_id, reason, Task._utc_now_str()) +def _record_left_up(runs: Sequence[TaskRun]) -> None: + """Keep on each run's instance the context its teardown ended with when it left sandboxes up, so ``agent-env task + teardown`` finds them, those a step a signal cut short had recorded included. Like ``_mark_cancelled``, it runs + once the loop has closed.""" + for run in runs: + if run.torn_down is not None and run.torn_down.still_up and run.instance_id: + record_teardown(run.instance_id, run.context) + + def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRun, ...], skipped: tuple[BundleEntry, ...]) -> BundleRun: by_id = {run.task.id: run for run in runs} diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 013809aaa..f7fc7f66e 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -16,6 +16,7 @@ from agent_env.bundle.plan import NotInStore from agent_env.bundle.requires import UnmetRequirements from agent_env.cli._install_prompt import offer_to_install +from agent_env.cli.teardown_output import echo_finish_teardown from agent_env.plugins._bundles import ALIAS_TABLES, installed_aliases from agent_env.plugins._discovery import discovery_error from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox @@ -364,6 +365,8 @@ def _report_teardown(result: BundleRun) -> None: click.echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow")) for sandbox in still_up: click.echo(f" {_still_up(sandbox)}") + echo_finish_teardown([run.instance_id for run, report in zip(result.runs, reports) + if run.instance_id and any(sandbox in still_up for sandbox in report.still_up)]) def _still_up(sandbox: RecordedSandbox) -> str: diff --git a/src/agent_env/cli/task/__init__.py b/src/agent_env/cli/task/__init__.py index 50066ffd3..41541af66 100644 --- a/src/agent_env/cli/task/__init__.py +++ b/src/agent_env/cli/task/__init__.py @@ -4,6 +4,7 @@ from .get import get from .get_instance import get_instance from .run import run, run_batch +from .teardown import teardown from .validate import validate @@ -18,4 +19,5 @@ def task(): task.add_command(get_instance) task.add_command(run) task.add_command(run_batch) +task.add_command(teardown) task.add_command(validate) diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index 41753d304..ddb4f5e5c 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -18,13 +18,13 @@ from agent_env.config.runtime import GATEWAY_OVERRIDE, SERVICE_DB_OVERRIDE, WEBSITE_BROWSER_OVERRIDE from agent_env.cli import submitted_runs from agent_env.cli.identity import get_agent_env_client_id -from agent_env.cli.teardown_output import echo_teardown +from agent_env.cli.teardown_output import echo_finish_teardown, echo_teardown from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox from agent_env.runner.runner import KEEP_KEY, RunStatus from agent_env.store.ids import derive_id, fs_safe, is_local_id, validate_local_id from agent_env.store.routing import run_scope from agent_env.task.interrupts import Interrupts -from agent_env.task.store import get_task_instance_store, new_instance_id +from agent_env.task.store import get_task_instance_store, new_instance_id, record_teardown from agent_env.task.teardown import TeardownReport, kind, teardown_run from agent_env.task_step.context import TaskStepContext, read_dual_keyed from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep @@ -293,9 +293,19 @@ def _tear_down_kept(kept) -> None: reports = stopper.run(stopper.stopping(teardown_run(context) for _, context, _ in kept)) except asyncio.CancelledError: click.echo(click.style("Stopped the teardown; what it hadn't reached is still up.", fg="yellow")) + _left_up([context for _, context, _ in kept]) return for (tag, _, _), report in zip(kept, reports): echo_teardown(tag, report) + _left_up([context for (_, context, _), report in zip(kept, reports) if report.still_up]) + + +def _left_up(contexts: list[TaskStepContext]) -> None: + """Keep on each instance what its teardown left up, and name the command that finishes it.""" + left = [context for context in contexts if context.instance_id] + for context in left: + record_teardown(context.instance_id, context) + echo_finish_teardown([context.instance_id for context in left]) def _settle_kept(tagged_contexts, interrupts: Interrupts, interrupted) -> None: diff --git a/src/agent_env/cli/task/teardown.py b/src/agent_env/cli/task/teardown.py new file mode 100644 index 000000000..4e7ac578a --- /dev/null +++ b/src/agent_env/cli/task/teardown.py @@ -0,0 +1,41 @@ +"""Tear down what a task run left up.""" + +import click + +from agent_env.cli.teardown_output import echo_teardown +from agent_env.store.base import NotFoundError +from agent_env.task.interrupts import Interrupts +from agent_env.task.store import get_task_instance_store, record_teardown +from agent_env.task.teardown import TeardownReport, teardown_run +from agent_env.task_step.context import TaskStepContext + + +@click.command("teardown") +@click.option("--instance", "instance_id", required=True, help="The task instance whose sandboxes to tear down.") +@click.option("--force", is_flag=True, + help="Tear down an instance recorded running too, as a run killed outright leaves it.") +def teardown(instance_id: str, force: bool): + """Tear down the sandboxes a task run left up, as its own teardown does: the recordings it left open end, each + env still up closes, and each sandbox is terminated by id, a local one's folder removed. For a run whose teardown + was stopped or failed, which `agent-env run` names. Ctrl-C stops it; it exits 1 while any is still up, and + running it again goes on from there.""" + try: + instance = get_task_instance_store().get(instance_id) + except NotFoundError: + raise click.ClickException(f"no task instance {instance_id!r}") from None + if instance.status == "running" and not force: + raise click.ClickException(f"instance {instance_id} is recorded running, so its run may still be using its " + "sandboxes; pass --force once it isn't, as for a run killed outright") + context = TaskStepContext.from_dict(instance.context or {}) + left = TeardownReport.skipped(context).left + if not left: + click.echo(f"Nothing instance {instance_id} recorded is still up.") + return + click.echo(f"Tearing down {len(left)} sandbox{'es' if len(left) != 1 else ''} of instance {instance_id} " + "(Ctrl-C to stop)") + with Interrupts() as stopper: + [report] = stopper.run(stopper.stopping([teardown_run(context)])) + record_teardown(instance_id, context) + echo_teardown("", report) + if report.still_up: + raise SystemExit(1) diff --git a/src/agent_env/cli/teardown_output.py b/src/agent_env/cli/teardown_output.py index 8bc2b4cb7..40f1ec59e 100644 --- a/src/agent_env/cli/teardown_output.py +++ b/src/agent_env/cli/teardown_output.py @@ -14,3 +14,11 @@ def echo_teardown(tag: str, report: TeardownReport) -> None: click.echo(click.style(f"{prefix}Couldn't tear down {sandbox.sandbox_id}: {why}", fg="red")) for sandbox in report.left: click.echo(click.style(f"{prefix}Still up: {sandbox.sandbox_id} ({kind(sandbox)})", fg="red")) + + +def echo_finish_teardown(instance_ids: list[str]) -> None: + """The command that finishes the teardown of each instance whose run left sandboxes up.""" + if instance_ids: + click.echo("Tear down what's left with:") + for instance_id in instance_ids: + click.echo(f" agent-env task teardown --instance {instance_id}") diff --git a/src/agent_env/task/store.py b/src/agent_env/task/store.py index 72a1b1fbb..11857b8d4 100644 --- a/src/agent_env/task/store.py +++ b/src/agent_env/task/store.py @@ -18,6 +18,7 @@ from agent_env.config import get_config from agent_env.store.document_store import DocumentStore, DuplicateKeyError, Filter, Lte, Ne, Sort, SortKey, UnknownDocumentTypeError, UpdateSpec, VersionedEntityStore, VersionedEntityStoreCache, compare_and_swap from agent_env.store.query import QueryBuilder, to_document_query +from agent_env.store.routing import run_scope from agent_env.task.step_journal import ( _RESERVED_STEP_IDS, _SCHEDULER_STEP_ID, @@ -791,6 +792,12 @@ def record_task_cancelled_sync( ), ) + def record_teardown_sync(self, instance_id: str, context: "TaskStepContext") -> None: + """Keep the context a teardown left when it didn't take everything down: what the run deployed, writes of a + step a signal cut short included, and what the teardown terminated, so a later one finds what's still up.""" + self._doc_store.update(TASK_INSTANCES_COLLECTION, Filter.of(instance_id=instance_id), + UpdateSpec(set={"context": context.to_safe_dict()}, inc={"rev": 1})) + def append_step_attempt_failure_sync(self, instance_id: str, failure: "StepAttemptFailure") -> None: """Best-effort append of one attempt's failure, designed not to conflict with the ``record_task_failure`` write. ``add_to_set`` creates the array on the first append and @@ -1000,6 +1007,24 @@ async def record_task_failure( +def record_teardown(instance_id: str, context: "TaskStepContext") -> None: + """Keep the context a teardown that left sandboxes up ended with (``record_teardown_sync``), where the run's + records go, retried like ``record_task_cancelled``, logging a write that keeps failing rather than raising.""" + store = get_task_instance_store() + last_exc: BaseException | None = None + for attempt in range(_FAILURE_WRITE_MAX_ATTEMPTS): + try: + with run_scope(instance_id): + store.record_teardown_sync(instance_id, context) + return + except Exception as exc: # noqa: BLE001 + last_exc = exc + if attempt + 1 < _FAILURE_WRITE_MAX_ATTEMPTS: + time.sleep(_FAILURE_WRITE_BACKOFF_SECONDS * (2**attempt)) + logger.warning("Failed to record what instance_id=%s left up after %d attempts", instance_id, + _FAILURE_WRITE_MAX_ATTEMPTS, exc_info=last_exc) + + def record_task_cancelled(instance_id: str, reason: str, completed_at_utc: str, attempt_epoch: int | None = None) -> None: """Write a cancelled run's terminal state, retried like ``record_task_failure``. It logs a write that keeps failing rather than raising. Synchronous: ``run_bundle`` calls it once its event loop has closed.""" diff --git a/tst/unit/bundle/run_teardown_test.py b/tst/unit/bundle/run_teardown_test.py index 40af715de..4a2002bc4 100644 --- a/tst/unit/bundle/run_teardown_test.py +++ b/tst/unit/bundle/run_teardown_test.py @@ -17,7 +17,7 @@ from agent_env.cli import cli from agent_env.config import get_config from agent_env.config.runtime import Config -from agent_env.store.document_store import Filter +from agent_env.store.document_store import Filter, UpdateSpec from agent_env.task import TaskInstanceStore, teardown from agent_env.env.env import DeployedGatewayEnv from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext @@ -287,7 +287,68 @@ def test_the_cli_prints_a_sandbox_it_couldnt_tear_down_as_still_up(bundle_dir, t assert result.exit_code == 0, result.output assert "[tasks/ok.json] couldn't tear down sb-ok: RuntimeError: stuck\n" in result.output - assert result.output.endswith("\nStill up, 1 sandbox:\n sb-ok fake\n") + assert result.output.endswith(f"\nStill up, 1 sandbox:\n sb-ok fake\nTear down what's left with:\n" + f" agent-env task teardown --instance {_instance(result.output, 'ok')}\n") + + +def _instance(output: str, name: str) -> str: + return re.search(rf"instance (@local/\S+/{name}-\w+)", output).group(1) + + +def _stored_context(instance_id: str) -> TaskStepContext: + return TaskStepContext.from_dict(local_store().find_one("task_instances", Filter.of(instance_id=instance_id))["context"]) + + +def test_a_teardown_a_second_signal_stopped_is_finished_by_the_command_the_run_names(bundle_dir, terminated, + quiet_logs): + layout(bundle_dir, {"tasks/a.json": _task(nap=30, send="SIGINT")}) + terminated.second_signal.add("sb-a") + + stopped = CliRunner().invoke(cli, ["run", str(bundle_dir)]) + instance = _instance(stopped.output, "a") + terminated.second_signal.clear() + finished = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + again = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + + assert stopped.exit_code == 130, stopped.output + assert stopped.output.endswith(f"Tear down what's left with:\n agent-env task teardown --instance {instance}\n") + assert (finished.exit_code, terminated) == (0, ["sb-a"]), finished.output + assert finished.output == f"Tearing down 1 sandbox of instance {instance} (Ctrl-C to stop)\nTore down 1 sandbox\n" + assert (again.exit_code, again.output) == (0, f"Nothing instance {instance} recorded is still up.\n") + + +def test_a_failed_teardown_is_recorded_on_the_instance_and_the_command_tries_again(bundle_dir, terminated, + quiet_logs): + layout(bundle_dir, {"tasks/ok.json": _task()}) + terminated.failing.add("sb-ok") + instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") + recorded = _stored_context(instance).metadata.get(teardown.TORN_DOWN_KEY) + + still_stuck = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + terminated.failing.clear() + finished = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + + assert recorded is None + assert still_stuck.exit_code == 1 and "Couldn't tear down sb-ok: RuntimeError: stuck" in still_stuck.output + assert (finished.exit_code, terminated) == (0, ["sb-ok"]), finished.output + assert _stored_context(instance).metadata[teardown.TORN_DOWN_KEY] == ["sb-ok"] + + +def test_the_command_leaves_a_run_recorded_running_alone_unless_forced_and_refuses_an_unknown_instance( + bundle_dir, terminated, quiet_logs): + layout(bundle_dir, {"tasks/ok.json": _task()}) + terminated.failing.add("sb-ok") + instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") + terminated.failing.clear() + local_store().update("task_instances", Filter.of(instance_id=instance), UpdateSpec(set={"status": "running"})) + + refused = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + forced = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance, "--force"]) + unknown = CliRunner().invoke(cli, ["task", "teardown", "--instance", "@local/~/nope/t-12345678"]) + + assert refused.exit_code == 1 and "is recorded running" in refused.output and "--force" in refused.output + assert (forced.exit_code, terminated) == (0, ["sb-ok"]), forced.output + assert unknown.exit_code == 1 and "no task instance '@local/~/nope/t-12345678'" in unknown.output def test_the_cli_keep_prints_the_endpoints_holds_and_tears_down_on_ctrl_c(bundle_dir, terminated, quiet_logs, From cc5716b1200c49f89e2cff006c09fcdbc0838fa1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 12:54:33 -0800 Subject: [PATCH 2/4] test(task): the task run --keep fake sets its instance id as Task.run does; AGENTS.md names agent-env task teardown Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- tst/unit/cli/task_run_teardown_test.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 02995ff96..0d4113079 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ or one of them only where it is used. bridge address say: `host.docker.internal`, which grant URLs name by default, resolves only in containers. - Local sandbox teardown reports a failure when it cannot list or remove its own Docker containers or networks, or when marked Linux processes survive. The run keeps the sandbox work folder so teardown - can be retried. With no Docker installed there is nothing to remove; with Docker unreachable, it fails only + can be retried (`agent-env task teardown --instance `). With no Docker installed there is nothing to remove; with Docker unreachable, it fails only for a sandbox whose folder records its own container, compose stack or closed network. - `[agents] default_a2a_agent_id`: the agent a `deploy_agent` step without an id deploys; built-in `@local/agentenv-framework/agents/claude-code`, the `agents` bundle's Claude Code @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). A verifier on the `VerifierTaskStep` base (`verifiers/verifier.py`; `agent_prompt_response_verifier` so far, the other built-ins still write their entries themselves) implements `verify` and returns a `Verification`; the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings` with `scoring_rules`) beside the legacy `verifications` entries every reader still takes, and an exception from `verify` is a failed grade raised again. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). A verifier on the `VerifierTaskStep` base (`verifiers/verifier.py`; `agent_prompt_response_verifier` so far, the other built-ins still write their entries themselves) implements `verify` and returns a `Verification`; the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings` with `scoring_rules`) beside the legacy `verifications` entries every reader still takes, and an exception from `verify` is a failed grade raised again. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/tst/unit/cli/task_run_teardown_test.py b/tst/unit/cli/task_run_teardown_test.py index a1bb4b505..ccefe9f69 100644 --- a/tst/unit/cli/task_run_teardown_test.py +++ b/tst/unit/cli/task_run_teardown_test.py @@ -6,6 +6,7 @@ import importlib import logging import os +import re import signal import threading import types @@ -18,7 +19,7 @@ from agent_env.runner import store as run_store from agent_env.runner.runner import RunStatus from agent_env.task import Task -from agent_env.task.store import register_task_instance, seed_task_instance_context +from agent_env.task.store import get_task_instance_store, register_task_instance, seed_task_instance_context from agent_env.task.teardown import TORN_DOWN_KEY, RecordedSandbox, TeardownReport from agent_env.task_step.context import DeployedSandbox @@ -39,6 +40,7 @@ async def run(self, context, instance_id=None, **_): context.deployed_sandboxes.append(DeployedSandbox( sandbox_name="box", sandbox_id=f"sb-{len(self.contexts)}", sandbox_mode="vm", sandbox_type="fake")) if instance_id: + context.instance_id = instance_id register_task_instance("t", 1, 1, instance_id=instance_id) seed_task_instance_context(instance_id, context) await asyncio.sleep(self.nap) @@ -123,6 +125,23 @@ def test_keep_holds_a_run_up_until_ctrl_c_then_tears_it_down(monkeypatch, tmp_pa assert _ids(torn_down) == ["sb-1"] +def test_keep_names_the_command_that_finishes_a_teardown_that_left_a_sandbox_up_and_keeps_its_context( + monkeypatch, tmp_path, torn_down): + async def stuck(context): + context.metadata["teardown_tried"] = True + sandbox = context.deployed_sandboxes[0] + return TeardownReport(failed=((RecordedSandbox(sandbox.sandbox_id, sandbox.sandbox_type, "sandbox"), "stuck"),)) + + monkeypatch.setattr(task_run, "teardown_run", stuck) + + result = _invoke(monkeypatch, tmp_path, _Deploys(), "run", "--keep", signal_after=1.0) + + instance = re.search(r"Tear down what's left with:\n agent-env task teardown --instance (\S+)\n", + result.output).group(1) + assert "Couldn't tear down sb-1: stuck" in result.output + assert get_task_instance_store().get(instance).context["metadata"]["teardown_tried"] is True + + def test_keep_still_tears_down_a_run_ctrl_c_cancels(monkeypatch, tmp_path, torn_down): task = _Deploys(nap=30) From 1a66d8046af2d32342bb86bc54da85e264f3226a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:14:14 -0800 Subject: [PATCH 3/4] fix(task): every teardown is recorded, fenced by its attempt, and the command counts what the run lists Greptile and an independent review found four gaps: - The command tore down outside the run's namespace, so a recording it ended or registered used the configured stores. It now runs in the instance's run scope. - A record written as a teardown ended could overwrite an attempt that took the instance over meanwhile. It's now fenced by the instance's attempt epoch as the teardown began. - A sandbox terminated whose folder wasn't removed was marked torn down, so the command found nothing to do. A run now keeps what its teardown left up (left_up_sandbox_ids), and the command takes those too. - Only a teardown that left something up was recorded, so a finished one still listed its sandboxes up. Every teardown is recorded now, and a local sandbox whose folder is gone counts as gone, as agent-env run lists it. The command also keeps its record inside its signal handling, and BundleRun.teardown routes namespaces, as run_bundle does. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/run.py | 40 ++++++++---- src/agent_env/cli/run.py | 5 +- src/agent_env/cli/task/run.py | 30 ++++----- src/agent_env/cli/task/teardown.py | 25 +++++--- src/agent_env/cli/teardown_output.py | 9 ++- src/agent_env/task/store.py | 29 ++------- src/agent_env/task/teardown.py | 37 +++++++++++ tst/unit/bundle/run_teardown_test.py | 96 +++++++++++++++++++++++++++- 8 files changed, 205 insertions(+), 66 deletions(-) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 06a1419ef..49f22bb93 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -30,11 +30,11 @@ from agent_env.providers import build_sandbox_provider from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing, run_scope -from agent_env.task.store import record_task_cancelled, record_teardown +from agent_env.task.store import record_task_cancelled from agent_env.task.interrupts import Interrupts from agent_env.utils.progress import run_progress from agent_env.task.task import Task, recorded_scores -from agent_env.task.teardown import TeardownReport, teardown_run +from agent_env.task.teardown import TeardownReport, attempt_epoch, record_teardown, teardown_run from agent_env.task_step.context import TaskStepContext, sandbox_overrides from .hub import SOURCE_KEY, HubSource @@ -73,6 +73,7 @@ class TaskRun: cancelled: bool = False started: bool = True # False when it was cancelled before it started torn_down: TeardownReport | None = None # None while its sandboxes are kept up + attempt_epoch: int | None = None # its instance's as its teardown began, which ``record_teardown`` is fenced by @property def instance_id(self) -> str | None: @@ -136,10 +137,10 @@ def teardown(self) -> BundleRun: kept = [run for run in self.runs if run.torn_down is None] if not kept: return self - with Interrupts() as interrupts: + with namespace_routing(), Interrupts() as interrupts: reports = interrupts.run(interrupts.stopping(teardown_run(run.context) for run in kept)) result = self._with_reports(kept, reports) - _record_left_up([replace(run, torn_down=report) for run, report in zip(kept, reports)]) + _record_teardowns([replace(run, torn_down=report) for run, report in zip(kept, reports)]) if interrupts.count: raise RunInterrupted(result, interrupts.signum) return result @@ -271,7 +272,7 @@ def run_bundle( solver_params(plan, prepared.materialization), keep, say, interrupts, source)) _mark_cancelled(runs, interrupts.reason) - _record_left_up(runs) + _record_teardowns(runs) result = _bundle_run(plan, prepared.materialization, runs, _skipped(plan, entries, every=not tasks and not evals)) if not interrupts.count: @@ -384,13 +385,25 @@ def _mark_cancelled(runs: tuple[TaskRun, ...], reason: str) -> None: record_task_cancelled(run.instance_id, reason, Task._utc_now_str()) -def _record_left_up(runs: Sequence[TaskRun]) -> None: - """Keep on each run's instance the context its teardown ended with when it left sandboxes up, so ``agent-env task - teardown`` finds them, those a step a signal cut short had recorded included. Like ``_mark_cancelled``, it runs - once the loop has closed.""" +def _record_teardowns(runs: Sequence[TaskRun]) -> None: + """Keep on each run's instance the context its teardown ended with, so ``agent-env task teardown`` finds what it + left up, those a step a signal cut short had recorded included, and nothing once all is down. Like + ``_mark_cancelled``, it runs once the loop has closed.""" for run in runs: - if run.torn_down is not None and run.torn_down.still_up and run.instance_id: - record_teardown(run.instance_id, run.context) + if run.torn_down is not None and run.instance_id and run.attempt_epoch is not None: + record_teardown(run.instance_id, run.context, run.torn_down, run.attempt_epoch) + + +def _epoch(instance_id: str | None) -> int | None: + """``instance_id``'s attempt epoch as its teardown begins, read synchronously so no signal cuts it short; None + when there's no instance or it can't be read.""" + if instance_id is None: + return None + try: + return attempt_epoch(instance_id) + except Exception as e: # noqa: BLE001 + logger.warning("Couldn't read instance %s before its teardown: %s", instance_id, e) + return None def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRun, ...], @@ -566,7 +579,7 @@ async def run_one(entry: BundleEntry, task: Task) -> TaskRun: me = asyncio.current_task() running.add(me) began = time.monotonic() - error, cancelled = None, False + error, cancelled, epoch = None, False, None try: try: with run_progress(lambda line: say(f"{tag} {line}")): @@ -585,11 +598,12 @@ async def run_one(entry: BundleEntry, task: Task) -> TaskRun: raise cancelled = True duration = time.monotonic() - began + epoch = _epoch(context.instance_id) report = await settle(context, cancelled) finally: running.discard(me) semaphore.release() - run = TaskRun(entry, task, context, error, duration, cancelled=cancelled, torn_down=report) + run = TaskRun(entry, task, context, error, duration, cancelled=cancelled, torn_down=report, attempt_epoch=epoch) say(f"{tag} {run.outcome.value} in {run.duration:.1f}s") for sandbox_record, why in (report.failed if report else ()): say(f"{tag} couldn't tear down {sandbox_record.sandbox_id}: {why}") diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index f7fc7f66e..fd1182662 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -16,7 +16,7 @@ from agent_env.bundle.plan import NotInStore from agent_env.bundle.requires import UnmetRequirements from agent_env.cli._install_prompt import offer_to_install -from agent_env.cli.teardown_output import echo_finish_teardown +from agent_env.cli.teardown_output import echo_finish_teardown, gone from agent_env.plugins._bundles import ALIAS_TABLES, installed_aliases from agent_env.plugins._discovery import discovery_error from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox @@ -355,8 +355,7 @@ def _records(context: TaskStepContext) -> list[tuple[str, object, list[tuple[str def _report_teardown(result: BundleRun) -> None: reports = [run.torn_down or TeardownReport.skipped(run.context) for run in result.runs] terminated = sum(len(report.terminated) for report in reports) - still_up = [sandbox for report in reports for sandbox in report.still_up - if sandbox.sandbox_type != "local" or LocalSandbox.find_work_dir(sandbox.sandbox_id)] + still_up = [sandbox for report in reports for sandbox in report.still_up if not gone(sandbox)] if terminated or still_up: click.echo() if terminated: diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index ddb4f5e5c..6e12140bb 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -18,14 +18,14 @@ from agent_env.config.runtime import GATEWAY_OVERRIDE, SERVICE_DB_OVERRIDE, WEBSITE_BROWSER_OVERRIDE from agent_env.cli import submitted_runs from agent_env.cli.identity import get_agent_env_client_id -from agent_env.cli.teardown_output import echo_finish_teardown, echo_teardown +from agent_env.cli.teardown_output import echo_finish_teardown, echo_teardown, gone from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox from agent_env.runner.runner import KEEP_KEY, RunStatus from agent_env.store.ids import derive_id, fs_safe, is_local_id, validate_local_id from agent_env.store.routing import run_scope from agent_env.task.interrupts import Interrupts -from agent_env.task.store import get_task_instance_store, new_instance_id, record_teardown -from agent_env.task.teardown import TeardownReport, kind, teardown_run +from agent_env.task.store import get_task_instance_store, new_instance_id +from agent_env.task.teardown import TeardownReport, attempt_epoch, kind, record_teardown, teardown_run from agent_env.task_step.context import TaskStepContext, read_dual_keyed from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.context_ops import build_context_update_ops @@ -288,24 +288,22 @@ def _tear_down_kept(kept) -> None: """Tear down what the runs kept; a Ctrl-C stops it. Counts signals afresh, so the one that got here doesn't.""" count = sum(len(left) for _, _, left in kept) click.echo(f"\nTearing down {count} {'sandbox' if count == 1 else 'sandboxes'} (Ctrl-C again to stop now)") + epochs = [attempt_epoch(context.instance_id) if context.instance_id else None for _, context, _ in kept] with Interrupts() as stopper: try: reports = stopper.run(stopper.stopping(teardown_run(context) for _, context, _ in kept)) except asyncio.CancelledError: click.echo(click.style("Stopped the teardown; what it hadn't reached is still up.", fg="yellow")) - _left_up([context for _, context, _ in kept]) - return - for (tag, _, _), report in zip(kept, reports): - echo_teardown(tag, report) - _left_up([context for (_, context, _), report in zip(kept, reports) if report.still_up]) - - -def _left_up(contexts: list[TaskStepContext]) -> None: - """Keep on each instance what its teardown left up, and name the command that finishes it.""" - left = [context for context in contexts if context.instance_id] - for context in left: - record_teardown(context.instance_id, context) - echo_finish_teardown([context.instance_id for context in left]) + reports = [TeardownReport.skipped(context) for _, context, _ in kept] + else: + for (tag, _, _), report in zip(kept, reports): + echo_teardown(tag, report) + contexts = [context for _, context, _ in kept] + for context, report, epoch in zip(contexts, reports, epochs): + if epoch is not None: + record_teardown(context.instance_id, context, report, epoch) + echo_finish_teardown([context.instance_id for context, report in zip(contexts, reports) + if context.instance_id and any(not gone(sandbox) for sandbox in report.still_up)]) def _settle_kept(tagged_contexts, interrupts: Interrupts, interrupted) -> None: diff --git a/src/agent_env/cli/task/teardown.py b/src/agent_env/cli/task/teardown.py index 4e7ac578a..5e3eba32f 100644 --- a/src/agent_env/cli/task/teardown.py +++ b/src/agent_env/cli/task/teardown.py @@ -1,12 +1,16 @@ """Tear down what a task run left up.""" +import asyncio +from dataclasses import replace + import click -from agent_env.cli.teardown_output import echo_teardown +from agent_env.cli.teardown_output import echo_teardown, gone from agent_env.store.base import NotFoundError +from agent_env.store.routing import run_scope from agent_env.task.interrupts import Interrupts -from agent_env.task.store import get_task_instance_store, record_teardown -from agent_env.task.teardown import TeardownReport, teardown_run +from agent_env.task.store import get_task_instance_store +from agent_env.task.teardown import TORN_DOWN_KEY, TeardownReport, record_teardown, recorded_still_up, teardown_run from agent_env.task_step.context import TaskStepContext @@ -27,15 +31,20 @@ def teardown(instance_id: str, force: bool): raise click.ClickException(f"instance {instance_id} is recorded running, so its run may still be using its " "sandboxes; pass --force once it isn't, as for a run killed outright") context = TaskStepContext.from_dict(instance.context or {}) - left = TeardownReport.skipped(context).left + context.metadata[TORN_DOWN_KEY] = [*context.metadata.get(TORN_DOWN_KEY, []), + *(sandbox.sandbox_id for sandbox in recorded_still_up(context) if gone(sandbox))] + left = recorded_still_up(context) if not left: click.echo(f"Nothing instance {instance_id} recorded is still up.") return click.echo(f"Tearing down {len(left)} sandbox{'es' if len(left) != 1 else ''} of instance {instance_id} " "(Ctrl-C to stop)") - with Interrupts() as stopper: - [report] = stopper.run(stopper.stopping([teardown_run(context)])) - record_teardown(instance_id, context) - echo_teardown("", report) + with Interrupts() as stopper, run_scope(instance_id): + try: + [report] = stopper.run(stopper.stopping([teardown_run(context)])) + except asyncio.CancelledError: + report = TeardownReport(left=recorded_still_up(context)) + record_teardown(instance_id, context, report, instance.attempt_epoch) + echo_teardown("", replace(report, terminated=tuple(sandbox for sandbox in left if sandbox not in report.still_up))) if report.still_up: raise SystemExit(1) diff --git a/src/agent_env/cli/teardown_output.py b/src/agent_env/cli/teardown_output.py index 40f1ec59e..615908acd 100644 --- a/src/agent_env/cli/teardown_output.py +++ b/src/agent_env/cli/teardown_output.py @@ -2,7 +2,8 @@ import click -from agent_env.task.teardown import TeardownReport, kind +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox +from agent_env.task.teardown import RecordedSandbox, TeardownReport, kind def echo_teardown(tag: str, report: TeardownReport) -> None: @@ -16,6 +17,12 @@ def echo_teardown(tag: str, report: TeardownReport) -> None: click.echo(click.style(f"{prefix}Still up: {sandbox.sandbox_id} ({kind(sandbox)})", fg="red")) +def gone(sandbox: RecordedSandbox) -> bool: + """Whether ``sandbox`` is a local one whose work folder is gone, which its teardown removes once the rest is down: + gone itself, so neither listed as still up nor terminated again.""" + return sandbox.sandbox_type == "local" and LocalSandbox.find_work_dir(sandbox.sandbox_id) is None + + def echo_finish_teardown(instance_ids: list[str]) -> None: """The command that finishes the teardown of each instance whose run left sandboxes up.""" if instance_ids: diff --git a/src/agent_env/task/store.py b/src/agent_env/task/store.py index 11857b8d4..2a14c8a20 100644 --- a/src/agent_env/task/store.py +++ b/src/agent_env/task/store.py @@ -18,7 +18,6 @@ from agent_env.config import get_config from agent_env.store.document_store import DocumentStore, DuplicateKeyError, Filter, Lte, Ne, Sort, SortKey, UnknownDocumentTypeError, UpdateSpec, VersionedEntityStore, VersionedEntityStoreCache, compare_and_swap from agent_env.store.query import QueryBuilder, to_document_query -from agent_env.store.routing import run_scope from agent_env.task.step_journal import ( _RESERVED_STEP_IDS, _SCHEDULER_STEP_ID, @@ -792,10 +791,12 @@ def record_task_cancelled_sync( ), ) - def record_teardown_sync(self, instance_id: str, context: "TaskStepContext") -> None: - """Keep the context a teardown left when it didn't take everything down: what the run deployed, writes of a - step a signal cut short included, and what the teardown terminated, so a later one finds what's still up.""" - self._doc_store.update(TASK_INSTANCES_COLLECTION, Filter.of(instance_id=instance_id), + def record_teardown_sync(self, instance_id: str, context: "TaskStepContext", attempt_epoch: int) -> None: + """Keep the context a run's teardown ended with: what the run deployed, writes of a step a signal cut short + included, and what the teardown took down and left up, unless an attempt later than ``attempt_epoch``, the + instance's as the teardown began, owns the instance by now.""" + match = Filter.of(instance_id=instance_id).where("attempt_epoch", Lte(attempt_epoch)) + self._doc_store.update(TASK_INSTANCES_COLLECTION, match, UpdateSpec(set={"context": context.to_safe_dict()}, inc={"rev": 1})) def append_step_attempt_failure_sync(self, instance_id: str, failure: "StepAttemptFailure") -> None: @@ -1007,24 +1008,6 @@ async def record_task_failure( -def record_teardown(instance_id: str, context: "TaskStepContext") -> None: - """Keep the context a teardown that left sandboxes up ended with (``record_teardown_sync``), where the run's - records go, retried like ``record_task_cancelled``, logging a write that keeps failing rather than raising.""" - store = get_task_instance_store() - last_exc: BaseException | None = None - for attempt in range(_FAILURE_WRITE_MAX_ATTEMPTS): - try: - with run_scope(instance_id): - store.record_teardown_sync(instance_id, context) - return - except Exception as exc: # noqa: BLE001 - last_exc = exc - if attempt + 1 < _FAILURE_WRITE_MAX_ATTEMPTS: - time.sleep(_FAILURE_WRITE_BACKOFF_SECONDS * (2**attempt)) - logger.warning("Failed to record what instance_id=%s left up after %d attempts", instance_id, - _FAILURE_WRITE_MAX_ATTEMPTS, exc_info=last_exc) - - def record_task_cancelled(instance_id: str, reason: str, completed_at_utc: str, attempt_epoch: int | None = None) -> None: """Write a cancelled run's terminal state, retried like ``record_task_failure``. It logs a write that keeps failing rather than raising. Synchronous: ``run_bundle`` calls it once its event loop has closed.""" diff --git a/src/agent_env/task/teardown.py b/src/agent_env/task/teardown.py index be07a461d..a2480963d 100644 --- a/src/agent_env/task/teardown.py +++ b/src/agent_env/task/teardown.py @@ -14,6 +14,7 @@ import asyncio import logging +import time from collections.abc import Sequence from dataclasses import dataclass from typing import NamedTuple, Optional @@ -28,7 +29,9 @@ get_env_sandbox_provider, get_sandbox_provider, ) +from agent_env.store.routing import run_scope from agent_env.task.interrupts import tearing_down +from agent_env.task.store import get_task_instance_store from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.computer_recording import end_open_recordings, register_open_recordings from agent_env.task_step.task_steps.browser.recording import (end_open_browser_recordings, @@ -39,10 +42,15 @@ PROVIDER_CLOSE_TIMEOUT_SECONDS = 5 TERMINATE_TIMEOUT_SECONDS = 120 +#: How often ``record_teardown`` tries its write, backing off from the first wait, in seconds. +_RECORD_ATTEMPTS, _RECORD_BACKOFF_SECONDS = 3, 0.5 #: ``context.metadata[TORN_DOWN_KEY]``: the ids of the sandboxes a run has terminated, since its ``deployed_*`` #: records stay (context list fields persist additions only). TORN_DOWN_KEY = "torn_down_sandbox_ids" +#: ``context.metadata[LEFT_UP_KEY]``: the ids of the sandboxes the run's last teardown left up, as a run keeps it on its +#: instance (``record_teardown``): those it didn't reach or terminate, and those whose local folder it couldn't remove. +LEFT_UP_KEY = "left_up_sandbox_ids" class RecordedSandbox(NamedTuple): @@ -68,6 +76,13 @@ def skipped(cls, context: TaskStepContext) -> TeardownReport: return cls(left=_pending(context, recorded_sandboxes(context))) +def recorded_still_up(context: TaskStepContext) -> tuple[RecordedSandbox, ...]: + """Every sandbox ``context`` records that may still be up: not torn down, or left up by its last teardown.""" + left = set(context.metadata.get(LEFT_UP_KEY) or []) + return tuple(sandbox for sandbox in recorded_sandboxes(context) + if sandbox in _pending(context, [sandbox]) or sandbox.sandbox_id in left) + + def recorded_sandboxes(context: TaskStepContext) -> tuple[RecordedSandbox, ...]: """Every sandbox ``context`` records, once each, including those a retry rolled back, which only its ``retry_resets`` marker still names. An agent placed on another record's sandbox shares its id, so the @@ -167,6 +182,28 @@ async def take_down(sandbox: RecordedSandbox) -> None: return TeardownReport(tuple(terminated), tuple(failed), tuple(s for s in pending if s.sandbox_id not in reached)) +def record_teardown(instance_id: str, context: TaskStepContext, report: TeardownReport, attempt_epoch: int) -> None: + """Keep on ``instance_id``, where its run's records go, the context its teardown ended with and what that left up + (``LEFT_UP_KEY``), so ``agent-env task teardown`` finds what's still up, unless a later attempt owns the instance + (``record_teardown_sync``). Retried, and a write that keeps failing is logged, not raised.""" + context.metadata[LEFT_UP_KEY] = [sandbox.sandbox_id for sandbox in report.still_up] + for attempt in range(_RECORD_ATTEMPTS): + try: + with run_scope(instance_id): + get_task_instance_store().record_teardown_sync(instance_id, context, attempt_epoch) + return + except Exception as e: # noqa: BLE001 + if attempt + 1 == _RECORD_ATTEMPTS: + logger.warning("teardown: what instance %s left up wasn't recorded: %s", instance_id, _described(e)) + else: + time.sleep(_RECORD_BACKOFF_SECONDS * 2 ** attempt) + + +def attempt_epoch(instance_id: str) -> int: + """``instance_id``'s attempt epoch now, which a teardown about to start fences ``record_teardown`` with.""" + return get_task_instance_store().get(instance_id).attempt_epoch + + async def before_teardown(context: TaskStepContext, within: float, agent_names: Optional[list[str]] = None) -> None: """Drain the changelogs staged on ``agent_names``, or on every agent when None, within ``within`` seconds each.""" for name in [None] if agent_names is None else agent_names: diff --git a/tst/unit/bundle/run_teardown_test.py b/tst/unit/bundle/run_teardown_test.py index 4a2002bc4..b5f1a501c 100644 --- a/tst/unit/bundle/run_teardown_test.py +++ b/tst/unit/bundle/run_teardown_test.py @@ -3,6 +3,7 @@ the teardown. ``agent-env run`` prints what it tore down, what it keeps up and what is still up.""" import asyncio +import importlib import json import logging import re @@ -19,6 +20,8 @@ from agent_env.config.runtime import Config from agent_env.store.document_store import Filter, UpdateSpec from agent_env.task import TaskInstanceStore, teardown +from agent_env.task.store import register_task_instance, seed_task_instance_context +from agent_env.store import routing from agent_env.env.env import DeployedGatewayEnv from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext from agent_env.task_step.task_step import TaskStep @@ -26,6 +29,7 @@ from tst.unit.bundle._support import layout, local_store pytestmark = pytest.mark.usefixtures("sigint_handled") +teardown_command = importlib.import_module("agent_env.cli.task.teardown") # the package's ``teardown`` is the command class _Deploys(TaskStep): @@ -322,18 +326,106 @@ def test_a_failed_teardown_is_recorded_on_the_instance_and_the_command_tries_aga layout(bundle_dir, {"tasks/ok.json": _task()}) terminated.failing.add("sb-ok") instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") - recorded = _stored_context(instance).metadata.get(teardown.TORN_DOWN_KEY) + recorded = _stored_context(instance).metadata still_stuck = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) terminated.failing.clear() finished = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) - assert recorded is None + assert (recorded.get(teardown.TORN_DOWN_KEY), recorded[teardown.LEFT_UP_KEY]) == (None, ["sb-ok"]) assert still_stuck.exit_code == 1 and "Couldn't tear down sb-ok: RuntimeError: stuck" in still_stuck.output assert (finished.exit_code, terminated) == (0, ["sb-ok"]), finished.output assert _stored_context(instance).metadata[teardown.TORN_DOWN_KEY] == ["sb-ok"] +def test_a_finished_teardown_is_recorded_so_the_command_finds_nothing_and_terminates_nothing_again( + bundle_dir, terminated, quiet_logs): + layout(bundle_dir, {"tasks/ok.json": _task()}) + instance = CliRunner().invoke(cli, ["run", str(bundle_dir)]).output + instance = re.search(r"instance (@local/\S+/ok-\w+)", instance).group(1) + + again = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + + recorded = _stored_context(instance).metadata + assert (recorded[teardown.TORN_DOWN_KEY], recorded[teardown.LEFT_UP_KEY]) == (["sb-ok"], []) + assert (again.exit_code, again.output) == (0, f"Nothing instance {instance} recorded is still up.\n") + assert terminated == ["sb-ok"] + + +def test_a_sandbox_terminated_whose_folder_wasnt_removed_is_still_up_and_the_command_removes_only_its_folder( + bundle_dir, terminated, quiet_logs, monkeypatch): + layout(bundle_dir, {"tasks/ok.json": _task()}) + monkeypatch.setattr(SandboxProvider, "ON_THIS_MACHINE", True) + removals = [] + + def remove(sandbox_id): + removals.append(sandbox_id) + if len(removals) == 1: + raise OSError("busy") + + monkeypatch.setattr(teardown, "remove_local_work_dir", remove) + instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") + + finished = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + + assert finished.exit_code == 0, finished.output + assert finished.output == f"Tearing down 1 sandbox of instance {instance} (Ctrl-C to stop)\nTore down 1 sandbox\n" + assert (terminated, removals) == (["sb-ok"], ["sb-ok", "sb-ok"]) + assert _stored_context(instance).metadata[teardown.LEFT_UP_KEY] == [] + + +def test_a_local_sandbox_whose_folder_is_gone_counts_as_gone_as_the_run_lists_it(bundle_dir, quiet_logs): + """As an instance recorded before its run kept what its teardown took down: the folder goes last.""" + task_id = "@local/~/lifecycle/old" + instance = f"{task_id}-12345678" + context = TaskStepContext(metadata={"task_id": task_id}) + context.deployed_sandboxes.append(DeployedSandbox(sandbox_name="box", sandbox_id="local-gone", sandbox_mode="vm", + sandbox_type="local")) + with routing.namespace_routing(), routing.run_scope(task_id): + register_task_instance(task_id, 1, 1, instance_id=instance) + seed_task_instance_context(instance, context) + local_store().update("task_instances", Filter.of(instance_id=instance), UpdateSpec(set={"status": "completed"})) + + result = CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]) + + assert (result.exit_code, result.output) == (0, f"Nothing instance {instance} recorded is still up.\n") + + +def test_the_command_tears_down_inside_the_runs_namespace(bundle_dir, terminated, quiet_logs, monkeypatch): + """So a recording it ends, or registers, uses the run's own stores.""" + layout(bundle_dir, {"tasks/ok.json": _task()}) + terminated.failing.add("sb-ok") + instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") + terminated.failing.clear() + seen, real = [], teardown_command.teardown_run + + async def watched(context): + seen.append(routing.local_run_namespace()) + return await real(context) + + monkeypatch.setattr(teardown_command, "teardown_run", watched) + + assert CliRunner().invoke(cli, ["task", "teardown", "--instance", instance]).exit_code == 0 + assert seen == ["local"] + + +def test_a_teardown_record_doesnt_overwrite_an_attempt_that_took_the_instance_over_meanwhile(bundle_dir, terminated, + quiet_logs): + layout(bundle_dir, {"tasks/ok.json": _task()}) + terminated.failing.add("sb-ok") + instance = _instance(CliRunner().invoke(cli, ["run", str(bundle_dir)]).output, "ok") + later = _stored_context(instance) + later.metadata["attempt"] = "later" + with routing.namespace_routing(): + took_over_at = teardown.attempt_epoch(instance) + 1 + local_store().update("task_instances", Filter.of(instance_id=instance), + UpdateSpec(set={"attempt_epoch": took_over_at})) + teardown.record_teardown(instance, later, teardown.TeardownReport(), took_over_at) + teardown.record_teardown(instance, _stored_context(instance), teardown.TeardownReport(), took_over_at - 1) + + assert _stored_context(instance).metadata["attempt"] == "later" + + def test_the_command_leaves_a_run_recorded_running_alone_unless_forced_and_refuses_an_unknown_instance( bundle_dir, terminated, quiet_logs): layout(bundle_dir, {"tasks/ok.json": _task()}) From 6b71101692eed9d27c941bd34a1d6f7037568c6e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:23:28 -0800 Subject: [PATCH 4/4] fix(task): reading an instance to record its teardown never stops the teardown attempt_epoch now returns None, logged, for an instance it can't read, and a teardown it can't fence goes unrecorded, so a store error before task run --keep's teardown no longer leaves every kept sandbox up. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/run.py | 14 +------------- src/agent_env/cli/task/run.py | 2 +- src/agent_env/task/teardown.py | 14 +++++++++++--- tst/unit/cli/task_run_teardown_test.py | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 49f22bb93..15707cbe6 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -394,18 +394,6 @@ def _record_teardowns(runs: Sequence[TaskRun]) -> None: record_teardown(run.instance_id, run.context, run.torn_down, run.attempt_epoch) -def _epoch(instance_id: str | None) -> int | None: - """``instance_id``'s attempt epoch as its teardown begins, read synchronously so no signal cuts it short; None - when there's no instance or it can't be read.""" - if instance_id is None: - return None - try: - return attempt_epoch(instance_id) - except Exception as e: # noqa: BLE001 - logger.warning("Couldn't read instance %s before its teardown: %s", instance_id, e) - return None - - def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRun, ...], skipped: tuple[BundleEntry, ...]) -> BundleRun: by_id = {run.task.id: run for run in runs} @@ -598,7 +586,7 @@ async def run_one(entry: BundleEntry, task: Task) -> TaskRun: raise cancelled = True duration = time.monotonic() - began - epoch = _epoch(context.instance_id) + epoch = attempt_epoch(context.instance_id) # synchronous, so no signal cuts it short report = await settle(context, cancelled) finally: running.discard(me) diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index 6e12140bb..9c4abc3aa 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -288,7 +288,7 @@ def _tear_down_kept(kept) -> None: """Tear down what the runs kept; a Ctrl-C stops it. Counts signals afresh, so the one that got here doesn't.""" count = sum(len(left) for _, _, left in kept) click.echo(f"\nTearing down {count} {'sandbox' if count == 1 else 'sandboxes'} (Ctrl-C again to stop now)") - epochs = [attempt_epoch(context.instance_id) if context.instance_id else None for _, context, _ in kept] + epochs = [attempt_epoch(context.instance_id) for _, context, _ in kept] with Interrupts() as stopper: try: reports = stopper.run(stopper.stopping(teardown_run(context) for _, context, _ in kept)) diff --git a/src/agent_env/task/teardown.py b/src/agent_env/task/teardown.py index a2480963d..7716f8fc1 100644 --- a/src/agent_env/task/teardown.py +++ b/src/agent_env/task/teardown.py @@ -199,9 +199,17 @@ def record_teardown(instance_id: str, context: TaskStepContext, report: Teardown time.sleep(_RECORD_BACKOFF_SECONDS * 2 ** attempt) -def attempt_epoch(instance_id: str) -> int: - """``instance_id``'s attempt epoch now, which a teardown about to start fences ``record_teardown`` with.""" - return get_task_instance_store().get(instance_id).attempt_epoch +def attempt_epoch(instance_id: Optional[str]) -> Optional[int]: + """``instance_id``'s attempt epoch now, which a teardown about to start fences ``record_teardown`` with; None, + leaving the teardown unrecorded, when there's no instance or it can't be read, which never stops the teardown.""" + if instance_id is None: + return None + try: + return get_task_instance_store().get(instance_id).attempt_epoch + except Exception as e: # noqa: BLE001 + logger.warning("teardown: instance %s couldn't be read, so what its teardown leaves won't be recorded: %s", + instance_id, _described(e)) + return None async def before_teardown(context: TaskStepContext, within: float, agent_names: Optional[list[str]] = None) -> None: diff --git a/tst/unit/cli/task_run_teardown_test.py b/tst/unit/cli/task_run_teardown_test.py index ccefe9f69..957e30305 100644 --- a/tst/unit/cli/task_run_teardown_test.py +++ b/tst/unit/cli/task_run_teardown_test.py @@ -20,6 +20,7 @@ from agent_env.runner.runner import RunStatus from agent_env.task import Task from agent_env.task.store import get_task_instance_store, register_task_instance, seed_task_instance_context +from agent_env.task import teardown as teardown_core from agent_env.task.teardown import TORN_DOWN_KEY, RecordedSandbox, TeardownReport from agent_env.task_step.context import DeployedSandbox @@ -142,6 +143,19 @@ async def stuck(context): assert get_task_instance_store().get(instance).context["metadata"]["teardown_tried"] is True +def test_keep_tears_down_what_it_held_though_its_instance_cant_be_read_to_record_it(monkeypatch, tmp_path, torn_down): + class Unreadable: + def get(self, instance_id): + raise ConnectionError("the store went away") + + monkeypatch.setattr(teardown_core, "get_task_instance_store", Unreadable) + + result = _invoke(monkeypatch, tmp_path, _Deploys(), "run", "--keep", signal_after=1.0) + + assert result.exit_code == 0, result.output + assert _ids(torn_down) == ["sb-1"] + + def test_keep_still_tears_down_a_run_ctrl_c_cancels(monkeypatch, tmp_path, torn_down): task = _Deploys(nap=30)