Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md

Large diffs are not rendered by default.

21 changes: 17 additions & 4 deletions src/agent_env/bundle/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from agent_env.task.interrupts import Interrupts
from agent_env.utils.progress import run_progress
from agent_env.task.task import Task, recorded_scores
from agent_env.task.teardown import TeardownReport, teardown_run
from agent_env.task.teardown import TeardownReport, attempt_epoch, record_teardown, teardown_run
from agent_env.task_step.context import TaskStepContext, sandbox_overrides

from .hub import SOURCE_KEY, HubSource
Expand Down Expand Up @@ -73,6 +73,7 @@ class TaskRun:
cancelled: bool = False
started: bool = True # False when it was cancelled before it started
torn_down: TeardownReport | None = None # None while its sandboxes are kept up
attempt_epoch: int | None = None # its instance's as its teardown began, which ``record_teardown`` is fenced by

@property
def instance_id(self) -> str | None:
Expand Down Expand Up @@ -136,9 +137,10 @@ def teardown(self) -> BundleRun:
kept = [run for run in self.runs if run.torn_down is None]
if not kept:
return self
with Interrupts() as interrupts:
with namespace_routing(), Interrupts() as interrupts:
reports = interrupts.run(interrupts.stopping(teardown_run(run.context) for run in kept))
result = self._with_reports(kept, reports)
_record_teardowns([replace(run, torn_down=report) for run, report in zip(kept, reports)])
if interrupts.count:
raise RunInterrupted(result, interrupts.signum)
return result
Expand Down Expand Up @@ -270,6 +272,7 @@ def run_bundle(
solver_params(plan, prepared.materialization), keep, say, interrupts,
source))
_mark_cancelled(runs, interrupts.reason)
_record_teardowns(runs)
result = _bundle_run(plan, prepared.materialization, runs,
_skipped(plan, entries, every=not tasks and not evals))
if not interrupts.count:
Expand Down Expand Up @@ -382,6 +385,15 @@ def _mark_cancelled(runs: tuple[TaskRun, ...], reason: str) -> None:
record_task_cancelled(run.instance_id, reason, Task._utc_now_str())


def _record_teardowns(runs: Sequence[TaskRun]) -> None:
"""Keep on each run's instance the context its teardown ended with, so ``agent-env task teardown`` finds what it
left up, those a step a signal cut short had recorded included, and nothing once all is down. Like
``_mark_cancelled``, it runs once the loop has closed."""
for run in runs:
if run.torn_down is not None and run.instance_id and run.attempt_epoch is not None:
record_teardown(run.instance_id, run.context, run.torn_down, run.attempt_epoch)


def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRun, ...],
skipped: tuple[BundleEntry, ...]) -> BundleRun:
by_id = {run.task.id: run for run in runs}
Expand Down Expand Up @@ -555,7 +567,7 @@ async def run_one(entry: BundleEntry, task: Task) -> TaskRun:
me = asyncio.current_task()
running.add(me)
began = time.monotonic()
error, cancelled = None, False
error, cancelled, epoch = None, False, None
try:
try:
with run_progress(lambda line: say(f"{tag} {line}")):
Expand All @@ -574,11 +586,12 @@ async def run_one(entry: BundleEntry, task: Task) -> TaskRun:
raise
cancelled = True
duration = time.monotonic() - began
epoch = attempt_epoch(context.instance_id) # synchronous, so no signal cuts it short
report = await settle(context, cancelled)
finally:
running.discard(me)
semaphore.release()
run = TaskRun(entry, task, context, error, duration, cancelled=cancelled, torn_down=report)
run = TaskRun(entry, task, context, error, duration, cancelled=cancelled, torn_down=report, attempt_epoch=epoch)
say(f"{tag} {run.outcome.value} in {run.duration:.1f}s")
for sandbox_record, why in (report.failed if report else ()):
say(f"{tag} couldn't tear down {sandbox_record.sandbox_id}: {why}")
Expand Down
6 changes: 4 additions & 2 deletions src/agent_env/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from agent_env.bundle.plan import NotInStore
from agent_env.bundle.requires import UnmetRequirements
from agent_env.cli._install_prompt import offer_to_install
from agent_env.cli.teardown_output import echo_finish_teardown, gone
from agent_env.plugins._bundles import ALIAS_TABLES, installed_aliases
from agent_env.plugins._discovery import discovery_error
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox
Expand Down Expand Up @@ -354,8 +355,7 @@ def _records(context: TaskStepContext) -> list[tuple[str, object, list[tuple[str
def _report_teardown(result: BundleRun) -> None:
reports = [run.torn_down or TeardownReport.skipped(run.context) for run in result.runs]
terminated = sum(len(report.terminated) for report in reports)
still_up = [sandbox for report in reports for sandbox in report.still_up
if sandbox.sandbox_type != "local" or LocalSandbox.find_work_dir(sandbox.sandbox_id)]
still_up = [sandbox for report in reports for sandbox in report.still_up if not gone(sandbox)]
if terminated or still_up:
click.echo()
if terminated:
Expand All @@ -364,6 +364,8 @@ def _report_teardown(result: BundleRun) -> None:
click.echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow"))
for sandbox in still_up:
click.echo(f" {_still_up(sandbox)}")
echo_finish_teardown([run.instance_id for run, report in zip(result.runs, reports)
if run.instance_id and any(sandbox in still_up for sandbox in report.still_up)])


def _still_up(sandbox: RecordedSandbox) -> str:
Expand Down
2 changes: 2 additions & 0 deletions src/agent_env/cli/task/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from .get import get
from .get_instance import get_instance
from .run import run, run_batch
from .teardown import teardown
from .validate import validate


Expand All @@ -18,4 +19,5 @@ def task():
task.add_command(get_instance)
task.add_command(run)
task.add_command(run_batch)
task.add_command(teardown)
task.add_command(validate)
18 changes: 13 additions & 5 deletions src/agent_env/cli/task/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
from agent_env.config.runtime import GATEWAY_OVERRIDE, SERVICE_DB_OVERRIDE, WEBSITE_BROWSER_OVERRIDE
from agent_env.cli import submitted_runs
from agent_env.cli.identity import get_agent_env_client_id
from agent_env.cli.teardown_output import echo_teardown
from agent_env.cli.teardown_output import echo_finish_teardown, echo_teardown, gone
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox
from agent_env.runner.runner import KEEP_KEY, RunStatus
from agent_env.store.ids import derive_id, fs_safe, is_local_id, validate_local_id
from agent_env.store.routing import run_scope
from agent_env.task.interrupts import Interrupts
from agent_env.task.store import get_task_instance_store, new_instance_id
from agent_env.task.teardown import TeardownReport, kind, teardown_run
from agent_env.task.teardown import TeardownReport, attempt_epoch, kind, record_teardown, teardown_run
from agent_env.task_step.context import TaskStepContext, read_dual_keyed
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.context_ops import build_context_update_ops
Expand Down Expand Up @@ -288,14 +288,22 @@ def _tear_down_kept(kept) -> None:
"""Tear down what the runs kept; a Ctrl-C stops it. Counts signals afresh, so the one that got here doesn't."""
count = sum(len(left) for _, _, left in kept)
click.echo(f"\nTearing down {count} {'sandbox' if count == 1 else 'sandboxes'} (Ctrl-C again to stop now)")
epochs = [attempt_epoch(context.instance_id) for _, context, _ in kept]
with Interrupts() as stopper:
try:
reports = stopper.run(stopper.stopping(teardown_run(context) for _, context, _ in kept))
except asyncio.CancelledError:
click.echo(click.style("Stopped the teardown; what it hadn't reached is still up.", fg="yellow"))
return
for (tag, _, _), report in zip(kept, reports):
echo_teardown(tag, report)
reports = [TeardownReport.skipped(context) for _, context, _ in kept]
else:
for (tag, _, _), report in zip(kept, reports):
echo_teardown(tag, report)
contexts = [context for _, context, _ in kept]
for context, report, epoch in zip(contexts, reports, epochs):
if epoch is not None:
record_teardown(context.instance_id, context, report, epoch)
echo_finish_teardown([context.instance_id for context, report in zip(contexts, reports)
if context.instance_id and any(not gone(sandbox) for sandbox in report.still_up)])


def _settle_kept(tagged_contexts, interrupts: Interrupts, interrupted) -> None:
Expand Down
50 changes: 50 additions & 0 deletions src/agent_env/cli/task/teardown.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""Tear down what a task run left up."""

import asyncio
from dataclasses import replace

import click

from agent_env.cli.teardown_output import echo_teardown, gone
from agent_env.store.base import NotFoundError
from agent_env.store.routing import run_scope
from agent_env.task.interrupts import Interrupts
from agent_env.task.store import get_task_instance_store
from agent_env.task.teardown import TORN_DOWN_KEY, TeardownReport, record_teardown, recorded_still_up, teardown_run
from agent_env.task_step.context import TaskStepContext


@click.command("teardown")
@click.option("--instance", "instance_id", required=True, help="The task instance whose sandboxes to tear down.")
@click.option("--force", is_flag=True,
help="Tear down an instance recorded running too, as a run killed outright leaves it.")
def teardown(instance_id: str, force: bool):
"""Tear down the sandboxes a task run left up, as its own teardown does: the recordings it left open end, each
env still up closes, and each sandbox is terminated by id, a local one's folder removed. For a run whose teardown
was stopped or failed, which `agent-env run` names. Ctrl-C stops it; it exits 1 while any is still up, and
running it again goes on from there."""
try:
instance = get_task_instance_store().get(instance_id)
except NotFoundError:
raise click.ClickException(f"no task instance {instance_id!r}") from None
if instance.status == "running" and not force:
raise click.ClickException(f"instance {instance_id} is recorded running, so its run may still be using its "
"sandboxes; pass --force once it isn't, as for a run killed outright")
context = TaskStepContext.from_dict(instance.context or {})
context.metadata[TORN_DOWN_KEY] = [*context.metadata.get(TORN_DOWN_KEY, []),
*(sandbox.sandbox_id for sandbox in recorded_still_up(context) if gone(sandbox))]
left = recorded_still_up(context)
if not left:
click.echo(f"Nothing instance {instance_id} recorded is still up.")
return
click.echo(f"Tearing down {len(left)} sandbox{'es' if len(left) != 1 else ''} of instance {instance_id} "
"(Ctrl-C to stop)")
with Interrupts() as stopper, run_scope(instance_id):
try:
[report] = stopper.run(stopper.stopping([teardown_run(context)]))
except asyncio.CancelledError:
report = TeardownReport(left=recorded_still_up(context))
record_teardown(instance_id, context, report, instance.attempt_epoch)
echo_teardown("", replace(report, terminated=tuple(sandbox for sandbox in left if sandbox not in report.still_up)))
if report.still_up:
raise SystemExit(1)
17 changes: 16 additions & 1 deletion src/agent_env/cli/teardown_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

import click

from agent_env.task.teardown import TeardownReport, kind
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox
from agent_env.task.teardown import RecordedSandbox, TeardownReport, kind


def echo_teardown(tag: str, report: TeardownReport) -> None:
Expand All @@ -14,3 +15,17 @@ def echo_teardown(tag: str, report: TeardownReport) -> None:
click.echo(click.style(f"{prefix}Couldn't tear down {sandbox.sandbox_id}: {why}", fg="red"))
for sandbox in report.left:
click.echo(click.style(f"{prefix}Still up: {sandbox.sandbox_id} ({kind(sandbox)})", fg="red"))


def gone(sandbox: RecordedSandbox) -> bool:
"""Whether ``sandbox`` is a local one whose work folder is gone, which its teardown removes once the rest is down:
gone itself, so neither listed as still up nor terminated again."""
return sandbox.sandbox_type == "local" and LocalSandbox.find_work_dir(sandbox.sandbox_id) is None


def echo_finish_teardown(instance_ids: list[str]) -> None:
"""The command that finishes the teardown of each instance whose run left sandboxes up."""
if instance_ids:
click.echo("Tear down what's left with:")
for instance_id in instance_ids:
click.echo(f" agent-env task teardown --instance {instance_id}")
8 changes: 8 additions & 0 deletions src/agent_env/task/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,14 @@ def record_task_cancelled_sync(
),
)

def record_teardown_sync(self, instance_id: str, context: "TaskStepContext", attempt_epoch: int) -> None:
"""Keep the context a run's teardown ended with: what the run deployed, writes of a step a signal cut short
included, and what the teardown took down and left up, unless an attempt later than ``attempt_epoch``, the
instance's as the teardown began, owns the instance by now."""
match = Filter.of(instance_id=instance_id).where("attempt_epoch", Lte(attempt_epoch))
self._doc_store.update(TASK_INSTANCES_COLLECTION, match,
UpdateSpec(set={"context": context.to_safe_dict()}, inc={"rev": 1}))

def append_step_attempt_failure_sync(self, instance_id: str, failure: "StepAttemptFailure") -> None:
"""Best-effort append of one attempt's failure, designed not to conflict with the
``record_task_failure`` write. ``add_to_set`` creates the array on the first append and
Expand Down
45 changes: 45 additions & 0 deletions src/agent_env/task/teardown.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import asyncio
import logging
import time
from collections.abc import Sequence
from dataclasses import dataclass
from typing import NamedTuple, Optional
Expand All @@ -28,7 +29,9 @@
get_env_sandbox_provider,
get_sandbox_provider,
)
from agent_env.store.routing import run_scope
from agent_env.task.interrupts import tearing_down
from agent_env.task.store import get_task_instance_store
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps.computer_recording import end_open_recordings, register_open_recordings
from agent_env.task_step.task_steps.browser.recording import (end_open_browser_recordings,
Expand All @@ -39,10 +42,15 @@
PROVIDER_CLOSE_TIMEOUT_SECONDS = 5

TERMINATE_TIMEOUT_SECONDS = 120
#: How often ``record_teardown`` tries its write, backing off from the first wait, in seconds.
_RECORD_ATTEMPTS, _RECORD_BACKOFF_SECONDS = 3, 0.5

#: ``context.metadata[TORN_DOWN_KEY]``: the ids of the sandboxes a run has terminated, since its ``deployed_*``
#: records stay (context list fields persist additions only).
TORN_DOWN_KEY = "torn_down_sandbox_ids"
#: ``context.metadata[LEFT_UP_KEY]``: the ids of the sandboxes the run's last teardown left up, as a run keeps it on its
#: instance (``record_teardown``): those it didn't reach or terminate, and those whose local folder it couldn't remove.
LEFT_UP_KEY = "left_up_sandbox_ids"


class RecordedSandbox(NamedTuple):
Expand All @@ -68,6 +76,13 @@ def skipped(cls, context: TaskStepContext) -> TeardownReport:
return cls(left=_pending(context, recorded_sandboxes(context)))


def recorded_still_up(context: TaskStepContext) -> tuple[RecordedSandbox, ...]:
"""Every sandbox ``context`` records that may still be up: not torn down, or left up by its last teardown."""
left = set(context.metadata.get(LEFT_UP_KEY) or [])
return tuple(sandbox for sandbox in recorded_sandboxes(context)
if sandbox in _pending(context, [sandbox]) or sandbox.sandbox_id in left)


def recorded_sandboxes(context: TaskStepContext) -> tuple[RecordedSandbox, ...]:
"""Every sandbox ``context`` records, once each, including those a retry rolled back, which only its
``retry_resets`` marker still names. An agent placed on another record's sandbox shares its id, so the
Expand Down Expand Up @@ -167,6 +182,36 @@ async def take_down(sandbox: RecordedSandbox) -> None:
return TeardownReport(tuple(terminated), tuple(failed), tuple(s for s in pending if s.sandbox_id not in reached))


def record_teardown(instance_id: str, context: TaskStepContext, report: TeardownReport, attempt_epoch: int) -> None:
"""Keep on ``instance_id``, where its run's records go, the context its teardown ended with and what that left up
(``LEFT_UP_KEY``), so ``agent-env task teardown`` finds what's still up, unless a later attempt owns the instance
(``record_teardown_sync``). Retried, and a write that keeps failing is logged, not raised."""
context.metadata[LEFT_UP_KEY] = [sandbox.sandbox_id for sandbox in report.still_up]
for attempt in range(_RECORD_ATTEMPTS):
try:
with run_scope(instance_id):
get_task_instance_store().record_teardown_sync(instance_id, context, attempt_epoch)
return
except Exception as e: # noqa: BLE001
if attempt + 1 == _RECORD_ATTEMPTS:
logger.warning("teardown: what instance %s left up wasn't recorded: %s", instance_id, _described(e))
else:
time.sleep(_RECORD_BACKOFF_SECONDS * 2 ** attempt)


def attempt_epoch(instance_id: Optional[str]) -> Optional[int]:
"""``instance_id``'s attempt epoch now, which a teardown about to start fences ``record_teardown`` with; None,
leaving the teardown unrecorded, when there's no instance or it can't be read, which never stops the teardown."""
if instance_id is None:
return None
try:
return get_task_instance_store().get(instance_id).attempt_epoch
except Exception as e: # noqa: BLE001
logger.warning("teardown: instance %s couldn't be read, so what its teardown leaves won't be recorded: %s",
instance_id, _described(e))
return None


async def before_teardown(context: TaskStepContext, within: float, agent_names: Optional[list[str]] = None) -> None:
"""Drain the changelogs staged on ``agent_names``, or on every agent when None, within ``within`` seconds each."""
for name in [None] if agent_names is None else agent_names:
Expand Down
Loading
Loading