From 9c68c3962c351a29a0d5eddb4070cae2c831cd5f Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 02:58:17 +0000
Subject: [PATCH 01/27] feat: add native v1 debug CLI

---
 docs/reference.md                             |   2 +-
 pyproject.toml                                |   1 +
 .../composable/sandbox_debug_env.py           |   6 +
 .../experimental/composable/swe_debug_env.py  |   5 +-
 verifiers/v1/GUIDE.md                         |  34 ++-
 verifiers/v1/README.md                        |   1 +
 verifiers/v1/cli/debug.py                     | 267 ++++++++++++++++++
 verifiers/v1/cli/output.py                    |   6 +-
 verifiers/v1/cli/validate.py                  | 128 +++++++--
 verifiers/v1/configs/__init__.py              |   3 +-
 verifiers/v1/configs/debug.py                 |  78 +++++
 verifiers/v1/configs/validate.py              |   7 +
 12 files changed, 512 insertions(+), 26 deletions(-)
 create mode 100644 verifiers/v1/cli/debug.py
 create mode 100644 verifiers/v1/configs/debug.py

diff --git a/docs/reference.md b/docs/reference.md
index 6bc4f49a1d..6b657d7221 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -577,7 +577,7 @@ class SandboxDebugEnv(SandboxMixin, MultiTurnEnv):
     ): ...
 ```
 
-No-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the task sandbox, optionally runs task setup, runs one debug step (`none`, `gold_patch`, `command`, or `script`), and optionally runs tests and scores the result. `SWEDebugEnv` remains as a deprecated wrapper for older callers.
+Deprecated no-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the task sandbox, optionally runs task setup, runs one debug step (`none`, `gold_patch`, `command`, or `script`), and optionally runs tests and scores the result. For v1 tasksets, prefer the native `uv run debug ...` CLI; `SWEDebugEnv` remains as a deprecated compatibility wrapper for older callers.
 
 #### EnvGroup
 
diff --git a/pyproject.toml b/pyproject.toml
index 8b433cbc1a..01df90fa49 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -166,6 +166,7 @@ flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
 vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"
 validate = "verifiers.v1.cli.validate:main"
+debug = "verifiers.v1.cli.debug:main"
 serve = "verifiers.v1.cli.serve:main"
 init = "verifiers.v1.cli.init:main"
 vf-gepa = "verifiers.scripts.gepa:main"
diff --git a/verifiers/envs/experimental/composable/sandbox_debug_env.py b/verifiers/envs/experimental/composable/sandbox_debug_env.py
index 54a253b005..dae2568101 100644
--- a/verifiers/envs/experimental/composable/sandbox_debug_env.py
+++ b/verifiers/envs/experimental/composable/sandbox_debug_env.py
@@ -3,6 +3,7 @@
 import shlex
 import time
 from typing import Any, Literal
+from warnings import warn
 
 import verifiers as vf
 from prime_sandboxes import CreateSandboxRequest
@@ -59,6 +60,11 @@ def __init__(
         output_tail_chars: int = 2000,
         **sandbox_kwargs: Any,
     ):
+        warn(
+            "SandboxDebugEnv is deprecated; use the native v1 `debug` CLI for v1 tasksets.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         if debug_step not in ("none", "gold_patch", "command", "script"):
             raise ValueError(f"Unsupported debug_step: {debug_step!r}")
         if debug_step == "command" and not debug_command:
diff --git a/verifiers/envs/experimental/composable/swe_debug_env.py b/verifiers/envs/experimental/composable/swe_debug_env.py
index 675184d76a..e374ccb72a 100644
--- a/verifiers/envs/experimental/composable/swe_debug_env.py
+++ b/verifiers/envs/experimental/composable/swe_debug_env.py
@@ -11,7 +11,8 @@ class SWEDebugRubric(SandboxDebugRubric):
 
     def __init__(self, **kwargs: Any):
         warn(
-            "SWEDebugRubric is deprecated; use SandboxDebugRubric.",
+            "SWEDebugRubric is deprecated with the composable debug envs; use the "
+            "native v1 `debug` CLI for v1 tasksets.",
             DeprecationWarning,
             stacklevel=2,
         )
@@ -23,7 +24,7 @@ class SWEDebugEnv(SandboxDebugEnv):
 
     def __init__(self, *args: Any, **kwargs: Any):
         warn(
-            "SWEDebugEnv is deprecated; use SandboxDebugEnv.",
+            "SWEDebugEnv is deprecated; use the native v1 `debug` CLI for v1 tasksets.",
             DeprecationWarning,
             stacklevel=2,
         )
diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md
index 90c39911e1..a43773a9a1 100644
--- a/verifiers/v1/GUIDE.md
+++ b/verifiers/v1/GUIDE.md
@@ -681,8 +681,9 @@ pass `docker` / `prime` / `modal`.
 
 # CLI reference
 
-Three commands, all `uv run <cmd>`: **`eval`** (run + score a model), **`validate`** (model-free
-gold check), **`init`** (scaffold). They share a resolution layer:
+Four commands, all `uv run <cmd>`: **`eval`** (run + score a model), **`validate`**
+(model-free setup/gold checks), **`debug`** (setup + one shell action), **`init`**
+(scaffold). They share a resolution layer:
 
 - **Positional id** — a leading bare token is the taskset id: `eval gsm8k-v1` == `eval --taskset.id
   gsm8k-v1` (for `init` it's the env name).
@@ -739,16 +740,20 @@ trace per line, appended as each rollout finishes — durable mid-run), and `eva
 
 Run each task's `validate` hook — a model-free check that the ground truth holds (the gold patch
 makes the tests pass, the verifier accepts the gold answer) — in a runtime with the taskset's
-`setup` applied. No model, no harness.
+`setup` applied. No model, no harness. Use `--mode noop` to run setup only, or `--mode both`
+to run `apply-answer` and `noop` in independent runtimes and report one aggregate row.
 
 ```bash
 uv run validate gsm8k-v1 -n 20 --runtime.type subprocess
+uv run validate swebench-v1 -n 1 --runtime.type prime --mode noop
+uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
 ```
 
 | flag | default | meaning |
 | --- | --- | --- |
 | `<taskset-id>` / `--taskset.id` | — | taskset to validate |
 | `--runtime.type` | `docker` | runtime for `setup` + `validate` (a gold check often needs the task's container) |
+| `--mode` | `apply-answer` | `apply-answer`, `noop`, or `both` |
 | `--setup-timeout` / `--validate-timeout` | None | per-hook wall-clock caps |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
 | `-v`/`--verbose`, `--no-rich` | | logging / disable the dashboard |
@@ -759,6 +764,29 @@ captured as a result row (one bad task is data, not a crash) with reason `valid`
 default runtime is **docker** (unlike eval's subprocess), and a subprocess runtime against a
 `NEEDS_CONTAINER` / image-bearing taskset aborts with a clear error.
 
+## `debug`
+
+Set up each selected task, run one explicit shell action, and save the trace. This is for
+inspecting task state without a model: it does not apply gold patches, call `validate`, run
+`finalize`, or score rewards.
+
+```bash
+uv run debug swebench-v1 -n 1 --runtime.type prime --command 'pwd; git status --short | head'
+uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
+```
+
+| flag | default | meaning |
+| --- | --- | --- |
+| `<taskset-id>` / `--taskset.id` | — | taskset to debug |
+| `--command` / `--script-path` | — | exactly one inline command or host script to upload and execute |
+| `--runtime.type` | `docker` | runtime for setup + the debug action |
+| `--setup-timeout` / `--timeout` | None | setup and action wall-clock caps |
+| `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
+| `-o`/`--output-dir` | fresh debug run dir | where `config.toml` and `results.jsonl` are written |
+
+Each saved trace has command/script metadata, exit status, elapsed time, timeout/error fields,
+and stdout/stderr tails under `trace.info["debug"]`.
+
 ## `init`
 
 Scaffold a new v1 environment package under `--path` (default `./environments`), following the
diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md
index acdd2ee5e7..3327a3c9ec 100644
--- a/verifiers/v1/README.md
+++ b/verifiers/v1/README.md
@@ -42,6 +42,7 @@ uv sync --python 3.12 --extra harbor
 uv run init my-task-v1           # scaffold a new environment (--add-tool/--add-user/--add-harness)
 uv run eval gsm8k-v1 -n 5 -r 3   # single-turn math; default harness; subprocess runtime
 uv run validate gsm8k-v1 -n 5    # model-free: run each task's gold check (the `validate` hook)
+uv run debug gsm8k-v1 -n 1 --command 'pwd'  # setup, run one shell action, save traces
 uv run eval -h                   # typed help (+ the local example tasksets/harnesses)
 ```
 
diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
new file mode 100644
index 0000000000..1c15d18547
--- /dev/null
+++ b/verifiers/v1/cli/debug.py
@@ -0,0 +1,267 @@
+"""The debug entrypoint: setup tasks, run one shell action, and persist traces."""
+
+import asyncio
+import contextlib
+import logging
+import random
+import shlex
+import signal
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.output import append_trace, save_config
+from verifiers.v1.cli.resolve import (
+    extract_id,
+    references_config_file,
+    with_positional_taskset,
+)
+from verifiers.v1.configs.debug import DebugConfig
+from verifiers.v1.env import resolve_runtime_config
+from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
+from verifiers.v1.taskset import Taskset
+from verifiers.v1.trace import Trace
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = (
+    "usage: uv run debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
+    "[--runtime.type subprocess] [options] [@ file.toml]\n"
+    "       runs setup, then one command or uploaded host script, and saves traces"
+)
+
+
+def _narrow(argv: list[str]) -> type[DebugConfig]:
+    """`DebugConfig` with `taskset` narrowed to the config type of the id on the CLI."""
+    taskset_id = extract_id(argv, "taskset")
+    if not taskset_id:
+        return DebugConfig
+    ftype = vf.taskset_config_type(taskset_id)
+    return type(
+        DebugConfig.__name__,
+        (DebugConfig,),
+        {"__annotations__": {"taskset": ftype}, "taskset": ftype(id=taskset_id)},
+    )
+
+
+def output_path(config: DebugConfig) -> Path:
+    if config.output_dir is not None:
+        return config.output_dir
+    return Path("outputs") / f"{config.taskset.name}--debug" / config.uuid
+
+
+def tail(text: str, chars: int) -> str:
+    if not text or chars == 0:
+        return ""
+    return text[-chars:]
+
+
+def task_info(task) -> dict[str, Any]:
+    return {"idx": task.idx, "name": task.name, "workdir": task.workdir}
+
+
+def runtime_info(runtime: Runtime) -> dict[str, Any]:
+    return {
+        "type": runtime.type,
+        "name": runtime.name,
+        "descriptor": runtime.descriptor,
+    }
+
+
+def result_info(
+    result: ProgramResult,
+    start: float,
+    output_tail_chars: int,
+) -> dict[str, Any]:
+    ok = result.exit_code == 0
+    return {
+        "ok": ok,
+        "reason": "pass" if ok else "nonzero_exit",
+        "exit_code": result.exit_code,
+        "elapsed": round(time.time() - start, 2),
+        "stdout_tail": tail(result.stdout or "", output_tail_chars),
+        "stderr_tail": tail(result.stderr or "", output_tail_chars),
+    }
+
+
+def error_info(
+    error: Exception,
+    start: float,
+    timeout: float | None,
+    stage: str,
+) -> dict[str, Any]:
+    if isinstance(error, asyncio.TimeoutError):
+        message = (
+            f"{stage} timed out after {timeout}s"
+            if timeout is not None
+            else f"{stage} timed out"
+        )
+        reason = "timeout"
+    else:
+        message = str(error)
+        reason = "error"
+    return {
+        "ok": False,
+        "reason": reason,
+        "exit_code": None,
+        "elapsed": round(time.time() - start, 2),
+        "error": message,
+        "error_type": type(error).__name__,
+        "stdout_tail": "",
+        "stderr_tail": "",
+    }
+
+
+async def run_command(
+    runtime: Runtime,
+    command: str,
+    config: DebugConfig,
+) -> dict[str, Any]:
+    start = time.time()
+    try:
+        result = await asyncio.wait_for(
+            runtime.run(["sh", "-lc", command], {}),
+            config.timeout,
+        )
+    except Exception as e:
+        return error_info(e, start, config.timeout, "debug action")
+    return result_info(result, start, config.output_tail_chars)
+
+
+async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
+    if config.command is not None:
+        return {
+            "action": "command",
+            "command": config.command,
+            **(await run_command(runtime, config.command, config)),
+        }
+    assert config.script_path is not None
+    remote_path = config.remote_script_path
+    await runtime.write(remote_path, config.script_path.read_bytes())
+    command = f"chmod +x {shlex.quote(remote_path)} && {shlex.quote(remote_path)}"
+    return {
+        "action": "script",
+        "script_path": str(config.script_path),
+        "remote_script_path": remote_path,
+        "command": command,
+        **(await run_command(runtime, command, config)),
+    }
+
+
+async def debug_task(taskset: Taskset, task, config: DebugConfig) -> Trace:
+    trace = Trace(task=task)
+    debug = {
+        "task": task_info(task),
+        "action": "command" if config.command is not None else "script",
+        "ok": False,
+        "reason": "not_run",
+    }
+    runtime = make_runtime(
+        resolve_runtime_config(config.runtime, task), name=f"debug-{task.idx}"
+    )
+    setup_timeout = (
+        config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
+    )
+    try:
+        trace.timing.setup.start = time.time()
+        await runtime.start()
+        debug["runtime"] = runtime_info(runtime)
+        await asyncio.wait_for(taskset.setup(task, runtime), setup_timeout)
+        trace.timing.setup.end = time.time()
+
+        trace.timing.generation.start = time.time()
+        debug.update(await run_action(runtime, config))
+        trace.timing.generation.end = time.time()
+        trace.stop(str(debug["reason"]))
+    except Exception as e:
+        if not trace.timing.setup.end:
+            trace.timing.setup.end = time.time()
+        if trace.timing.generation.start and not trace.timing.generation.end:
+            trace.timing.generation.end = time.time()
+        stage = "debug action" if trace.timing.generation.start else "setup"
+        timeout = config.timeout if stage == "debug action" else setup_timeout
+        debug.update(
+            error_info(e, trace.timing.generation.start or time.time(), timeout, stage)
+        )
+        debug.setdefault("runtime", runtime_info(runtime))
+        trace.capture_error(e)
+    finally:
+        trace.info["debug"] = debug
+        try:
+            await runtime.stop()
+        except Exception:
+            logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
+    return trace
+
+
+async def run_debug(config: DebugConfig) -> list[Trace]:
+    taskset = vf.load_taskset(config.taskset)
+    tasks = taskset.load_tasks()
+    if config.shuffle:
+        random.Random(0).shuffle(tasks)
+    if config.num_tasks is not None:
+        tasks = tasks[: config.num_tasks]
+    if isinstance(config.runtime, vf.SubprocessConfig) and (
+        taskset.NEEDS_CONTAINER or any(t.image for t in tasks)
+    ):
+        raise SystemExit(
+            "taskset needs a container runtime to debug - pass --runtime.type docker (or prime)"
+        )
+
+    out = output_path(config)
+    save_config(config, out)
+    logger.info(
+        "debugging %d task(s) from %s on the %s runtime",
+        len(tasks),
+        config.name,
+        config.runtime.type,
+    )
+    logger.info("results: %s", out)
+
+    sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
+    write_lock = asyncio.Lock()
+
+    async def one(task) -> Trace:
+        async with sem or contextlib.nullcontext():
+            trace = await debug_task(taskset, task, config)
+        await append_trace(out, trace, write_lock)
+        info = trace.info["debug"]
+        detail = f" - {info['error']}" if info.get("error") else ""
+        logger.info(
+            "idx=%s ok=%s reason=%s%s",
+            task.idx,
+            info["ok"],
+            info["reason"],
+            detail,
+        )
+        return trace
+
+    return await asyncio.gather(*(one(task) for task in tasks))
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
+
+    if not argv or any(arg in ("-h", "--help") for arg in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        cli(_narrow(argv))
+        return
+    if not extract_id(argv, "taskset") and not references_config_file(argv):
+        raise SystemExit(USAGE)
+
+    config_type = _narrow(argv)
+    sys.argv = [sys.argv[0], *argv]
+    config = cli(config_type)
+    setup_logging("DEBUG" if config.verbose else "INFO")
+    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    asyncio.run(run_debug(config))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py
index 9e3118d092..c936a01c8d 100644
--- a/verifiers/v1/cli/output.py
+++ b/verifiers/v1/cli/output.py
@@ -15,7 +15,7 @@
 from pathlib import Path
 
 import tomli_w
-from pydantic import TypeAdapter
+from pydantic import BaseModel, TypeAdapter
 
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.trace import Trace
@@ -30,7 +30,7 @@ def output_path(config: EvalConfig) -> Path:
     return Path("outputs") / name / config.uuid
 
 
-def write_config(config: EvalConfig, results_dir: Path) -> Path:
+def write_config(config: BaseModel, results_dir: Path) -> Path:
     """Write the run's resolved `config.toml` (re-readable via `@ config.toml`); return its
     path. mode="json" makes values TOML-friendly (Path -> str, etc.); exclude_none drops the
     nulls TOML can't represent."""
@@ -41,7 +41,7 @@ def write_config(config: EvalConfig, results_dir: Path) -> Path:
     return config_path
 
 
-def save_config(config: EvalConfig, results_dir: Path) -> None:
+def save_config(config: BaseModel, results_dir: Path) -> None:
     """Set up the run's output dir: write `config.toml` and start a fresh (empty)
     `results.jsonl`. Call once up front, before traces start landing."""
     write_config(config, results_dir)
diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py
index 74506c00ed..083c184a2e 100644
--- a/verifiers/v1/cli/validate.py
+++ b/verifiers/v1/cli/validate.py
@@ -1,10 +1,9 @@
 """The validate entrypoint: `uv run validate <taskset-id> [--runtime.type subprocess] [options]`.
 
 Registered as the `validate` console script — the model-free sibling of `eval`. Where `eval`
-runs a model rollout per task, `validate` runs each task's `validate` hook: a per-task check
-that the ground truth holds (a SWE row's gold patch makes its tests pass, gsm8k's verifier
-accepts the gold answer), in a runtime with the taskset's `setup` already applied. Each task
-is provisioned, set up, validated, and torn down independently with bounded concurrency.
+runs a model rollout per task, `validate` can either run each task's `validate` hook, run
+setup only, or run both modes in independent runtimes. Each task is provisioned, set up,
+checked, and torn down independently with bounded concurrency.
 
 Fire-and-forget: progress is shown live (the `--rich` dashboard, or per-task log lines) and
 nothing is written to disk.
@@ -17,12 +16,12 @@
 import signal
 import sys
 import time
+from typing import Any
 
 from pydantic_config import cli
 
 import verifiers.v1 as vf
 from verifiers.v1.cli.dashboard import TaskProgress, validate_dashboard
-from verifiers.v1.utils.logging import setup_logging
 from verifiers.v1.cli.resolve import (
     extract_id,
     references_config_file,
@@ -32,12 +31,13 @@
 from verifiers.v1.env import resolve_runtime_config
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.taskset import Taskset
+from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
 
 USAGE = (
     "usage: uv run validate [<taskset-id>] [--runtime.type subprocess] [options] [@ file.toml]\n"
-    "       runs each task's `validate` hook (per-task validation, no model)"
+    "       runs setup-only, apply-answer, or both validation modes (no model)"
 )
 
 
@@ -56,6 +56,9 @@ def _narrow(argv: list[str]) -> type[ValidateConfig]:
     )
 
 
+ResultRow = dict[str, Any]
+
+
 def _classify(valid: bool, exc: BaseException | None) -> str:
     """The outcome reason: `valid` (passed), `invalid` (returned False), `timeout` (a stage
     timed out), or `error` (raised) — the error's message carries the detail."""
@@ -68,13 +71,29 @@ def _classify(valid: bool, exc: BaseException | None) -> str:
     return "invalid"
 
 
-async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> dict:
-    """Validate one task: provision its runtime, run `setup` then `validate` (each under its
-    stage timeout), tear the runtime down, and return the result row. A raised error is
-    captured onto the row (one bad task is data, not a crash) — never re-raised."""
+def _row(
+    task, mode: str, valid: bool, exc: BaseException | None, start: float
+) -> ResultRow:
+    return {
+        "index": task.idx,
+        "name": task.name,
+        "mode": mode,
+        "valid": bool(valid),
+        "reason": _classify(valid, exc),
+        "elapsed": round(time.time() - start, 2),
+        "error": str(exc) if exc is not None else None,
+        "error_type": type(exc).__name__ if exc is not None else None,
+    }
+
+
+async def _run_apply_answer(
+    taskset: Taskset, task, config: ValidateConfig
+) -> ResultRow:
+    """Provision one runtime, run setup + validate, tear it down, and return a row."""
     start = time.time()
     runtime = make_runtime(
-        resolve_runtime_config(config.runtime, task), name=f"validate-{task.idx}"
+        resolve_runtime_config(config.runtime, task),
+        name=f"validate-apply-answer-{task.idx}",
     )
     setup_timeout = (
         config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
@@ -93,17 +112,93 @@ async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> dict
             await runtime.stop()
         except Exception:
             logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
+    return _row(task, "apply-answer", valid, exc, start)
+
+
+async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    """Provision one runtime, run setup only, tear it down, and return a row."""
+    start = time.time()
+    runtime = make_runtime(
+        resolve_runtime_config(config.runtime, task), name=f"validate-noop-{task.idx}"
+    )
+    setup_timeout = (
+        config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
+    )
+    valid, exc = False, None
+    try:
+        await runtime.start()
+        await asyncio.wait_for(taskset.setup(task, runtime), setup_timeout)
+        valid = True
+    except Exception as e:
+        exc = e
+    finally:
+        try:
+            await runtime.stop()
+        except Exception:
+            logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
+    return _row(task, "noop", valid, exc, start)
+
+
+def _both_reason(apply_answer: ResultRow, noop: ResultRow) -> str:
+    reasons = {str(apply_answer["reason"]), str(noop["reason"])}
+    if apply_answer["valid"] and noop["valid"]:
+        return "valid"
+    if "error" in reasons:
+        return "error"
+    if "timeout" in reasons:
+        return "timeout"
+    return "invalid"
+
+
+def _both_error(
+    apply_answer: ResultRow, noop: ResultRow
+) -> tuple[str | None, str | None]:
+    failed = [
+        ("apply_answer", apply_answer),
+        ("noop", noop),
+    ]
+    parts = [
+        f"{name}: {row['error'] or row['reason']}"
+        for name, row in failed
+        if not row["valid"]
+    ]
+    error_types = {
+        str(row["error_type"])
+        for _, row in failed
+        if not row["valid"] and row["error_type"]
+    }
+    return ("; ".join(parts) or None, "+".join(sorted(error_types)) or None)
+
+
+async def _run_both(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    """Run apply-answer and noop as independent high-level validations."""
+    start = time.time()
+    apply_answer = await _run_apply_answer(taskset, task, config)
+    noop = await _run_noop(taskset, task, config)
+    valid = bool(apply_answer["valid"] and noop["valid"])
+    error, error_type = _both_error(apply_answer, noop)
     return {
         "index": task.idx,
         "name": task.name,
-        "valid": bool(valid),
-        "reason": _classify(valid, exc),
+        "mode": "both",
+        "valid": valid,
+        "reason": _both_reason(apply_answer, noop),
         "elapsed": round(time.time() - start, 2),
-        "error": str(exc) if exc is not None else None,
-        "error_type": type(exc).__name__ if exc is not None else None,
+        "error": error,
+        "error_type": error_type,
+        "apply_answer": apply_answer,
+        "noop": noop,
     }
 
 
+async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    if config.mode == "apply-answer":
+        return await _run_apply_answer(taskset, task, config)
+    if config.mode == "noop":
+        return await _run_noop(taskset, task, config)
+    return await _run_both(taskset, task, config)
+
+
 async def run_validate(config: ValidateConfig) -> list[dict]:
     """Run each task's `validate` hook with bounded concurrency, showing progress live. Returns
     the result rows in memory — nothing is persisted."""
@@ -120,10 +215,11 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
             "taskset needs a container runtime to validate - pass --runtime.type docker (or prime)"
         )
     logger.info(
-        "validating %d task(s) from %s on the %s runtime",
+        "validating %d task(s) from %s on the %s runtime (mode=%s)",
         len(tasks),
         config.name,
         config.runtime.type,
+        config.mode,
     )
 
     sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
diff --git a/verifiers/v1/configs/__init__.py b/verifiers/v1/configs/__init__.py
index 5a930106e8..06ab1259dd 100644
--- a/verifiers/v1/configs/__init__.py
+++ b/verifiers/v1/configs/__init__.py
@@ -1,8 +1,9 @@
 """Config schemas for the v1 entrypoints — the objects the CLIs parse."""
 
 from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.configs.debug import DebugConfig
 from verifiers.v1.configs.init import InitConfig
 from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.configs.validate import ValidateConfig
 
-__all__ = ["EvalConfig", "InitConfig", "ServeConfig", "ValidateConfig"]
+__all__ = ["DebugConfig", "EvalConfig", "InitConfig", "ServeConfig", "ValidateConfig"]
diff --git a/verifiers/v1/configs/debug.py b/verifiers/v1/configs/debug.py
new file mode 100644
index 0000000000..b606dbe304
--- /dev/null
+++ b/verifiers/v1/configs/debug.py
@@ -0,0 +1,78 @@
+"""The `DebugConfig`: setup a task runtime and run one command or host script."""
+
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
+from pydantic_config import BaseConfig
+
+from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
+from verifiers.v1.taskset import TasksetConfig
+
+
+class DebugConfig(BaseConfig):
+    """A taskset plus one setup-only debug action. The taskset is selected by a positional
+    id or `--taskset.id`; after setup, each task runs either an inline shell command or a
+    host script uploaded into the runtime. Results are persisted as traces."""
+
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id, used as the default output directory leaf."""
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    runtime: RuntimeConfig = DockerConfig()
+    """Where each task's setup hook and debug action run."""
+    command: str | None = None
+    """Inline shell command executed as `sh -lc <command>` after setup."""
+    script_path: Path | None = Field(
+        None, validation_alias=AliasChoices("script_path", "script")
+    )
+    """Host script to upload and execute after setup."""
+    remote_script_path: str = "/tmp/vf-debug-script.sh"
+    """Runtime path used for the uploaded host script."""
+    setup_timeout: float | None = None
+    """Max wall-clock for the taskset's `setup` hook per task (None = no limit)."""
+    timeout: float | None = None
+    """Max wall-clock for the debug command/script per task (None = no limit)."""
+    output_tail_chars: int = 2000
+    """How many trailing stdout/stderr characters to persist in `trace.info["debug"]`."""
+    num_tasks: int | None = Field(
+        None,
+        validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
+    )
+    """How many tasks to debug (None = all)."""
+    shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
+    """Shuffle tasks before taking the first `num_tasks`."""
+    max_concurrent: int | None = Field(
+        128, validation_alias=AliasChoices("max_concurrent", "c")
+    )
+    """Max tasks debugged in flight at once."""
+    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
+    """Log at debug level instead of the default info."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write `config.toml` and `results.jsonl`. None = a fresh per-run dir."""
+
+    @property
+    def name(self) -> str:
+        return self.taskset.name
+
+    @model_validator(mode="before")
+    @classmethod
+    def resolve_taskset(cls, data):
+        """Narrow the generic `taskset` to its concrete config type by id."""
+        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+
+        narrow_plugin_field(data, "taskset", taskset_config_type)
+        return data
+
+    @model_validator(mode="after")
+    def validate_action(self):
+        if bool(self.command) == bool(self.script_path):
+            raise ValueError("pass exactly one of `--command` or `--script-path`")
+        if self.script_path is not None and not self.script_path.is_file():
+            raise ValueError(
+                f"script_path does not exist or is not a file: {self.script_path}"
+            )
+        if self.output_tail_chars < 0:
+            raise ValueError("output_tail_chars must be non-negative")
+        return self
diff --git a/verifiers/v1/configs/validate.py b/verifiers/v1/configs/validate.py
index 3d5c1de43a..a964b2942e 100644
--- a/verifiers/v1/configs/validate.py
+++ b/verifiers/v1/configs/validate.py
@@ -8,12 +8,16 @@
 resume / dry-run.
 """
 
+from typing import Literal
+
 from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
 from verifiers.v1.taskset import TasksetConfig
 
+ValidateMode = Literal["apply-answer", "noop", "both"]
+
 
 class ValidateConfig(BaseConfig):
     """A taskset plus how to validate it. The taskset is selected by `--taskset.id` (with a
@@ -31,6 +35,9 @@ class ValidateConfig(BaseConfig):
     """Max wall-clock for the taskset's `setup` hook per task (None = no limit)."""
     validate_timeout: float | None = None
     """Max wall-clock for the taskset's `validate` hook per task (None = no limit)."""
+    mode: ValidateMode = "apply-answer"
+    """Validation mode: `apply-answer` runs setup + validate, `noop` runs setup only, and
+    `both` runs both modes in independent runtimes."""
     num_tasks: int | None = Field(
         None,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),

From e5b0e57c76a3fab33cfd0f0b7cb4c36e9da9a027 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 03:19:14 +0000
Subject: [PATCH 02/27] fix: report debug setup elapsed time

---
 verifiers/v1/cli/debug.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 1c15d18547..b21d7b6fed 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -185,9 +185,12 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> Trace:
             trace.timing.generation.end = time.time()
         stage = "debug action" if trace.timing.generation.start else "setup"
         timeout = config.timeout if stage == "debug action" else setup_timeout
-        debug.update(
-            error_info(e, trace.timing.generation.start or time.time(), timeout, stage)
+        error_start = (
+            trace.timing.generation.start
+            if stage == "debug action"
+            else trace.timing.setup.start
         )
+        debug.update(error_info(e, error_start, timeout, stage))
         debug.setdefault("runtime", runtime_info(runtime))
         trace.capture_error(e)
     finally:

From b8fd9a169605467f65e06f4dd5f40a1eecfc4fce Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 03:21:01 +0000
Subject: [PATCH 03/27] fix: persist cancelled debug traces

---
 verifiers/v1/cli/debug.py | 72 ++++++++++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 19 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index b21d7b6fed..82aabf8954 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -8,6 +8,7 @@
 import signal
 import sys
 import time
+import traceback
 from pathlib import Path
 from typing import Any
 
@@ -24,7 +25,7 @@
 from verifiers.v1.env import resolve_runtime_config
 from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.taskset import Taskset
-from verifiers.v1.trace import Trace
+from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
@@ -90,7 +91,7 @@ def result_info(
 
 
 def error_info(
-    error: Exception,
+    error: BaseException,
     start: float,
     timeout: float | None,
     stage: str,
@@ -102,6 +103,9 @@ def error_info(
             else f"{stage} timed out"
         )
         reason = "timeout"
+    elif isinstance(error, asyncio.CancelledError):
+        message = f"{stage} cancelled"
+        reason = "cancelled"
     else:
         message = str(error)
         reason = "error"
@@ -117,6 +121,43 @@ def error_info(
     }
 
 
+def capture_trace_error(trace: Trace, error: BaseException) -> None:
+    if isinstance(error, Exception):
+        trace.capture_error(error)
+        return
+    trace.errors.append(
+        Error(
+            type=type(error).__name__,
+            message=str(error),
+            traceback=traceback.format_exc(),
+        )
+    )
+    trace.stop("error")
+
+
+def record_debug_error(
+    trace: Trace,
+    debug: dict[str, Any],
+    runtime: Runtime,
+    error: BaseException,
+    setup_timeout: float | None,
+    action_timeout: float | None,
+) -> None:
+    if not trace.timing.setup.end:
+        trace.timing.setup.end = time.time()
+    if trace.timing.generation.start and not trace.timing.generation.end:
+        trace.timing.generation.end = time.time()
+    in_action = bool(trace.timing.generation.start)
+    stage = "debug action" if in_action else "setup"
+    timeout = action_timeout if in_action else setup_timeout
+    error_start = (
+        trace.timing.generation.start if in_action else trace.timing.setup.start
+    )
+    debug.update(error_info(error, error_start, timeout, stage))
+    debug.setdefault("runtime", runtime_info(runtime))
+    capture_trace_error(trace, error)
+
+
 async def run_command(
     runtime: Runtime,
     command: str,
@@ -153,7 +194,7 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
     }
 
 
-async def debug_task(taskset: Taskset, task, config: DebugConfig) -> Trace:
+async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(task=task)
     debug = {
         "task": task_info(task),
@@ -161,6 +202,7 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> Trace:
         "ok": False,
         "reason": "not_run",
     }
+    cancelled = False
     runtime = make_runtime(
         resolve_runtime_config(config.runtime, task), name=f"debug-{task.idx}"
     )
@@ -178,28 +220,18 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> Trace:
         debug.update(await run_action(runtime, config))
         trace.timing.generation.end = time.time()
         trace.stop(str(debug["reason"]))
+    except asyncio.CancelledError as e:
+        cancelled = True
+        record_debug_error(trace, debug, runtime, e, setup_timeout, config.timeout)
     except Exception as e:
-        if not trace.timing.setup.end:
-            trace.timing.setup.end = time.time()
-        if trace.timing.generation.start and not trace.timing.generation.end:
-            trace.timing.generation.end = time.time()
-        stage = "debug action" if trace.timing.generation.start else "setup"
-        timeout = config.timeout if stage == "debug action" else setup_timeout
-        error_start = (
-            trace.timing.generation.start
-            if stage == "debug action"
-            else trace.timing.setup.start
-        )
-        debug.update(error_info(e, error_start, timeout, stage))
-        debug.setdefault("runtime", runtime_info(runtime))
-        trace.capture_error(e)
+        record_debug_error(trace, debug, runtime, e, setup_timeout, config.timeout)
     finally:
         trace.info["debug"] = debug
         try:
             await runtime.stop()
         except Exception:
             logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
-    return trace
+    return trace, cancelled
 
 
 async def run_debug(config: DebugConfig) -> list[Trace]:
@@ -231,7 +263,7 @@ async def run_debug(config: DebugConfig) -> list[Trace]:
 
     async def one(task) -> Trace:
         async with sem or contextlib.nullcontext():
-            trace = await debug_task(taskset, task, config)
+            trace, cancelled = await debug_task(taskset, task, config)
         await append_trace(out, trace, write_lock)
         info = trace.info["debug"]
         detail = f" - {info['error']}" if info.get("error") else ""
@@ -242,6 +274,8 @@ async def one(task) -> Trace:
             info["reason"],
             detail,
         )
+        if cancelled:
+            raise asyncio.CancelledError
         return trace
 
     return await asyncio.gather(*(one(task) for task in tasks))

From 214ff8d4465a91080f7bed02d9248847e8a5541b Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 03:27:44 +0000
Subject: [PATCH 04/27] fix: update swe debug deprecation test

---
 tests/test_sandbox_debug_env.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_sandbox_debug_env.py b/tests/test_sandbox_debug_env.py
index cb8051d6a0..8577df11c2 100644
--- a/tests/test_sandbox_debug_env.py
+++ b/tests/test_sandbox_debug_env.py
@@ -33,7 +33,7 @@ def test_swe_debug_env_is_deprecated_subclass_not_alias():
     assert issubclass(SWEDebugEnv, SandboxDebugEnv)
     assert SandboxDebugRubric.__name__ == "SandboxDebugRubric"
 
-    with pytest.warns(DeprecationWarning, match="use SandboxDebugEnv"):
+    with pytest.warns(DeprecationWarning, match="native v1 `debug` CLI"):
         env = SWEDebugEnv(
             DummySandboxTaskSet(),
             debug_step="command",

From ed5efa5a6b57d2f3d0dca2f62dd43ea522e8bc2d Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 03:33:56 +0000
Subject: [PATCH 05/27] fix: address debug cli bugbot comments

---
 docs/evaluation.md                            | 26 ++++++++++++++++
 docs/reference.md                             | 31 +++++++++++++++++++
 tests/test_sandbox_debug_env.py               |  3 +-
 .../composable/sandbox_debug_env.py           | 15 +++++----
 .../experimental/composable/swe_debug_env.py  |  2 ++
 5 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/docs/evaluation.md b/docs/evaluation.md
index 3284552a30..fd984b7ab9 100644
--- a/docs/evaluation.md
+++ b/docs/evaluation.md
@@ -5,6 +5,7 @@ This section explains how to run evaluations with Verifiers environments. See [E
 ## Table of Contents
 - [Basic Usage](#basic-usage)
 - [Hosted Evaluations](#hosted-evaluations)
+- [V1 Taskset Validation and Debugging](#v1-taskset-validation-and-debugging)
 - [Command Reference](#command-reference)
   - [Environment Selection](#environment-selection)
   - [Model Configuration](#model-configuration)
@@ -49,6 +50,31 @@ prime eval run configs/eval/benchmark-hosted.toml --hosted
 
 For the full hosted workflow and hosted-only flags such as `--follow`, `--timeout-minutes`, `--allow-sandbox-access`, and `--custom-secrets`, see the official [Hosted Evaluations](https://docs.primeintellect.ai/tutorials-environments/hosted-evaluations) guide.
 
+## V1 Taskset Validation and Debugging
+
+Native v1 tasksets also ship model-free developer CLIs. Use `uv run validate`
+to run a taskset's setup and validation hook without sampling a model, and use
+`--mode noop` to check setup only. `--mode both` runs `apply-answer` and `noop`
+as independent validations and reports one aggregate result.
+
+```bash
+uv run validate gsm8k-v1 -n 5 --runtime.type subprocess
+uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
+```
+
+Use `uv run debug` to start each selected task runtime, call the taskset setup
+hook, run one explicit shell action, and save `config.toml` plus `results.jsonl`.
+The debug CLI accepts either an inline command or a host script path and records
+the command/script diagnostics in `trace.info["debug"]`.
+
+```bash
+uv run debug swebench-v1 -n 1 --runtime.type prime --command 'pwd; git status --short'
+uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
+```
+
+See [the v1 CLI reference](../verifiers/v1/GUIDE.md#cli-reference) for the
+full `eval`, `validate`, `debug`, and `init` command surface.
+
 ## Command Reference
 
 ### Environment Selection
diff --git a/docs/reference.md b/docs/reference.md
index 6b657d7221..be2f82a1b0 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -10,6 +10,7 @@
   - [Rubric Classes](#rubric-classes)
 - [Client Classes](#client-classes)
 - [Configuration Types](#configuration-types)
+- [V1 Console Scripts](#v1-console-scripts)
 - [Prime CLI Plugin](#prime-cli-plugin)
 - [Decorators](#decorators)
 - [Utility Functions](#utility-functions)
@@ -910,6 +911,36 @@ materialized API key.
 
 ---
 
+## V1 Console Scripts
+
+The v1 taskset/harness runtime exposes repo-local console scripts through
+`pyproject.toml`. Run them as `uv run <command>` from a checkout or an
+environment where `verifiers` is installed.
+
+| Command | Description |
+|---------|-------------|
+| `eval <taskset-id>` | Run and score model rollouts for a v1 taskset. |
+| `validate <taskset-id>` | Run setup plus the taskset `validate` hook without a model. |
+| `debug <taskset-id>` | Run setup plus one explicit shell command or uploaded host script and save traces. |
+| `init <name>` | Scaffold a v1 environment package. |
+| `serve` | Serve a v1 environment over the framework protocol. |
+
+`validate` supports `--mode apply-answer` (default), `--mode noop` (setup
+only), and `--mode both` (independent apply-answer and noop runs with nested
+subresults). `debug` requires exactly one of `--command` or `--script-path` and
+persists command/script diagnostics under `trace.info["debug"]` in
+`results.jsonl`.
+
+```bash
+uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
+uv run debug swebench-v1 -n 1 --runtime.type prime --command 'pwd; git status --short'
+```
+
+See [the v1 CLI reference](../verifiers/v1/GUIDE.md#cli-reference) for the
+full flag tables and TOML examples.
+
+---
+
 ## Prime CLI Plugin
 
 Verifiers exposes a plugin contract consumed by `prime` for command execution.
diff --git a/tests/test_sandbox_debug_env.py b/tests/test_sandbox_debug_env.py
index 8577df11c2..f81097383c 100644
--- a/tests/test_sandbox_debug_env.py
+++ b/tests/test_sandbox_debug_env.py
@@ -33,12 +33,13 @@ def test_swe_debug_env_is_deprecated_subclass_not_alias():
     assert issubclass(SWEDebugEnv, SandboxDebugEnv)
     assert SandboxDebugRubric.__name__ == "SandboxDebugRubric"
 
-    with pytest.warns(DeprecationWarning, match="native v1 `debug` CLI"):
+    with pytest.warns(DeprecationWarning, match="native v1 `debug` CLI") as warnings:
         env = SWEDebugEnv(
             DummySandboxTaskSet(),
             debug_step="command",
             debug_command="true",
         )
 
+    assert len(warnings) == 1
     assert isinstance(env, SandboxDebugEnv)
     assert env.labels == ["sandbox-debug"]
diff --git a/verifiers/envs/experimental/composable/sandbox_debug_env.py b/verifiers/envs/experimental/composable/sandbox_debug_env.py
index dae2568101..450786597b 100644
--- a/verifiers/envs/experimental/composable/sandbox_debug_env.py
+++ b/verifiers/envs/experimental/composable/sandbox_debug_env.py
@@ -2,7 +2,7 @@
 
 import shlex
 import time
-from typing import Any, Literal
+from typing import Any, ClassVar, Literal
 from warnings import warn
 
 import verifiers as vf
@@ -39,6 +39,8 @@ class SandboxDebugEnv(SandboxMixin, vf.MultiTurnEnv):
     - exit: optionally run task tests and score them
     """
 
+    emit_deprecation_warning: ClassVar[bool] = True
+
     def __init__(
         self,
         taskset: SandboxTaskSet,
@@ -60,11 +62,12 @@ def __init__(
         output_tail_chars: int = 2000,
         **sandbox_kwargs: Any,
     ):
-        warn(
-            "SandboxDebugEnv is deprecated; use the native v1 `debug` CLI for v1 tasksets.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
+        if self.emit_deprecation_warning:
+            warn(
+                "SandboxDebugEnv is deprecated; use the native v1 `debug` CLI for v1 tasksets.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
         if debug_step not in ("none", "gold_patch", "command", "script"):
             raise ValueError(f"Unsupported debug_step: {debug_step!r}")
         if debug_step == "command" and not debug_command:
diff --git a/verifiers/envs/experimental/composable/swe_debug_env.py b/verifiers/envs/experimental/composable/swe_debug_env.py
index e374ccb72a..e68d5e5926 100644
--- a/verifiers/envs/experimental/composable/swe_debug_env.py
+++ b/verifiers/envs/experimental/composable/swe_debug_env.py
@@ -22,6 +22,8 @@ def __init__(self, **kwargs: Any):
 class SWEDebugEnv(SandboxDebugEnv):
     """Deprecated wrapper for SandboxDebugEnv."""
 
+    emit_deprecation_warning = False
+
     def __init__(self, *args: Any, **kwargs: Any):
         warn(
             "SWEDebugEnv is deprecated; use the native v1 `debug` CLI for v1 tasksets.",

From e6515f3d0c24929351c8101b745cee06c693c311 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 1 Jul 2026 03:40:45 +0000
Subject: [PATCH 06/27] fix: address remaining debug review comments

---
 skills/evaluate-environments/SKILL.md | 14 ++++++++++++++
 verifiers/v1/cli/debug.py             | 13 +++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/skills/evaluate-environments/SKILL.md b/skills/evaluate-environments/SKILL.md
index e0e7860213..bac0d7a1fe 100644
--- a/skills/evaluate-environments/SKILL.md
+++ b/skills/evaluate-environments/SKILL.md
@@ -14,6 +14,20 @@ Run reliable environment evaluations and produce actionable summaries, not raw l
 3. Standard `prime eval run` runs save results automatically, keeping them available in the user's private Evaluations tab and locally in `prime eval view`.
 4. For Prime Inference models with available pricing, eval output and saved metadata include estimated total-run USD cost automatically; no extra flags or API-key handling are needed.
 
+## Native V1 Model-Free Checks
+1. For native v1 tasksets, use `uv run validate` before or alongside model smoke tests when the task setup, gold answer, or verifier contract is the risk.
+2. Use `--mode apply-answer` for the normal setup plus `taskset.validate()` path, `--mode noop` for setup-only checks, and `--mode both` to run those paths independently:
+```bash
+uv run validate my-task-v1 -n 1 --runtime.type subprocess --mode both
+```
+3. Use `uv run debug` when you need to inspect a prepared task runtime without sampling a model. Provide exactly one inline command or host script path:
+```bash
+uv run debug my-task-v1 -n 1 --runtime.type docker --command 'pwd; ls -la'
+uv run debug my-task-v1 -n 1 --runtime.type docker --script-path ./inspect.sh
+```
+4. Read `debug` results from the saved `results.jsonl`; command/script metadata, exit status, elapsed time, timeout/error details, and stdout/stderr tails live under `trace.info["debug"]`.
+5. These checks complement `prime eval run`; they do not replace model rollouts when the user asks to measure model behavior or benchmark performance.
+
 ## Core Loop
 1. Run a smoke evaluation first (do not require pre-install):
 ```bash
diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 82aabf8954..86c89da7d5 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -158,6 +158,17 @@ def record_debug_error(
     capture_trace_error(trace, error)
 
 
+def record_action_failure(trace: Trace, debug: dict[str, Any]) -> None:
+    message = debug.get("error") or f"debug action failed: {debug['reason']}"
+    trace.errors.append(
+        Error(
+            type=debug.get("error_type") or "DebugActionError",
+            message=str(message),
+            traceback=None,
+        )
+    )
+
+
 async def run_command(
     runtime: Runtime,
     command: str,
@@ -219,6 +230,8 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace
         trace.timing.generation.start = time.time()
         debug.update(await run_action(runtime, config))
         trace.timing.generation.end = time.time()
+        if not debug.get("ok"):
+            record_action_failure(trace, debug)
         trace.stop(str(debug["reason"]))
     except asyncio.CancelledError as e:
         cancelled = True

From 720ddd3cc11bee893b4421c0f2ba76d4bdc14dfd Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Thu, 2 Jul 2026 22:36:02 +0000
Subject: [PATCH 07/27] fix: key debug action validation on presence, matching
 run_action dispatch

An empty --command alongside --script-path previously passed the
exactly-one check (bool("") is falsy) but run_action dispatches on
`command is not None`, so the empty command ran and the script was
silently ignored. Validate on `is None` so validation and execution
agree.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/configs/debug.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/verifiers/v1/configs/debug.py b/verifiers/v1/configs/debug.py
index b606dbe304..78621e6463 100644
--- a/verifiers/v1/configs/debug.py
+++ b/verifiers/v1/configs/debug.py
@@ -67,7 +67,7 @@ def resolve_taskset(cls, data):
 
     @model_validator(mode="after")
     def validate_action(self):
-        if bool(self.command) == bool(self.script_path):
+        if (self.command is None) == (self.script_path is None):
             raise ValueError("pass exactly one of `--command` or `--script-path`")
         if self.script_path is not None and not self.script_path.is_file():
             raise ValueError(

From 30bf2769e3f2d24ff7052e4e16150fbe856f8658 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:56:12 +0000
Subject: [PATCH 08/27] fix: make debug/validate runtime names unique per run

SubprocessRuntime.start() mkdirs /tmp/<name>, so the fixed
debug-{task.idx} name failed on a concurrent session or a stale
workdir left by a failed teardown. Suffix debug and validate
runtime names with a short uuid, matching eval's per-rollout
unique naming.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/debug.py    | 4 +++-
 verifiers/v1/cli/validate.py | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 86c89da7d5..6251eaa6d7 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -11,6 +11,7 @@
 import traceback
 from pathlib import Path
 from typing import Any
+from uuid import uuid4
 
 from pydantic_config import cli
 
@@ -215,7 +216,8 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace
     }
     cancelled = False
     runtime = make_runtime(
-        resolve_runtime_config(config.runtime, task), name=f"debug-{task.idx}"
+        resolve_runtime_config(config.runtime, task),
+        name=f"debug-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
         config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py
index 083c184a2e..7db19afcbb 100644
--- a/verifiers/v1/cli/validate.py
+++ b/verifiers/v1/cli/validate.py
@@ -17,6 +17,7 @@
 import sys
 import time
 from typing import Any
+from uuid import uuid4
 
 from pydantic_config import cli
 
@@ -93,7 +94,7 @@ async def _run_apply_answer(
     start = time.time()
     runtime = make_runtime(
         resolve_runtime_config(config.runtime, task),
-        name=f"validate-apply-answer-{task.idx}",
+        name=f"validate-apply-answer-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
         config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
@@ -119,7 +120,8 @@ async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow
     """Provision one runtime, run setup only, tear it down, and return a row."""
     start = time.time()
     runtime = make_runtime(
-        resolve_runtime_config(config.runtime, task), name=f"validate-noop-{task.idx}"
+        resolve_runtime_config(config.runtime, task),
+        name=f"validate-noop-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
         config.setup_timeout if config.setup_timeout is not None else task.timeout.setup

From d3a8fd94ff6ac426a2aedfeb5bb471f7cd9011b5 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:21:24 +0000
Subject: [PATCH 09/27] docs: keep v1 CLI docs out of public docs/ for now

Public docs/ are still v0-facing (the v1 migration is tracked in
#1916); drop the V1 sections added to docs/evaluation.md and
docs/reference.md. The validate/debug CLI docs live in
verifiers/v1/GUIDE.md and README.md. The SandboxDebugEnv deprecation
note stays: the class warns at runtime either way.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/evaluation.md | 26 --------------------------
 docs/reference.md  | 31 -------------------------------
 2 files changed, 57 deletions(-)

diff --git a/docs/evaluation.md b/docs/evaluation.md
index fd984b7ab9..3284552a30 100644
--- a/docs/evaluation.md
+++ b/docs/evaluation.md
@@ -5,7 +5,6 @@ This section explains how to run evaluations with Verifiers environments. See [E
 ## Table of Contents
 - [Basic Usage](#basic-usage)
 - [Hosted Evaluations](#hosted-evaluations)
-- [V1 Taskset Validation and Debugging](#v1-taskset-validation-and-debugging)
 - [Command Reference](#command-reference)
   - [Environment Selection](#environment-selection)
   - [Model Configuration](#model-configuration)
@@ -50,31 +49,6 @@ prime eval run configs/eval/benchmark-hosted.toml --hosted
 
 For the full hosted workflow and hosted-only flags such as `--follow`, `--timeout-minutes`, `--allow-sandbox-access`, and `--custom-secrets`, see the official [Hosted Evaluations](https://docs.primeintellect.ai/tutorials-environments/hosted-evaluations) guide.
 
-## V1 Taskset Validation and Debugging
-
-Native v1 tasksets also ship model-free developer CLIs. Use `uv run validate`
-to run a taskset's setup and validation hook without sampling a model, and use
-`--mode noop` to check setup only. `--mode both` runs `apply-answer` and `noop`
-as independent validations and reports one aggregate result.
-
-```bash
-uv run validate gsm8k-v1 -n 5 --runtime.type subprocess
-uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
-```
-
-Use `uv run debug` to start each selected task runtime, call the taskset setup
-hook, run one explicit shell action, and save `config.toml` plus `results.jsonl`.
-The debug CLI accepts either an inline command or a host script path and records
-the command/script diagnostics in `trace.info["debug"]`.
-
-```bash
-uv run debug swebench-v1 -n 1 --runtime.type prime --command 'pwd; git status --short'
-uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
-```
-
-See [the v1 CLI reference](../verifiers/v1/GUIDE.md#cli-reference) for the
-full `eval`, `validate`, `debug`, and `init` command surface.
-
 ## Command Reference
 
 ### Environment Selection
diff --git a/docs/reference.md b/docs/reference.md
index be2f82a1b0..6b657d7221 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -10,7 +10,6 @@
   - [Rubric Classes](#rubric-classes)
 - [Client Classes](#client-classes)
 - [Configuration Types](#configuration-types)
-- [V1 Console Scripts](#v1-console-scripts)
 - [Prime CLI Plugin](#prime-cli-plugin)
 - [Decorators](#decorators)
 - [Utility Functions](#utility-functions)
@@ -911,36 +910,6 @@ materialized API key.
 
 ---
 
-## V1 Console Scripts
-
-The v1 taskset/harness runtime exposes repo-local console scripts through
-`pyproject.toml`. Run them as `uv run <command>` from a checkout or an
-environment where `verifiers` is installed.
-
-| Command | Description |
-|---------|-------------|
-| `eval <taskset-id>` | Run and score model rollouts for a v1 taskset. |
-| `validate <taskset-id>` | Run setup plus the taskset `validate` hook without a model. |
-| `debug <taskset-id>` | Run setup plus one explicit shell command or uploaded host script and save traces. |
-| `init <name>` | Scaffold a v1 environment package. |
-| `serve` | Serve a v1 environment over the framework protocol. |
-
-`validate` supports `--mode apply-answer` (default), `--mode noop` (setup
-only), and `--mode both` (independent apply-answer and noop runs with nested
-subresults). `debug` requires exactly one of `--command` or `--script-path` and
-persists command/script diagnostics under `trace.info["debug"]` in
-`results.jsonl`.
-
-```bash
-uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
-uv run debug swebench-v1 -n 1 --runtime.type prime --command 'pwd; git status --short'
-```
-
-See [the v1 CLI reference](../verifiers/v1/GUIDE.md#cli-reference) for the
-full flag tables and TOML examples.
-
----
-
 ## Prime CLI Plugin
 
 Verifiers exposes a plugin contract consumed by `prime` for command execution.

From e371469a1334dc7b952cb57e5edf1b1fd3bc6665 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:21:40 +0000
Subject: [PATCH 10/27] docs: drop v1 model-free checks from
 evaluate-environments skill for now

Per review: reintroduce once v1 goes public. The validate/debug
guidance lives in verifiers/v1/GUIDE.md meanwhile.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 skills/evaluate-environments/SKILL.md | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/skills/evaluate-environments/SKILL.md b/skills/evaluate-environments/SKILL.md
index bac0d7a1fe..e0e7860213 100644
--- a/skills/evaluate-environments/SKILL.md
+++ b/skills/evaluate-environments/SKILL.md
@@ -14,20 +14,6 @@ Run reliable environment evaluations and produce actionable summaries, not raw l
 3. Standard `prime eval run` runs save results automatically, keeping them available in the user's private Evaluations tab and locally in `prime eval view`.
 4. For Prime Inference models with available pricing, eval output and saved metadata include estimated total-run USD cost automatically; no extra flags or API-key handling are needed.
 
-## Native V1 Model-Free Checks
-1. For native v1 tasksets, use `uv run validate` before or alongside model smoke tests when the task setup, gold answer, or verifier contract is the risk.
-2. Use `--mode apply-answer` for the normal setup plus `taskset.validate()` path, `--mode noop` for setup-only checks, and `--mode both` to run those paths independently:
-```bash
-uv run validate my-task-v1 -n 1 --runtime.type subprocess --mode both
-```
-3. Use `uv run debug` when you need to inspect a prepared task runtime without sampling a model. Provide exactly one inline command or host script path:
-```bash
-uv run debug my-task-v1 -n 1 --runtime.type docker --command 'pwd; ls -la'
-uv run debug my-task-v1 -n 1 --runtime.type docker --script-path ./inspect.sh
-```
-4. Read `debug` results from the saved `results.jsonl`; command/script metadata, exit status, elapsed time, timeout/error details, and stdout/stderr tails live under `trace.info["debug"]`.
-5. These checks complement `prime eval run`; they do not replace model rollouts when the user asks to measure model behavior or benchmark performance.
-
 ## Core Loop
 1. Run a smoke evaluation first (do not require pre-install):
 ```bash

From 9624b65bc5cc196550d58834b8b82e4b30997560 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:22:16 +0000
Subject: [PATCH 11/27] refactor: make the remote script path a constant, not a
 config field

Per review, no reason for this to be a knob: /tmp is writable in any
task image. Traces still record the path under
trace.info["debug"]["remote_script_path"].

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/debug.py     | 11 +++++++----
 verifiers/v1/configs/debug.py |  2 --
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index ad4d7e6e76..3d9a94ed1d 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -33,6 +33,9 @@
 
 logger = logging.getLogger(__name__)
 
+REMOTE_SCRIPT_PATH = "/tmp/vf-debug-script.sh"
+"""Runtime path the uploaded host script is written to and executed from."""
+
 USAGE = (
     "usage: uv run debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
     "[--runtime.type subprocess] [options] [@ file.toml]\n"
@@ -196,13 +199,13 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
             **(await run_command(runtime, config.command, config)),
         }
     assert config.script_path is not None
-    remote_path = config.remote_script_path
-    await runtime.write(remote_path, config.script_path.read_bytes())
-    command = f"chmod +x {shlex.quote(remote_path)} && {shlex.quote(remote_path)}"
+    await runtime.write(REMOTE_SCRIPT_PATH, config.script_path.read_bytes())
+    quoted = shlex.quote(REMOTE_SCRIPT_PATH)
+    command = f"chmod +x {quoted} && {quoted}"
     return {
         "action": "script",
         "script_path": str(config.script_path),
-        "remote_script_path": remote_path,
+        "remote_script_path": REMOTE_SCRIPT_PATH,
         "command": command,
         **(await run_command(runtime, command, config)),
     }
diff --git a/verifiers/v1/configs/debug.py b/verifiers/v1/configs/debug.py
index 78621e6463..100df25eab 100644
--- a/verifiers/v1/configs/debug.py
+++ b/verifiers/v1/configs/debug.py
@@ -26,8 +26,6 @@ class DebugConfig(BaseConfig):
         None, validation_alias=AliasChoices("script_path", "script")
     )
     """Host script to upload and execute after setup."""
-    remote_script_path: str = "/tmp/vf-debug-script.sh"
-    """Runtime path used for the uploaded host script."""
     setup_timeout: float | None = None
     """Max wall-clock for the taskset's `setup` hook per task (None = no limit)."""
     timeout: float | None = None

From ad5dc7944b7713107d621a19be2a30548820d1e7 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:24:00 +0000
Subject: [PATCH 12/27] refactor: bundle validate/debug timeouts into nested
 --timeout.* flags

Replace the flat --setup-timeout/--validate-timeout (validate) and
--setup-timeout/--timeout (debug) with a shared CheckTimeoutConfig:
--timeout.setup and --timeout.total, mirroring eval's nested
--timeout.* TOML shape. Renaming eval's --timeout.rollout to
--timeout.total is deferred to a follow-up since that flag has
downstream consumers.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/GUIDE.md            |  4 ++--
 verifiers/v1/cli/debug.py        | 14 +++++++++-----
 verifiers/v1/cli/validate.py     |  6 +++---
 verifiers/v1/configs/debug.py    |  8 ++++----
 verifiers/v1/configs/validate.py | 18 ++++++++++++++----
 5 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md
index fad994d38d..61022d95a8 100644
--- a/verifiers/v1/GUIDE.md
+++ b/verifiers/v1/GUIDE.md
@@ -752,7 +752,7 @@ uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
 | `<taskset-id>` / `--taskset.id` | — | taskset to validate |
 | `--runtime.type` | `docker` | runtime for `setup` + `validate` (a gold check often needs the task's container) |
 | `--mode` | `apply-answer` | `apply-answer`, `noop`, or `both` |
-| `--setup-timeout` / `--validate-timeout` | None | per-hook wall-clock caps |
+| `--timeout.setup` / `--timeout.total` | None | per-task wall-clock caps for `setup` and the `validate` hook |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
 | `-v`/`--verbose`, `--no-rich` | | logging / disable the dashboard |
 
@@ -778,7 +778,7 @@ uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
 | `<taskset-id>` / `--taskset.id` | — | taskset to debug |
 | `--command` / `--script-path` | — | exactly one inline command or host script to upload and execute |
 | `--runtime.type` | `docker` | runtime for setup + the debug action |
-| `--setup-timeout` / `--timeout` | None | setup and action wall-clock caps |
+| `--timeout.setup` / `--timeout.total` | None | per-task wall-clock caps for `setup` and the debug action |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
 | `-o`/`--output-dir` | fresh debug run dir | where `config.toml` and `results.jsonl` are written |
 
diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 3d9a94ed1d..855cc39d95 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -184,10 +184,10 @@ async def run_command(
     try:
         result = await asyncio.wait_for(
             runtime.run(["sh", "-lc", command], {}),
-            config.timeout,
+            config.timeout.total,
         )
     except Exception as e:
-        return error_info(e, start, config.timeout, "debug action")
+        return error_info(e, start, config.timeout.total, "debug action")
     return result_info(result, start, config.output_tail_chars)
 
 
@@ -225,7 +225,7 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace
         name=f"debug-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
-        config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
+        config.timeout.setup if config.timeout.setup is not None else task.timeout.setup
     )
     try:
         trace.timing.setup.start = time.time()
@@ -245,9 +245,13 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace
         trace.stop(str(debug["reason"]))
     except asyncio.CancelledError as e:
         cancelled = True
-        record_debug_error(trace, debug, runtime, e, setup_timeout, config.timeout)
+        record_debug_error(
+            trace, debug, runtime, e, setup_timeout, config.timeout.total
+        )
     except Exception as e:
-        record_debug_error(trace, debug, runtime, e, setup_timeout, config.timeout)
+        record_debug_error(
+            trace, debug, runtime, e, setup_timeout, config.timeout.total
+        )
     finally:
         trace.info["debug"] = debug
         try:
diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py
index c734a9f790..9efca90a48 100644
--- a/verifiers/v1/cli/validate.py
+++ b/verifiers/v1/cli/validate.py
@@ -100,7 +100,7 @@ async def _run_apply_answer(
         name=f"validate-apply-answer-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
-        config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
+        config.timeout.setup if config.timeout.setup is not None else task.timeout.setup
     )
     valid, exc = False, None
     try:
@@ -111,7 +111,7 @@ async def _run_apply_answer(
             setup_timeout,
         )
         valid = await asyncio.wait_for(
-            taskset.validate(task, runtime), config.validate_timeout
+            taskset.validate(task, runtime), config.timeout.total
         )
     except Exception as e:
         exc = e
@@ -131,7 +131,7 @@ async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow
         name=f"validate-noop-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
-        config.setup_timeout if config.setup_timeout is not None else task.timeout.setup
+        config.timeout.setup if config.timeout.setup is not None else task.timeout.setup
     )
     valid, exc = False, None
     try:
diff --git a/verifiers/v1/configs/debug.py b/verifiers/v1/configs/debug.py
index 100df25eab..c5bc1a349f 100644
--- a/verifiers/v1/configs/debug.py
+++ b/verifiers/v1/configs/debug.py
@@ -6,6 +6,7 @@
 from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
+from verifiers.v1.configs.validate import CheckTimeoutConfig
 from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
 from verifiers.v1.taskset import TasksetConfig
 
@@ -26,10 +27,9 @@ class DebugConfig(BaseConfig):
         None, validation_alias=AliasChoices("script_path", "script")
     )
     """Host script to upload and execute after setup."""
-    setup_timeout: float | None = None
-    """Max wall-clock for the taskset's `setup` hook per task (None = no limit)."""
-    timeout: float | None = None
-    """Max wall-clock for the debug command/script per task (None = no limit)."""
+    timeout: CheckTimeoutConfig = CheckTimeoutConfig()
+    """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
+    for the debug command/script."""
     output_tail_chars: int = 2000
     """How many trailing stdout/stderr characters to persist in `trace.info["debug"]`."""
     num_tasks: int | None = Field(
diff --git a/verifiers/v1/configs/validate.py b/verifiers/v1/configs/validate.py
index a964b2942e..0dca164607 100644
--- a/verifiers/v1/configs/validate.py
+++ b/verifiers/v1/configs/validate.py
@@ -19,6 +19,17 @@
 ValidateMode = Literal["apply-answer", "noop", "both"]
 
 
+class CheckTimeoutConfig(BaseConfig):
+    """Per-task wall-clock timeouts for the model-free check CLIs (validate/debug), in
+    seconds (None = no limit). The nested shape mirrors eval's `--timeout.*` flags."""
+
+    setup: float | None = None
+    """Max wall-clock for the taskset's `setup` hook per task."""
+    total: float | None = None
+    """Max wall-clock for the check itself per task — the `validate` hook, or the debug
+    command/script."""
+
+
 class ValidateConfig(BaseConfig):
     """A taskset plus how to validate it. The taskset is selected by `--taskset.id` (with a
     bare positional shorthand, `validate gsm8k-v1`); its fields stay typed and overridable
@@ -31,10 +42,9 @@ class ValidateConfig(BaseConfig):
     (subprocess), a gold check often needs the task's declared container; a SWE task overrides
     the image per task. Use `--runtime.type subprocess` for a check that needs no container —
     one that only reads task data or runs a uv-script verifier (e.g. gsm8k)."""
-    setup_timeout: float | None = None
-    """Max wall-clock for the taskset's `setup` hook per task (None = no limit)."""
-    validate_timeout: float | None = None
-    """Max wall-clock for the taskset's `validate` hook per task (None = no limit)."""
+    timeout: CheckTimeoutConfig = CheckTimeoutConfig()
+    """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
+    for the `validate` hook."""
     mode: ValidateMode = "apply-answer"
     """Validation mode: `apply-answer` runs setup + validate, `noop` runs setup only, and
     `both` runs both modes in independent runtimes."""

From becda842badc72dbc7592d0fc5a7ae72d9584b3b Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:25:16 +0000
Subject: [PATCH 13/27] refactor: persist full debug stdout/stderr, drop
 output_tail_chars

Per review: a debug CLI should save everything the action printed,
not a configurable 2000-char tail. trace.info["debug"] keys are
renamed stdout_tail/stderr_tail -> stdout/stderr accordingly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/GUIDE.md         |  2 +-
 verifiers/v1/cli/debug.py     | 22 ++++++----------------
 verifiers/v1/configs/debug.py |  4 ----
 3 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md
index 61022d95a8..0a95678595 100644
--- a/verifiers/v1/GUIDE.md
+++ b/verifiers/v1/GUIDE.md
@@ -783,7 +783,7 @@ uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
 | `-o`/`--output-dir` | fresh debug run dir | where `config.toml` and `results.jsonl` are written |
 
 Each saved trace has command/script metadata, exit status, elapsed time, timeout/error fields,
-and stdout/stderr tails under `trace.info["debug"]`.
+and the full stdout/stderr under `trace.info["debug"]`.
 
 ## `init`
 
diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 855cc39d95..057f4a91d6 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -62,12 +62,6 @@ def output_path(config: DebugConfig) -> Path:
     return Path("outputs") / f"{config.taskset.name}--debug" / config.uuid
 
 
-def tail(text: str, chars: int) -> str:
-    if not text or chars == 0:
-        return ""
-    return text[-chars:]
-
-
 def task_info(task) -> dict[str, Any]:
     return {"idx": task.idx, "name": task.name, "workdir": task.workdir}
 
@@ -80,19 +74,15 @@ def runtime_info(runtime: Runtime) -> dict[str, Any]:
     }
 
 
-def result_info(
-    result: ProgramResult,
-    start: float,
-    output_tail_chars: int,
-) -> dict[str, Any]:
+def result_info(result: ProgramResult, start: float) -> dict[str, Any]:
     ok = result.exit_code == 0
     return {
         "ok": ok,
         "reason": "pass" if ok else "nonzero_exit",
         "exit_code": result.exit_code,
         "elapsed": round(time.time() - start, 2),
-        "stdout_tail": tail(result.stdout or "", output_tail_chars),
-        "stderr_tail": tail(result.stderr or "", output_tail_chars),
+        "stdout": result.stdout or "",
+        "stderr": result.stderr or "",
     }
 
 
@@ -122,8 +112,8 @@ def error_info(
         "elapsed": round(time.time() - start, 2),
         "error": message,
         "error_type": type(error).__name__,
-        "stdout_tail": "",
-        "stderr_tail": "",
+        "stdout": "",
+        "stderr": "",
     }
 
 
@@ -188,7 +178,7 @@ async def run_command(
         )
     except Exception as e:
         return error_info(e, start, config.timeout.total, "debug action")
-    return result_info(result, start, config.output_tail_chars)
+    return result_info(result, start)
 
 
 async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
diff --git a/verifiers/v1/configs/debug.py b/verifiers/v1/configs/debug.py
index c5bc1a349f..39fc81e25a 100644
--- a/verifiers/v1/configs/debug.py
+++ b/verifiers/v1/configs/debug.py
@@ -30,8 +30,6 @@ class DebugConfig(BaseConfig):
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
     for the debug command/script."""
-    output_tail_chars: int = 2000
-    """How many trailing stdout/stderr characters to persist in `trace.info["debug"]`."""
     num_tasks: int | None = Field(
         None,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
@@ -71,6 +69,4 @@ def validate_action(self):
             raise ValueError(
                 f"script_path does not exist or is not a file: {self.script_path}"
             )
-        if self.output_tail_chars < 0:
-            raise ValueError("output_tail_chars must be non-negative")
         return self

From c3b08b25098fc53eacc36ef4801e3a4f0d2c44f4 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:26:45 +0000
Subject: [PATCH 14/27] refactor: rename validate mode both -> all

Per review: future-proofs the aggregate mode against a third
high-level validation. The aggregation helpers now take a list of
subresult rows instead of exactly two; the nested apply_answer/noop
keys are unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/GUIDE.md            |  6 ++--
 verifiers/v1/cli/validate.py     | 47 ++++++++++++--------------------
 verifiers/v1/configs/validate.py |  4 +--
 3 files changed, 22 insertions(+), 35 deletions(-)

diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md
index 0a95678595..f897b04abb 100644
--- a/verifiers/v1/GUIDE.md
+++ b/verifiers/v1/GUIDE.md
@@ -738,20 +738,20 @@ trace per line, appended as each rollout finishes — durable mid-run), and `eva
 
 Run each task's `validate` hook — a model-free check that the ground truth holds (the gold patch
 makes the tests pass, the verifier accepts the gold answer) — in a runtime with the taskset's
-`setup` applied. No model, no harness. Use `--mode noop` to run setup only, or `--mode both`
+`setup` applied. No model, no harness. Use `--mode noop` to run setup only, or `--mode all`
 to run `apply-answer` and `noop` in independent runtimes and report one aggregate row.
 
 ```bash
 uv run validate gsm8k-v1 -n 20 --runtime.type subprocess
 uv run validate swebench-v1 -n 1 --runtime.type prime --mode noop
-uv run validate swebench-v1 -n 1 --runtime.type prime --mode both
+uv run validate swebench-v1 -n 1 --runtime.type prime --mode all
 ```
 
 | flag | default | meaning |
 | --- | --- | --- |
 | `<taskset-id>` / `--taskset.id` | — | taskset to validate |
 | `--runtime.type` | `docker` | runtime for `setup` + `validate` (a gold check often needs the task's container) |
-| `--mode` | `apply-answer` | `apply-answer`, `noop`, or `both` |
+| `--mode` | `apply-answer` | `apply-answer`, `noop`, or `all` |
 | `--timeout.setup` / `--timeout.total` | None | per-task wall-clock caps for `setup` and the `validate` hook |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
 | `-v`/`--verbose`, `--no-rich` | | logging / disable the dashboard |
diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py
index 9efca90a48..f96943fb66 100644
--- a/verifiers/v1/cli/validate.py
+++ b/verifiers/v1/cli/validate.py
@@ -2,7 +2,7 @@
 
 Registered as the `validate` console script — the model-free sibling of `eval`. Where `eval`
 runs a model rollout per task, `validate` can either run each task's `validate` hook, run
-setup only, or run both modes in independent runtimes. Each task is provisioned, set up,
+setup only, or run all modes in independent runtimes. Each task is provisioned, set up,
 checked, and torn down independently with bounded concurrency.
 
 Fire-and-forget: progress is shown live (the `--rich` dashboard, or per-task log lines) and
@@ -41,7 +41,7 @@
 
 USAGE = (
     "usage: uv run validate [<taskset-id>] [--runtime.type subprocess] [options] [@ file.toml]\n"
-    "       runs setup-only, apply-answer, or both validation modes (no model)"
+    "       runs setup-only, apply-answer, or all validation modes (no model)"
 )
 
 
@@ -152,10 +152,10 @@ async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow
     return _row(task, "noop", valid, exc, start)
 
 
-def _both_reason(apply_answer: ResultRow, noop: ResultRow) -> str:
-    reasons = {str(apply_answer["reason"]), str(noop["reason"])}
-    if apply_answer["valid"] and noop["valid"]:
+def _all_reason(rows: list[ResultRow]) -> str:
+    if all(row["valid"] for row in rows):
         return "valid"
+    reasons = {str(row["reason"]) for row in rows}
     if "error" in reasons:
         return "error"
     if "timeout" in reasons:
@@ -163,39 +163,26 @@ def _both_reason(apply_answer: ResultRow, noop: ResultRow) -> str:
     return "invalid"
 
 
-def _both_error(
-    apply_answer: ResultRow, noop: ResultRow
-) -> tuple[str | None, str | None]:
-    failed = [
-        ("apply_answer", apply_answer),
-        ("noop", noop),
-    ]
-    parts = [
-        f"{name}: {row['error'] or row['reason']}"
-        for name, row in failed
-        if not row["valid"]
-    ]
-    error_types = {
-        str(row["error_type"])
-        for _, row in failed
-        if not row["valid"] and row["error_type"]
-    }
+def _all_error(rows: list[ResultRow]) -> tuple[str | None, str | None]:
+    failed = [row for row in rows if not row["valid"]]
+    parts = [f"{row['mode']}: {row['error'] or row['reason']}" for row in failed]
+    error_types = {str(row["error_type"]) for row in failed if row["error_type"]}
     return ("; ".join(parts) or None, "+".join(sorted(error_types)) or None)
 
 
-async def _run_both(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
-    """Run apply-answer and noop as independent high-level validations."""
+async def _run_all(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    """Run every mode (apply-answer, noop) as independent high-level validations."""
     start = time.time()
     apply_answer = await _run_apply_answer(taskset, task, config)
     noop = await _run_noop(taskset, task, config)
-    valid = bool(apply_answer["valid"] and noop["valid"])
-    error, error_type = _both_error(apply_answer, noop)
+    rows = [apply_answer, noop]
+    error, error_type = _all_error(rows)
     return {
         "index": task.idx,
         "name": task.name,
-        "mode": "both",
-        "valid": valid,
-        "reason": _both_reason(apply_answer, noop),
+        "mode": "all",
+        "valid": all(row["valid"] for row in rows),
+        "reason": _all_reason(rows),
         "elapsed": round(time.time() - start, 2),
         "error": error,
         "error_type": error_type,
@@ -209,7 +196,7 @@ async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> Resu
         return await _run_apply_answer(taskset, task, config)
     if config.mode == "noop":
         return await _run_noop(taskset, task, config)
-    return await _run_both(taskset, task, config)
+    return await _run_all(taskset, task, config)
 
 
 async def run_validate(config: ValidateConfig) -> list[dict]:
diff --git a/verifiers/v1/configs/validate.py b/verifiers/v1/configs/validate.py
index 0dca164607..d7dae17cec 100644
--- a/verifiers/v1/configs/validate.py
+++ b/verifiers/v1/configs/validate.py
@@ -16,7 +16,7 @@
 from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
 from verifiers.v1.taskset import TasksetConfig
 
-ValidateMode = Literal["apply-answer", "noop", "both"]
+ValidateMode = Literal["apply-answer", "noop", "all"]
 
 
 class CheckTimeoutConfig(BaseConfig):
@@ -47,7 +47,7 @@ class ValidateConfig(BaseConfig):
     for the `validate` hook."""
     mode: ValidateMode = "apply-answer"
     """Validation mode: `apply-answer` runs setup + validate, `noop` runs setup only, and
-    `both` runs both modes in independent runtimes."""
+    `all` runs every mode in independent runtimes and reports one aggregate row."""
     num_tasks: int | None = Field(
         None,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),

From 3a4fa3b2c3420636b9b64ac854ed5740b8f7cd9b Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 05:45:34 +0000
Subject: [PATCH 15/27] docs: revert the SandboxDebugEnv note in public docs
 entirely

Per review, keep docs/ untouched by this PR; the runtime
DeprecationWarning already points v1 users at the debug CLI.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/reference.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference.md b/docs/reference.md
index 6b657d7221..6bc4f49a1d 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -577,7 +577,7 @@ class SandboxDebugEnv(SandboxMixin, MultiTurnEnv):
     ): ...
 ```
 
-Deprecated no-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the task sandbox, optionally runs task setup, runs one debug step (`none`, `gold_patch`, `command`, or `script`), and optionally runs tests and scores the result. For v1 tasksets, prefer the native `uv run debug ...` CLI; `SWEDebugEnv` remains as a deprecated compatibility wrapper for older callers.
+No-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the task sandbox, optionally runs task setup, runs one debug step (`none`, `gold_patch`, `command`, or `script`), and optionally runs tests and scores the result. `SWEDebugEnv` remains as a deprecated wrapper for older callers.
 
 #### EnvGroup
 

From 5e09617a1e1f7b4ca09c52151d5aa1e3ed8c7846 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 06:37:18 +0000
Subject: [PATCH 16/27] fix: upload the debug script to a per-runtime path

A fixed /tmp path is a shared host file on the subprocess runtime
(write resolves absolute paths outside the per-runtime workdir), so
concurrent runs with different scripts could execute each other's.
Key the path by the run-unique runtime name; /tmp rather than the
workdir so the script never pollutes the repo state being inspected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/debug.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 057f4a91d6..366cb38208 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -33,9 +33,6 @@
 
 logger = logging.getLogger(__name__)
 
-REMOTE_SCRIPT_PATH = "/tmp/vf-debug-script.sh"
-"""Runtime path the uploaded host script is written to and executed from."""
-
 USAGE = (
     "usage: uv run debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
     "[--runtime.type subprocess] [options] [@ file.toml]\n"
@@ -189,13 +186,17 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
             **(await run_command(runtime, config.command, config)),
         }
     assert config.script_path is not None
-    await runtime.write(REMOTE_SCRIPT_PATH, config.script_path.read_bytes())
-    quoted = shlex.quote(REMOTE_SCRIPT_PATH)
+    # /tmp keyed by the (run-unique) runtime name: outside the task workdir so the script
+    # doesn't show up in the repo state being inspected, and never shared between runs on
+    # the subprocess runtime, where an absolute path is a host path.
+    remote_path = f"/tmp/{runtime.name}-script.sh"
+    await runtime.write(remote_path, config.script_path.read_bytes())
+    quoted = shlex.quote(remote_path)
     command = f"chmod +x {quoted} && {quoted}"
     return {
         "action": "script",
         "script_path": str(config.script_path),
-        "remote_script_path": REMOTE_SCRIPT_PATH,
+        "remote_script_path": remote_path,
         "command": command,
         **(await run_command(runtime, command, config)),
     }

From 809f2c91320e60b2c220ef882894ab9b66079e4c Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 06:42:52 +0000
Subject: [PATCH 17/27] fix: persist the debug trace when cancellation lands
 during teardown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A CancelledError delivered while awaiting runtime.stop() in the
finally sailed past 'except Exception' and aborted debug_task before
the caller could append_trace — losing exactly the cancelled trace
the cancelled-flag path exists to persist. Absorb it there and let
the caller re-raise after appending. Composes with #1920, which
makes stop() complete teardown and then re-raise the cancellation.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/debug.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index 366cb38208..b84789b0ec 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -247,6 +247,10 @@ async def debug_task(taskset: Taskset, task, config: DebugConfig) -> tuple[Trace
         trace.info["debug"] = debug
         try:
             await runtime.stop()
+        except asyncio.CancelledError:
+            # a task cancellation delivered mid-stop would abort before the caller can
+            # persist the trace — absorb it here; the caller re-raises after appending
+            cancelled = True
         except Exception:
             logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
     return trace, cancelled

From 95f17f1af4690ef1fdfce729d05657604c84869d Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 06:45:32 +0000
Subject: [PATCH 18/27] fix: put the script upload under the debug action
 timeout

The upload is a remote call on prime/modal and ran outside
asyncio.wait_for, so a stalled write hung forever with
--timeout.total never applying. Upload + execution now share one
total budget via run_timed (formerly run_command), rather than each
getting their own.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/debug.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py
index b84789b0ec..e8e3f4d44b 100644
--- a/verifiers/v1/cli/debug.py
+++ b/verifiers/v1/cli/debug.py
@@ -9,6 +9,7 @@
 import sys
 import time
 import traceback
+from collections.abc import Awaitable
 from pathlib import Path
 from typing import Any
 from uuid import uuid4
@@ -162,17 +163,13 @@ def record_action_failure(trace: Trace, debug: dict[str, Any]) -> None:
     )
 
 
-async def run_command(
-    runtime: Runtime,
-    command: str,
-    config: DebugConfig,
+async def run_timed(
+    action: Awaitable[ProgramResult], config: DebugConfig
 ) -> dict[str, Any]:
+    """Run the whole debug action under one `--timeout.total` budget."""
     start = time.time()
     try:
-        result = await asyncio.wait_for(
-            runtime.run(["sh", "-lc", command], {}),
-            config.timeout.total,
-        )
+        result = await asyncio.wait_for(action, config.timeout.total)
     except Exception as e:
         return error_info(e, start, config.timeout.total, "debug action")
     return result_info(result, start)
@@ -183,22 +180,26 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
         return {
             "action": "command",
             "command": config.command,
-            **(await run_command(runtime, config.command, config)),
+            **(await run_timed(runtime.run(["sh", "-lc", config.command], {}), config)),
         }
     assert config.script_path is not None
     # /tmp keyed by the (run-unique) runtime name: outside the task workdir so the script
     # doesn't show up in the repo state being inspected, and never shared between runs on
     # the subprocess runtime, where an absolute path is a host path.
     remote_path = f"/tmp/{runtime.name}-script.sh"
-    await runtime.write(remote_path, config.script_path.read_bytes())
     quoted = shlex.quote(remote_path)
     command = f"chmod +x {quoted} && {quoted}"
+
+    async def upload_and_run() -> ProgramResult:
+        await runtime.write(remote_path, config.script_path.read_bytes())
+        return await runtime.run(["sh", "-lc", command], {})
+
     return {
         "action": "script",
         "script_path": str(config.script_path),
         "remote_script_path": remote_path,
         "command": command,
-        **(await run_command(runtime, command, config)),
+        **(await run_timed(upload_and_run(), config)),
     }
 
 

From 5bd13251d58666df13484bbabb583efcca6a771f Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Fri, 3 Jul 2026 18:01:50 -0700
Subject: [PATCH 19/27] fix: shield runtime teardown from cancellation
 framework-wide (#1920)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: shield runtime teardown from cancellation framework-wide

A Ctrl-C / SIGTERM (converted to KeyboardInterrupt by the CLIs) cancels the
owning task, and the cancellation lands inside the finally that awaits
runtime.stop(), truncating teardown mid-await and leaking live docker
containers or paid Prime sandboxes. The unshielded pattern existed at five
sites: rollout.py, both validate modes, debug_task, and the AsyncExitStack
callback in mcp/launch.py.

Fix it once at the framework level: Runtime.stop() is now a concrete
framework method that runs the overridable teardown() (modal/prime move
their bodies there) to completion under cancellation via run_shielded, then
re-raises the cancellation. A bare asyncio.shield is not enough — it
re-raises immediately and the orphaned inner task is cancelled by
asyncio.run's shutdown anyway — so the helper keeps awaiting until teardown
finishes. All call sites stay unchanged and every future caller is safe by
construction.

run_shielded also replaces the two inline copies of the pattern
(host_endpoint's finally — fixing a corner where a teardown error could
swallow a captured cancellation — and append_trace, whose recovery await
was itself unshielded against a second cancellation).

The sync atexit backstop is unchanged: a second Ctrl-C raises
KeyboardInterrupt out of the event loop itself, beyond any task-level
shield.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix: keep modal's atexit backstop armed through a truncated teardown

ModalRuntime.teardown consumed its `_sandbox` guard before the first await,
so loop death (second Ctrl-C) mid-terminate left cleanup() keyed off None —
the atexit backstop could not terminate the sandbox and it leaked to its
max-lifetime. Pre-existing (identical body under the old stop()), with a
wider window before the shield; consume the guard only after the terminate
attempt, so a truncating CancelledError propagates with the backstop still
armed. Document the contract on Runtime.teardown: overrides must not consume
state cleanup() keys off before their first await.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* test: remove teardown test files

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/cli/output.py     | 11 ++++------
 verifiers/v1/runtimes/base.py  | 40 +++++++++++++++++++---------------
 verifiers/v1/runtimes/modal.py | 12 ++++++----
 verifiers/v1/runtimes/prime.py |  6 ++---
 verifiers/v1/utils/aio.py      | 31 ++++++++++++++++++++++++++
 5 files changed, 69 insertions(+), 31 deletions(-)
 create mode 100644 verifiers/v1/utils/aio.py

diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py
index c936a01c8d..3c565e67a1 100644
--- a/verifiers/v1/cli/output.py
+++ b/verifiers/v1/cli/output.py
@@ -19,6 +19,7 @@
 
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.aio import run_shielded
 
 
 def output_path(config: EvalConfig) -> Path:
@@ -65,10 +66,6 @@ async def persist() -> None:
         async with lock:
             await asyncio.to_thread(write_trace, results_dir, trace)
 
-    # Shield lock acquisition and the worker so finalized traces survive cancellation.
-    persist_task = asyncio.create_task(persist())
-    try:
-        await asyncio.shield(persist_task)
-    except asyncio.CancelledError:
-        await persist_task
-        raise
+    # Run lock acquisition and the worker to completion even under cancellation, so
+    # finalized traces are never lost mid-write (`run_shielded` re-raises the cancellation).
+    await run_shielded(persist())
diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py
index 8029deda4e..3becd19658 100644
--- a/verifiers/v1/runtimes/base.py
+++ b/verifiers/v1/runtimes/base.py
@@ -20,6 +20,7 @@
 from typing import ClassVar, TypeVar
 
 from verifiers.v1.retries import retrying
+from verifiers.v1.utils.aio import run_shielded
 
 logger = logging.getLogger(__name__)
 
@@ -76,10 +77,12 @@ def parse_gpu(gpu: str | None) -> tuple[str | None, int]:
     return head, 1
 
 
-# `stop()` frees a runtime's external resource on the normal path (the rollout's `finally`).
-# A Ctrl-C / SIGTERM can cancel that `finally` mid-teardown, so runtimes are tracked in
-# `_LIVE` and freed by a *synchronous* `atexit` hook (`cleanup`) — sync because the event
-# loop is gone at interpreter shutdown. SIGKILL runs none of this.
+# `stop()` frees a runtime's external resource on the normal path (the rollout's `finally`),
+# shielded so a Ctrl-C / SIGTERM task cancellation can't cut it short. A second Ctrl-C
+# raises KeyboardInterrupt out of the event loop itself — no task-level shield survives
+# that — so runtimes are also tracked in `_LIVE` and freed by a *synchronous* `atexit`
+# hook (`cleanup`), sync because the loop is gone at interpreter shutdown. SIGKILL runs
+# none of this.
 _LIVE: "weakref.WeakSet[Runtime]" = weakref.WeakSet()
 _atexit_armed = False
 
@@ -140,8 +143,19 @@ async def start(self) -> None:
         host port into a URL the program can reach."""
 
     async def stop(self) -> None:
-        """Free the provisioned resource on the normal path, off the event loop. Override
-        only for teardown that must be async (e.g. a remote API call)."""
+        """Free the provisioned resource on the normal path (the owner's `finally`),
+        shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await,
+        and an interrupted teardown leaks the container / paid sandbox. Runs `teardown`
+        to completion, then re-raises the cancellation. Framework method — override
+        `teardown`, not this."""
+        await run_shielded(self.teardown())
+
+    async def teardown(self) -> None:
+        """Free the provisioned resource, off the event loop. Override only for teardown
+        that must be async (e.g. a remote API call); `stop` shields it from cancellation.
+        Best-effort and idempotent, like `cleanup`. An override must not consume state
+        `cleanup` keys off before its first await: if the event loop dies mid-teardown
+        (second Ctrl-C), the atexit backstop must still find the resource."""
         await asyncio.to_thread(self.cleanup)
 
     def cleanup(self) -> None:
@@ -325,18 +339,10 @@ async def _start() -> tuple[Tunnel, str]:
     try:
         yield url
     finally:
-        # Delay cancellation until the synchronous stop has finished.
-        cancelled = None
-        stop_task = asyncio.create_task(asyncio.to_thread(tunnel.sync_stop))
+        # Run the synchronous stop to completion even under cancellation (`run_shielded`
+        # re-raises the cancellation after); tunnel-stop failures are best-effort.
         with contextlib.suppress(Exception):
-            while not stop_task.done():
-                try:
-                    await asyncio.shield(stop_task)
-                except asyncio.CancelledError as e:
-                    cancelled = e
-            stop_task.result()
-        if cancelled is not None:
-            raise cancelled
+            await run_shielded(asyncio.to_thread(tunnel.sync_stop))
 
 
 class _Host:
diff --git a/verifiers/v1/runtimes/modal.py b/verifiers/v1/runtimes/modal.py
index 306a136dd6..a6423766e6 100644
--- a/verifiers/v1/runtimes/modal.py
+++ b/verifiers/v1/runtimes/modal.py
@@ -189,11 +189,14 @@ def cleanup(self) -> None:
             with contextlib.suppress(Exception):
                 sandbox.terminate()
 
-    async def stop(self) -> None:
+    async def teardown(self) -> None:
         # Best-effort, idempotent teardown on the normal path: terminate the sandbox (the costly
-        # resource) via the async API. Runs from the rollout's `finally`, so it fires on success,
-        # error, and cancellation; `_sandbox` is nulled as the idempotency guard (atexit no-ops).
-        sandbox, self._sandbox = self._sandbox, None
+        # resource) via the async API. Runs via `stop`, shielded from cancellation, so it fires on
+        # success, error, and Ctrl-C. `_sandbox` — the atexit backstop's key — is consumed only
+        # after the terminate attempt: if loop death (second Ctrl-C) truncates the await, the
+        # CancelledError propagates before the null and the backstop can still terminate. (A
+        # concurrent second terminate is a no-op on Modal's side.)
+        sandbox = self._sandbox
         if sandbox is None:
             return
         try:
@@ -202,3 +205,4 @@ async def stop(self) -> None:
             logger.warning(
                 "modal: failed to terminate sandbox %s: %s", self._sandbox_id, e
             )
+        self._sandbox = None
diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py
index 1cf3b74e52..230b72d877 100644
--- a/verifiers/v1/runtimes/prime.py
+++ b/verifiers/v1/runtimes/prime.py
@@ -226,9 +226,9 @@ def cleanup(self) -> None:
             with contextlib.suppress(Exception):
                 SandboxClient(APIClient()).delete(self._sandbox_id)
 
-    async def stop(self) -> None:
-        # Best-effort, idempotent teardown: delete the sandbox (the costly resource). Runs from the
-        # rollout's `finally`, so it fires on success, error, and cancellation.
+    async def teardown(self) -> None:
+        # Best-effort, idempotent teardown: delete the sandbox (the costly resource). Runs via
+        # `stop`, shielded from cancellation, so it fires on success, error, and Ctrl-C.
         client, self._client = self._client, None  # `_client` is the idempotency guard
         if client is None:
             return
diff --git a/verifiers/v1/utils/aio.py b/verifiers/v1/utils/aio.py
new file mode 100644
index 0000000000..a62887f509
--- /dev/null
+++ b/verifiers/v1/utils/aio.py
@@ -0,0 +1,31 @@
+"""Asyncio helpers shared across the framework."""
+
+import asyncio
+from collections.abc import Awaitable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+async def run_shielded(coro: Awaitable[T]) -> T:
+    """Run `coro` to completion even if the surrounding task is cancelled, then re-raise
+    the cancellation. A bare `asyncio.shield` is not enough: on cancellation it re-raises
+    immediately while the inner task runs on orphaned — and the loop's shutdown then
+    cancels that orphan mid-await anyway. This keeps awaiting (absorbing repeated task
+    cancellations) until `coro` finishes; the first CancelledError is re-raised after.
+    If `coro` raises without a cancellation, the error propagates unchanged; with one,
+    the cancellation wins and the error is chained under it (`from`), so it is never
+    silently lost. A second Ctrl-C that raises KeyboardInterrupt out of the event loop
+    itself is beyond any task-level shield — that path is the atexit backstop's job."""
+    task = asyncio.ensure_future(coro)
+    cancelled: asyncio.CancelledError | None = None
+    while not task.done():
+        try:
+            await asyncio.shield(task)
+        except asyncio.CancelledError as e:
+            cancelled = e
+        except BaseException:
+            pass  # `coro` itself failed → task is done; re-raised (or chained) below
+    if cancelled is not None:
+        raise cancelled from (None if task.cancelled() else task.exception())
+    return task.result()

From 5e1e5720996a280c3ea1c8c679ba9a1721831b5f Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 01:45:19 +0000
Subject: [PATCH 20/27] feat: report interrupted teardown drains; never
 subordinate a user abort
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When Ctrl-C lands while runtime teardowns are in flight, the shielded stop()
(#1920) keeps them running — but silently, so the wait reads as a hang and
users escalate to kill -9, which skips the atexit backstop entirely. Track
in-flight teardowns in a registry and report the drain once, in aggregate
(one line for 512 teardowns, not 512): count, oldest age, and that a second
Ctrl-C aborts with best-effort cleanup at exit; one closing line when the
drain finishes. Per-runtime detail at DEBUG.

Also narrow run_shielded's inner catch from BaseException to Exception: a
KeyboardInterrupt/SystemExit raised out of the teardown (a signal delivered
while the interpreter executes teardown code) must propagate immediately —
never be swallowed by the loop or chained under the pending CancelledError —
and a force-closed coroutine must not ignore GeneratorExit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .cligym-image-push/failed-image-map.jsonl |   1 +
 .cligym-image-push/image-map.jsonl        |   1 +
 tests/v1/test_runtimes.py                 | 117 ++++++++++++++++++++++
 verifiers/v1/runtimes/base.py             |  44 +++++++-
 verifiers/v1/utils/aio.py                 |  18 +++-
 5 files changed, 173 insertions(+), 8 deletions(-)
 create mode 100644 .cligym-image-push/failed-image-map.jsonl
 create mode 100644 .cligym-image-push/image-map.jsonl
 create mode 100644 tests/v1/test_runtimes.py

diff --git a/.cligym-image-push/failed-image-map.jsonl b/.cligym-image-push/failed-image-map.jsonl
new file mode 100644
index 0000000000..e40976b025
--- /dev/null
+++ b/.cligym-image-push/failed-image-map.jsonl
@@ -0,0 +1 @@
+{"artifact_full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "build_id": "w6pqn7ttyu8vdm7h7s4i72vu", "error": "Build failed with exit code 1", "full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "image_name": "tmax", "image_tag": "task_004697_bd627cb1", "item_id": "task_004697_bd627cb1", "item_key": "tmax:task_004697_bd627cb1", "status": "FAILED", "synthetic_empty_dirs": [], "tar_size_bytes": 5093}
diff --git a/.cligym-image-push/image-map.jsonl b/.cligym-image-push/image-map.jsonl
new file mode 100644
index 0000000000..a18731df56
--- /dev/null
+++ b/.cligym-image-push/image-map.jsonl
@@ -0,0 +1 @@
+{"build_id": "w6pqn7ttyu8vdm7h7s4i72vu", "full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "image_name": "tmax", "image_tag": "task_004697_bd627cb1", "item_id": "task_004697_bd627cb1", "item_key": "tmax:task_004697_bd627cb1", "synthetic_empty_dirs": [], "tar_size_bytes": 5093}
diff --git a/tests/v1/test_runtimes.py b/tests/v1/test_runtimes.py
new file mode 100644
index 0000000000..6069783abe
--- /dev/null
+++ b/tests/v1/test_runtimes.py
@@ -0,0 +1,117 @@
+"""Runtime.stop teardown reporting and abort semantics: an interrupt landing
+mid-teardown is reported once (with the in-flight drain snapshot) so the user can
+decide whether to wait or Ctrl-C again, and a user abort (KeyboardInterrupt) is never
+subordinated to a pending cancellation."""
+
+import asyncio
+import logging
+
+import pytest
+
+from verifiers.v1.runtimes.base import ProgramResult, Runtime
+
+
+class FakeRuntime(Runtime):
+    """Teardown gated on an event so tests can interrupt mid-teardown deterministically."""
+
+    def __init__(self, exc: BaseException | None = None) -> None:
+        super().__init__(name="fake")
+        self.exc = exc
+        self.teardown_started = asyncio.Event()
+        self.release = asyncio.Event()
+        self.teardown_finished = False
+
+    async def start(self) -> None: ...
+
+    async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
+        return ProgramResult(0, "", "")
+
+    async def read(self, path: str) -> bytes:
+        return b""
+
+    async def write(self, path: str, data: bytes) -> None: ...
+
+    async def teardown(self) -> None:
+        self.teardown_started.set()
+        await self.release.wait()
+        if self.exc is not None:
+            raise self.exc
+        self.teardown_finished = True
+
+
+async def test_interrupted_drain_reported_as_one_aggregate_line(caplog, monkeypatch):
+    # The legacy logging setup disables propagation on the package logger at import;
+    # re-enable it so caplog's root handler sees the report.
+    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
+    runtimes = [FakeRuntime() for _ in range(3)]
+    tasks = [asyncio.create_task(rt.stop()) for rt in runtimes]
+    for rt in runtimes:
+        await rt.teardown_started.wait()
+    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
+        for task in tasks:  # Ctrl-C lands while all three stops are in flight
+            task.cancel()
+        await asyncio.sleep(0)
+        for rt in runtimes:
+            rt.release.set()
+        for task in tasks:
+            with pytest.raises(asyncio.CancelledError):
+                await task
+    reports = [r for r in caplog.records if "Ctrl-C again" in r.getMessage()]
+    assert len(reports) == 1  # one aggregate line, not one per teardown
+    assert "3 in-flight teardown(s)" in reports[0].getMessage()
+    done = [r for r in caplog.records if "teardowns finished" in r.getMessage()]
+    assert len(done) == 1  # one completion line closes the drain
+    assert all(rt.teardown_finished for rt in runtimes)
+
+
+async def test_later_drain_reports_again(caplog, monkeypatch):
+    # The dedup flag resets when the drain empties: a second interrupted drain in the
+    # same process (e.g. a long-lived server) reports again.
+    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
+    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
+        for _ in range(2):
+            rt = FakeRuntime()
+            task = asyncio.create_task(rt.stop())
+            await rt.teardown_started.wait()
+            task.cancel()
+            rt.release.set()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+    reports = [r for r in caplog.records if "Ctrl-C again" in r.getMessage()]
+    assert len(reports) == 2
+
+
+async def test_no_report_without_interrupt(caplog, monkeypatch):
+    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
+    rt = FakeRuntime()
+    rt.release.set()
+    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
+        await rt.stop()
+    assert not [r for r in caplog.records if "finishing teardown" in r.getMessage()]
+
+
+class UserAbort(BaseException):
+    """Stand-in for KeyboardInterrupt: a signal delivered while the interpreter executes
+    teardown code manifests as the teardown raising it. Task.__step re-raises the real
+    KeyboardInterrupt/SystemExit into the event loop (killing it — correct in production,
+    fatal to the test session), so the shield-layer semantics are asserted with a plain
+    BaseException, which run_shielded treats identically."""
+
+
+async def test_teardown_base_exception_wins_over_cancellation():
+    # The user's abort must win over the pending cancellation, not get chained under it.
+    rt = FakeRuntime(exc=UserAbort())
+    task = asyncio.create_task(rt.stop())
+    await rt.teardown_started.wait()
+    task.cancel()
+    rt.release.set()
+    with pytest.raises(UserAbort):
+        await task
+
+
+async def test_teardown_error_propagates_when_not_cancelled():
+    # The call sites' `except Exception: logger.warning(...)` depends on this.
+    rt = FakeRuntime(exc=RuntimeError("boom"))
+    rt.release.set()
+    with pytest.raises(RuntimeError, match="boom"):
+        await rt.stop()
diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py
index 3becd19658..d2384de298 100644
--- a/verifiers/v1/runtimes/base.py
+++ b/verifiers/v1/runtimes/base.py
@@ -11,6 +11,7 @@
 import hashlib
 import logging
 import shlex
+import time
 import uuid
 import weakref
 from abc import ABC, abstractmethod
@@ -86,6 +87,17 @@ def parse_gpu(gpu: str | None) -> tuple[str | None, int]:
 _LIVE: "weakref.WeakSet[Runtime]" = weakref.WeakSet()
 _atexit_armed = False
 
+_STOPPING: "dict[Runtime, float]" = {}
+"""In-flight teardowns -> start time. When an interrupt lands mid-`stop()`, one aggregate
+report tells the user how much work is draining, so they can judge whether to wait or
+press Ctrl-C again; also a debugger's view of what a shutdown is stuck on. Entries are
+popped when their `stop()` returns."""
+
+_drain_reported = False
+"""Whether the current drain has been reported — one aggregate line per drain, however
+many teardowns absorb the interrupt (512 in-flight must not print 512 lines). Reset when
+`_STOPPING` empties, so a later drain in a long-lived process reports again."""
+
 
 def register(runtime: "Runtime") -> None:
     """Track a runtime so the atexit hook can free it if a signal cuts its `finally` short.
@@ -146,9 +158,35 @@ async def stop(self) -> None:
         """Free the provisioned resource on the normal path (the owner's `finally`),
         shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await,
         and an interrupted teardown leaks the container / paid sandbox. Runs `teardown`
-        to completion, then re-raises the cancellation. Framework method — override
-        `teardown`, not this."""
-        await run_shielded(self.teardown())
+        to completion, then re-raises the cancellation; when the interrupt lands
+        mid-teardown, reports the drain (name, in-flight count, age) so the wait reads
+        as intentional and the user can decide whether to Ctrl-C again. Framework
+        method — override `teardown`, not this."""
+
+        def interrupted() -> None:
+            global _drain_reported
+            logger.debug(
+                "finishing teardown of %s (%s)", self.name, self.descriptor or "-"
+            )
+            if _drain_reported:
+                return
+            _drain_reported = True
+            logger.warning(
+                "finishing %d in-flight teardown(s) (oldest %.0fs) — Ctrl-C again to "
+                "abort; leftover resources are freed best-effort at exit",
+                len(_STOPPING),
+                time.time() - min(_STOPPING.values()),
+            )
+
+        global _drain_reported
+        _STOPPING[self] = time.time()
+        try:
+            await run_shielded(self.teardown(), interrupted=interrupted)
+        finally:
+            _STOPPING.pop(self, None)
+            if _drain_reported and not _STOPPING:
+                _drain_reported = False
+                logger.warning("all in-flight teardowns finished")
 
     async def teardown(self) -> None:
         """Free the provisioned resource, off the event loop. Override only for teardown
diff --git a/verifiers/v1/utils/aio.py b/verifiers/v1/utils/aio.py
index a62887f509..7705d780da 100644
--- a/verifiers/v1/utils/aio.py
+++ b/verifiers/v1/utils/aio.py
@@ -1,30 +1,38 @@
 """Asyncio helpers shared across the framework."""
 
 import asyncio
-from collections.abc import Awaitable
+from collections.abc import Awaitable, Callable
 from typing import TypeVar
 
 T = TypeVar("T")
 
 
-async def run_shielded(coro: Awaitable[T]) -> T:
+async def run_shielded(
+    coro: Awaitable[T], interrupted: Callable[[], None] | None = None
+) -> T:
     """Run `coro` to completion even if the surrounding task is cancelled, then re-raise
     the cancellation. A bare `asyncio.shield` is not enough: on cancellation it re-raises
     immediately while the inner task runs on orphaned — and the loop's shutdown then
     cancels that orphan mid-await anyway. This keeps awaiting (absorbing repeated task
     cancellations) until `coro` finishes; the first CancelledError is re-raised after.
+    `interrupted`, if given, is invoked once on the first absorbed cancellation — so the
+    caller can tell the user the wait is intentional, not a hang.
     If `coro` raises without a cancellation, the error propagates unchanged; with one,
     the cancellation wins and the error is chained under it (`from`), so it is never
-    silently lost. A second Ctrl-C that raises KeyboardInterrupt out of the event loop
-    itself is beyond any task-level shield — that path is the atexit backstop's job."""
+    silently lost — except a `BaseException` (KeyboardInterrupt, SystemExit), which
+    propagates immediately: a user abort must never lose to a pending cancellation.
+    A second Ctrl-C that raises KeyboardInterrupt out of the event loop itself is beyond
+    any task-level shield — that path is the atexit backstop's job."""
     task = asyncio.ensure_future(coro)
     cancelled: asyncio.CancelledError | None = None
     while not task.done():
         try:
             await asyncio.shield(task)
         except asyncio.CancelledError as e:
+            if cancelled is None and interrupted is not None:
+                interrupted()
             cancelled = e
-        except BaseException:
+        except Exception:
             pass  # `coro` itself failed → task is done; re-raised (or chained) below
     if cancelled is not None:
         raise cancelled from (None if task.cancelled() else task.exception())

From 1a217e2020e75aeaf0134276dcb42f88761350cb Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 01:46:24 +0000
Subject: [PATCH 21/27] chore: untrack stray local image-push maps committed by
 mistake

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .cligym-image-push/failed-image-map.jsonl | 1 -
 .cligym-image-push/image-map.jsonl        | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 .cligym-image-push/failed-image-map.jsonl
 delete mode 100644 .cligym-image-push/image-map.jsonl

diff --git a/.cligym-image-push/failed-image-map.jsonl b/.cligym-image-push/failed-image-map.jsonl
deleted file mode 100644
index e40976b025..0000000000
--- a/.cligym-image-push/failed-image-map.jsonl
+++ /dev/null
@@ -1 +0,0 @@
-{"artifact_full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "build_id": "w6pqn7ttyu8vdm7h7s4i72vu", "error": "Build failed with exit code 1", "full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "image_name": "tmax", "image_tag": "task_004697_bd627cb1", "item_id": "task_004697_bd627cb1", "item_key": "tmax:task_004697_bd627cb1", "status": "FAILED", "synthetic_empty_dirs": [], "tar_size_bytes": 5093}
diff --git a/.cligym-image-push/image-map.jsonl b/.cligym-image-push/image-map.jsonl
deleted file mode 100644
index a18731df56..0000000000
--- a/.cligym-image-push/image-map.jsonl
+++ /dev/null
@@ -1 +0,0 @@
-{"build_id": "w6pqn7ttyu8vdm7h7s4i72vu", "full_image_path": "prime/primeintellect/tmax:task_004697_bd627cb1", "image_name": "tmax", "image_tag": "task_004697_bd627cb1", "item_id": "task_004697_bd627cb1", "item_key": "tmax:task_004697_bd627cb1", "synthetic_empty_dirs": [], "tar_size_bytes": 5093}

From 96f4b4d615e415d086cec18416f987ce7b551f85 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 02:03:31 +0000
Subject: [PATCH 22/27] test: remove teardown drain test files

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tests/v1/test_runtimes.py | 117 --------------------------------------
 1 file changed, 117 deletions(-)
 delete mode 100644 tests/v1/test_runtimes.py

diff --git a/tests/v1/test_runtimes.py b/tests/v1/test_runtimes.py
deleted file mode 100644
index 6069783abe..0000000000
--- a/tests/v1/test_runtimes.py
+++ /dev/null
@@ -1,117 +0,0 @@
-"""Runtime.stop teardown reporting and abort semantics: an interrupt landing
-mid-teardown is reported once (with the in-flight drain snapshot) so the user can
-decide whether to wait or Ctrl-C again, and a user abort (KeyboardInterrupt) is never
-subordinated to a pending cancellation."""
-
-import asyncio
-import logging
-
-import pytest
-
-from verifiers.v1.runtimes.base import ProgramResult, Runtime
-
-
-class FakeRuntime(Runtime):
-    """Teardown gated on an event so tests can interrupt mid-teardown deterministically."""
-
-    def __init__(self, exc: BaseException | None = None) -> None:
-        super().__init__(name="fake")
-        self.exc = exc
-        self.teardown_started = asyncio.Event()
-        self.release = asyncio.Event()
-        self.teardown_finished = False
-
-    async def start(self) -> None: ...
-
-    async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
-        return ProgramResult(0, "", "")
-
-    async def read(self, path: str) -> bytes:
-        return b""
-
-    async def write(self, path: str, data: bytes) -> None: ...
-
-    async def teardown(self) -> None:
-        self.teardown_started.set()
-        await self.release.wait()
-        if self.exc is not None:
-            raise self.exc
-        self.teardown_finished = True
-
-
-async def test_interrupted_drain_reported_as_one_aggregate_line(caplog, monkeypatch):
-    # The legacy logging setup disables propagation on the package logger at import;
-    # re-enable it so caplog's root handler sees the report.
-    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
-    runtimes = [FakeRuntime() for _ in range(3)]
-    tasks = [asyncio.create_task(rt.stop()) for rt in runtimes]
-    for rt in runtimes:
-        await rt.teardown_started.wait()
-    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
-        for task in tasks:  # Ctrl-C lands while all three stops are in flight
-            task.cancel()
-        await asyncio.sleep(0)
-        for rt in runtimes:
-            rt.release.set()
-        for task in tasks:
-            with pytest.raises(asyncio.CancelledError):
-                await task
-    reports = [r for r in caplog.records if "Ctrl-C again" in r.getMessage()]
-    assert len(reports) == 1  # one aggregate line, not one per teardown
-    assert "3 in-flight teardown(s)" in reports[0].getMessage()
-    done = [r for r in caplog.records if "teardowns finished" in r.getMessage()]
-    assert len(done) == 1  # one completion line closes the drain
-    assert all(rt.teardown_finished for rt in runtimes)
-
-
-async def test_later_drain_reports_again(caplog, monkeypatch):
-    # The dedup flag resets when the drain empties: a second interrupted drain in the
-    # same process (e.g. a long-lived server) reports again.
-    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
-    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
-        for _ in range(2):
-            rt = FakeRuntime()
-            task = asyncio.create_task(rt.stop())
-            await rt.teardown_started.wait()
-            task.cancel()
-            rt.release.set()
-            with pytest.raises(asyncio.CancelledError):
-                await task
-    reports = [r for r in caplog.records if "Ctrl-C again" in r.getMessage()]
-    assert len(reports) == 2
-
-
-async def test_no_report_without_interrupt(caplog, monkeypatch):
-    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
-    rt = FakeRuntime()
-    rt.release.set()
-    with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
-        await rt.stop()
-    assert not [r for r in caplog.records if "finishing teardown" in r.getMessage()]
-
-
-class UserAbort(BaseException):
-    """Stand-in for KeyboardInterrupt: a signal delivered while the interpreter executes
-    teardown code manifests as the teardown raising it. Task.__step re-raises the real
-    KeyboardInterrupt/SystemExit into the event loop (killing it — correct in production,
-    fatal to the test session), so the shield-layer semantics are asserted with a plain
-    BaseException, which run_shielded treats identically."""
-
-
-async def test_teardown_base_exception_wins_over_cancellation():
-    # The user's abort must win over the pending cancellation, not get chained under it.
-    rt = FakeRuntime(exc=UserAbort())
-    task = asyncio.create_task(rt.stop())
-    await rt.teardown_started.wait()
-    task.cancel()
-    rt.release.set()
-    with pytest.raises(UserAbort):
-        await task
-
-
-async def test_teardown_error_propagates_when_not_cancelled():
-    # The call sites' `except Exception: logger.warning(...)` depends on this.
-    rt = FakeRuntime(exc=RuntimeError("boom"))
-    rt.release.set()
-    with pytest.raises(RuntimeError, match="boom"):
-        await rt.stop()

From c85b16c196a1cfc04948bb7f1d15e82522ffcad5 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 03:32:50 +0000
Subject: [PATCH 23/27] refactor: validate runs all checks by default;
 --only-setup / --only-gold to narrow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Per review: replace --mode {apply-answer, noop, all} with
all-by-default plus two restriction flags. The checks are named for
what they are — gold (setup + validate hook) and setup-only — in
flags, row modes, nested subresult keys, and runtime names.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/GUIDE.md            | 15 +++++----
 verifiers/v1/README.md           |  2 +-
 verifiers/v1/cli/validate.py     | 57 +++++++++++++++++---------------
 verifiers/v1/configs/validate.py | 19 +++++++----
 4 files changed, 51 insertions(+), 42 deletions(-)

diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md
index f897b04abb..0df3c1a448 100644
--- a/verifiers/v1/GUIDE.md
+++ b/verifiers/v1/GUIDE.md
@@ -736,22 +736,23 @@ trace per line, appended as each rollout finishes — durable mid-run), and `eva
 
 ## `validate`
 
-Run each task's `validate` hook — a model-free check that the ground truth holds (the gold patch
-makes the tests pass, the verifier accepts the gold answer) — in a runtime with the taskset's
-`setup` applied. No model, no harness. Use `--mode noop` to run setup only, or `--mode all`
-to run `apply-answer` and `noop` in independent runtimes and report one aggregate row.
+Model-free checks that a task is sound. By default each task gets two independent
+runtimes — a **gold** check (the taskset's `setup` then its `validate` hook: the gold patch
+makes the tests pass, the verifier accepts the gold answer) and a **setup-only** check —
+reported as one aggregate row. Restrict to one check with `--only-gold` / `--only-setup`.
+No model, no harness.
 
 ```bash
 uv run validate gsm8k-v1 -n 20 --runtime.type subprocess
-uv run validate swebench-v1 -n 1 --runtime.type prime --mode noop
-uv run validate swebench-v1 -n 1 --runtime.type prime --mode all
+uv run validate swebench-v1 -n 1 --runtime.type prime --only-setup
+uv run validate swebench-v1 -n 1 --runtime.type prime --only-gold
 ```
 
 | flag | default | meaning |
 | --- | --- | --- |
 | `<taskset-id>` / `--taskset.id` | — | taskset to validate |
 | `--runtime.type` | `docker` | runtime for `setup` + `validate` (a gold check often needs the task's container) |
-| `--mode` | `apply-answer` | `apply-answer`, `noop`, or `all` |
+| `--only-setup` / `--only-gold` | off | run just the setup-only or just the gold check (default: both) |
 | `--timeout.setup` / `--timeout.total` | None | per-task wall-clock caps for `setup` and the `validate` hook |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
 | `-v`/`--verbose`, `--no-rich` | | logging / disable the dashboard |
diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md
index 3327a3c9ec..0914aa8399 100644
--- a/verifiers/v1/README.md
+++ b/verifiers/v1/README.md
@@ -41,7 +41,7 @@ uv sync --python 3.12 --extra harbor
 ```bash
 uv run init my-task-v1           # scaffold a new environment (--add-tool/--add-user/--add-harness)
 uv run eval gsm8k-v1 -n 5 -r 3   # single-turn math; default harness; subprocess runtime
-uv run validate gsm8k-v1 -n 5    # model-free: run each task's gold check (the `validate` hook)
+uv run validate gsm8k-v1 -n 5    # model-free gold + setup-only checks (--only-gold / --only-setup)
 uv run debug gsm8k-v1 -n 1 --command 'pwd'  # setup, run one shell action, save traces
 uv run eval -h                   # typed help (+ the local example tasksets/harnesses)
 ```
diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py
index f96943fb66..d742fd0ec2 100644
--- a/verifiers/v1/cli/validate.py
+++ b/verifiers/v1/cli/validate.py
@@ -1,9 +1,10 @@
 """The validate entrypoint: `uv run validate <taskset-id> [--runtime.type subprocess] [options]`.
 
 Registered as the `validate` console script — the model-free sibling of `eval`. Where `eval`
-runs a model rollout per task, `validate` can either run each task's `validate` hook, run
-setup only, or run all modes in independent runtimes. Each task is provisioned, set up,
-checked, and torn down independently with bounded concurrency.
+runs a model rollout per task, `validate` runs each task's gold check (`setup` + the
+`validate` hook) and a setup-only check in independent runtimes — or just one of them via
+`--only-gold` / `--only-setup`. Each task is provisioned, set up, checked, and torn down
+independently with bounded concurrency.
 
 Fire-and-forget: progress is shown live (the `--rich` dashboard, or per-task log lines) and
 nothing is written to disk.
@@ -40,8 +41,9 @@
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run validate [<taskset-id>] [--runtime.type subprocess] [options] [@ file.toml]\n"
-    "       runs setup-only, apply-answer, or all validation modes (no model)"
+    "usage: uv run validate [<taskset-id>] [--only-setup | --only-gold] "
+    "[--runtime.type subprocess] [options] [@ file.toml]\n"
+    "       runs the gold and setup-only checks per task (no model)"
 )
 
 
@@ -90,14 +92,12 @@ def _row(
     }
 
 
-async def _run_apply_answer(
-    taskset: Taskset, task, config: ValidateConfig
-) -> ResultRow:
-    """Provision one runtime, run setup + validate, tear it down, and return a row."""
+async def _run_gold(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    """The gold check: provision one runtime, run setup + validate, tear it down."""
     start = time.time()
     runtime = make_runtime(
         resolve_runtime_config(config.runtime, task),
-        name=f"validate-apply-answer-{task.idx}-{uuid4().hex[:8]}",
+        name=f"validate-gold-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
         config.timeout.setup if config.timeout.setup is not None else task.timeout.setup
@@ -120,15 +120,15 @@ async def _run_apply_answer(
             await runtime.stop()
         except Exception:
             logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
-    return _row(task, "apply-answer", valid, exc, start)
+    return _row(task, "gold", valid, exc, start)
 
 
-async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
-    """Provision one runtime, run setup only, tear it down, and return a row."""
+async def _run_setup(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
+    """The setup-only check: provision one runtime, run setup, tear it down."""
     start = time.time()
     runtime = make_runtime(
         resolve_runtime_config(config.runtime, task),
-        name=f"validate-noop-{task.idx}-{uuid4().hex[:8]}",
+        name=f"validate-setup-{task.idx}-{uuid4().hex[:8]}",
     )
     setup_timeout = (
         config.timeout.setup if config.timeout.setup is not None else task.timeout.setup
@@ -149,7 +149,7 @@ async def _run_noop(taskset: Taskset, task, config: ValidateConfig) -> ResultRow
             await runtime.stop()
         except Exception:
             logger.warning("runtime teardown failed (task %s)", task.idx, exc_info=True)
-    return _row(task, "noop", valid, exc, start)
+    return _row(task, "setup", valid, exc, start)
 
 
 def _all_reason(rows: list[ResultRow]) -> str:
@@ -171,11 +171,11 @@ def _all_error(rows: list[ResultRow]) -> tuple[str | None, str | None]:
 
 
 async def _run_all(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
-    """Run every mode (apply-answer, noop) as independent high-level validations."""
+    """Run every check (gold, setup-only) as independent high-level validations."""
     start = time.time()
-    apply_answer = await _run_apply_answer(taskset, task, config)
-    noop = await _run_noop(taskset, task, config)
-    rows = [apply_answer, noop]
+    gold = await _run_gold(taskset, task, config)
+    setup = await _run_setup(taskset, task, config)
+    rows = [gold, setup]
     error, error_type = _all_error(rows)
     return {
         "index": task.idx,
@@ -186,16 +186,16 @@ async def _run_all(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
         "elapsed": round(time.time() - start, 2),
         "error": error,
         "error_type": error_type,
-        "apply_answer": apply_answer,
-        "noop": noop,
+        "gold": gold,
+        "setup": setup,
     }
 
 
 async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> ResultRow:
-    if config.mode == "apply-answer":
-        return await _run_apply_answer(taskset, task, config)
-    if config.mode == "noop":
-        return await _run_noop(taskset, task, config)
+    if config.only_gold:
+        return await _run_gold(taskset, task, config)
+    if config.only_setup:
+        return await _run_setup(taskset, task, config)
     return await _run_all(taskset, task, config)
 
 
@@ -214,12 +214,15 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
         raise SystemExit(
             "taskset needs a container runtime to validate - pass --runtime.type docker (or prime)"
         )
+    checks = (
+        "gold" if config.only_gold else "setup" if config.only_setup else "gold+setup"
+    )
     logger.info(
-        "validating %d task(s) from %s on the %s runtime (mode=%s)",
+        "validating %d task(s) from %s on the %s runtime (%s)",
         len(tasks),
         config.name,
         config.runtime.type,
-        config.mode,
+        checks,
     )
 
     sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
diff --git a/verifiers/v1/configs/validate.py b/verifiers/v1/configs/validate.py
index d7dae17cec..15e79d7e85 100644
--- a/verifiers/v1/configs/validate.py
+++ b/verifiers/v1/configs/validate.py
@@ -8,16 +8,12 @@
 resume / dry-run.
 """
 
-from typing import Literal
-
 from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
 from verifiers.v1.taskset import TasksetConfig
 
-ValidateMode = Literal["apply-answer", "noop", "all"]
-
 
 class CheckTimeoutConfig(BaseConfig):
     """Per-task wall-clock timeouts for the model-free check CLIs (validate/debug), in
@@ -45,9 +41,12 @@ class ValidateConfig(BaseConfig):
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     """Per-task stage timeouts: `--timeout.setup` for the `setup` hook, `--timeout.total`
     for the `validate` hook."""
-    mode: ValidateMode = "apply-answer"
-    """Validation mode: `apply-answer` runs setup + validate, `noop` runs setup only, and
-    `all` runs every mode in independent runtimes and reports one aggregate row."""
+    only_setup: bool = False
+    """Run only the setup check per task (the `setup` hook, no gold check). By default
+    every check — setup-only and gold — runs in independent runtimes per task, reported
+    as one aggregate row."""
+    only_gold: bool = False
+    """Run only the gold check per task (`setup` + the `validate` hook)."""
     num_tasks: int | None = Field(
         None,
         validation_alias=AliasChoices("num_tasks", "n", "num_examples", "batch_size"),
@@ -68,6 +67,12 @@ class ValidateConfig(BaseConfig):
     def name(self) -> str:
         return self.taskset.name
 
+    @model_validator(mode="after")
+    def _validate_only(self):
+        if self.only_setup and self.only_gold:
+            raise ValueError("pass at most one of `--only-setup` or `--only-gold`")
+        return self
+
     @model_validator(mode="before")
     @classmethod
     def _resolve_taskset(cls, data):

From bc98d9e6e0a015400540349e5216a269f08fa21e Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 03:33:37 +0000
Subject: [PATCH 24/27] test: e2e trip-wire for the shielded-teardown guard on
 a real sandbox

Cancels the owner while stop() is in flight against a live Prime sandbox and
asserts, in-process: the DELETE still completed server-side (TERMINATED, not
RUNNING), the drain was reported, and the cancellation propagated afterwards.
Verified to fail when the run_shielded guard in Runtime.stop is removed.
prime-marked: runs under -m prime locally, excluded from CI.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tests/v1/test_e2e.py | 69 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py
index 91d702a0fc..11e1355e31 100644
--- a/tests/v1/test_e2e.py
+++ b/tests/v1/test_e2e.py
@@ -207,3 +207,72 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns >= 1  # ran a command, then finished
     assert trace.reward == 1.0
+
+
+@pytest.mark.prime
+async def test_cancelled_stop_still_deletes_sandbox(caplog, monkeypatch):
+    """The cancellation guard, end-to-end on a real sandbox: a Ctrl-C landing while
+    `stop()` is in flight must not truncate teardown (leaking the paid sandbox), must
+    report the drain, and must still propagate the cancellation afterwards. This is the
+    trip-wire for the `run_shielded` shield inside `Runtime.stop` — it fails with the
+    sandbox left RUNNING if the shield is removed or reordered."""
+    import asyncio
+    import contextlib
+    import logging
+    import time
+
+    from prime_sandboxes import SandboxClient
+    from prime_sandboxes.core import APIClient
+
+    from verifiers.v1.runtimes.prime import PrimeConfig, PrimeRuntime
+
+    class GatedPrime(PrimeRuntime):
+        """Real runtime; teardown holds a window open so the cancel lands mid-stop,
+        before the DELETE goes out."""
+
+        def __init__(self, *args, **kwargs) -> None:
+            super().__init__(*args, **kwargs)
+            self.teardown_entered = asyncio.Event()
+
+        async def teardown(self) -> None:
+            self.teardown_entered.set()
+            await asyncio.sleep(0.2)
+            await super().teardown()
+
+    monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
+    runtime = GatedPrime(PrimeConfig(labels=["vf-ci"]), name="cancel-guard-e2e")
+    await runtime.start()
+    sandbox_id = runtime.descriptor
+    assert sandbox_id is not None
+    try:
+
+        async def owner() -> None:
+            try:
+                pass  # the rollout body finished; teardown is on the happy path
+            finally:
+                await runtime.stop()
+
+        task = asyncio.create_task(owner())
+        await runtime.teardown_entered.wait()
+        with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
+            task.cancel()  # Ctrl-C while stop() is in flight
+            with pytest.raises(asyncio.CancelledError):
+                await task
+        assert task.cancelled()  # cancellation still propagated after teardown
+        assert any("Ctrl-C again" in r.getMessage() for r in caplog.records)
+
+        # Server-side truth, checked in-process (at exit the atexit backstop would mask
+        # the result): the DELETE ran to completion despite the cancellation.
+        client = SandboxClient(APIClient())
+        deadline = time.time() + 30
+        while (
+            status := await asyncio.to_thread(lambda: client.get(sandbox_id).status)
+        ) != "TERMINATED" and time.time() < deadline:
+            await asyncio.sleep(2)
+        assert status == "TERMINATED", (
+            f"sandbox {sandbox_id} leaked after a cancelled teardown (status={status})"
+        )
+    finally:
+        # A failing run must not keep paying for the sandbox (delete of a terminated one 404s).
+        with contextlib.suppress(Exception):
+            SandboxClient(APIClient()).delete(sandbox_id)

From 6de5883c3ba6b965bab3f6aa8b50c3f93d39b040 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Sat, 4 Jul 2026 04:49:46 +0000
Subject: [PATCH 25/27] test: run the cancellation trip-wire in CI
 (subprocess/docker), prime local
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The prime-marked variant alone never gates PRs (CI excludes real sandboxes),
so parametrize the guard test over the runtimes CI does run: subprocess
(always) and docker (CI matrix), with prime as the local full-realism
variant. The gate holds teardown open so the cancel lands before the real
cleanup starts — an unshielded stop() genuinely leaks the workdir/container
and fails the leak check. Verified: both CI variants fail with the shield
removed and pass with it intact.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tests/v1/test_e2e.py | 115 +++++++++++++++++++++++++++++--------------
 1 file changed, 77 insertions(+), 38 deletions(-)

diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py
index 11e1355e31..ecfd6fa2f4 100644
--- a/tests/v1/test_e2e.py
+++ b/tests/v1/test_e2e.py
@@ -209,41 +209,65 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     assert trace.reward == 1.0
 
 
-@pytest.mark.prime
-async def test_cancelled_stop_still_deletes_sandbox(caplog, monkeypatch):
-    """The cancellation guard, end-to-end on a real sandbox: a Ctrl-C landing while
-    `stop()` is in flight must not truncate teardown (leaking the paid sandbox), must
-    report the drain, and must still propagate the cancellation afterwards. This is the
-    trip-wire for the `run_shielded` shield inside `Runtime.stop` — it fails with the
-    sandbox left RUNNING if the shield is removed or reordered."""
+@pytest.mark.parametrize(
+    "runtime_type",
+    [
+        pytest.param("subprocess", id="cancel-guard-subprocess"),
+        pytest.param("docker", marks=pytest.mark.docker, id="cancel-guard-docker"),
+        pytest.param("prime", marks=pytest.mark.prime, id="cancel-guard-prime"),
+    ],
+)
+async def test_cancelled_stop_still_frees_runtime(runtime_type, caplog, monkeypatch):
+    """The cancellation guard, end-to-end on a real resource: a Ctrl-C landing while
+    `stop()` is in flight must not truncate teardown (leaking the workdir / container /
+    paid sandbox), must report the drain, and must still propagate the cancellation
+    afterwards. This is the trip-wire for the `run_shielded` shield inside
+    `Runtime.stop` — the gate below holds teardown open so the cancel deterministically
+    lands before the real cleanup starts, so an unshielded `stop()` genuinely leaks and
+    fails the leak check. subprocess/docker run in CI; prime is the local full-realism
+    variant (a real API DELETE observed server-side)."""
     import asyncio
     import contextlib
     import logging
+    import os
+    import subprocess
     import time
+    from pathlib import Path
+    from uuid import uuid4
 
-    from prime_sandboxes import SandboxClient
-    from prime_sandboxes.core import APIClient
+    from verifiers.v1.runtimes import make_runtime
+    from verifiers.v1.runtimes.docker import DockerConfig
+    from verifiers.v1.runtimes.prime import PrimeConfig
+    from verifiers.v1.runtimes.subprocess import SubprocessConfig
 
-    from verifiers.v1.runtimes.prime import PrimeConfig, PrimeRuntime
+    if runtime_type == "prime" and not os.environ.get("PRIME_API_KEY"):
+        pytest.skip("needs PRIME_API_KEY")
 
-    class GatedPrime(PrimeRuntime):
-        """Real runtime; teardown holds a window open so the cancel lands mid-stop,
-        before the DELETE goes out."""
+    config = {
+        "subprocess": lambda: SubprocessConfig(),
+        "docker": lambda: DockerConfig(),
+        "prime": lambda: PrimeConfig(labels=["vf-ci"]),
+    }[runtime_type]()
+    name = f"cancel-guard-{uuid4().hex[:8]}"
+    runtime = make_runtime(config, name=name)
 
-        def __init__(self, *args, **kwargs) -> None:
-            super().__init__(*args, **kwargs)
-            self.teardown_entered = asyncio.Event()
+    # Gate the real teardown: signal entry, hold the window open, then run it. The
+    # `finished` flag is the provider-agnostic trip — without the shield the cancel
+    # kills the sleep and the real teardown never runs.
+    entered, finished = asyncio.Event(), asyncio.Event()
+    real_teardown = runtime.teardown
 
-        async def teardown(self) -> None:
-            self.teardown_entered.set()
-            await asyncio.sleep(0.2)
-            await super().teardown()
+    async def gated_teardown() -> None:
+        entered.set()
+        await asyncio.sleep(0.2)
+        await real_teardown()
+        finished.set()
 
+    monkeypatch.setattr(runtime, "teardown", gated_teardown)
     monkeypatch.setattr(logging.getLogger("verifiers"), "propagate", True)
-    runtime = GatedPrime(PrimeConfig(labels=["vf-ci"]), name="cancel-guard-e2e")
+
     await runtime.start()
-    sandbox_id = runtime.descriptor
-    assert sandbox_id is not None
+    descriptor = runtime.descriptor
     try:
 
         async def owner() -> None:
@@ -253,26 +277,41 @@ async def owner() -> None:
                 await runtime.stop()
 
         task = asyncio.create_task(owner())
-        await runtime.teardown_entered.wait()
+        await entered.wait()
         with caplog.at_level(logging.WARNING, logger="verifiers.v1.runtimes.base"):
             task.cancel()  # Ctrl-C while stop() is in flight
             with pytest.raises(asyncio.CancelledError):
                 await task
-        assert task.cancelled()  # cancellation still propagated after teardown
+        assert finished.is_set()  # teardown ran to completion despite the cancel
+        assert task.cancelled()  # and the cancellation still propagated after it
         assert any("Ctrl-C again" in r.getMessage() for r in caplog.records)
 
-        # Server-side truth, checked in-process (at exit the atexit backstop would mask
-        # the result): the DELETE ran to completion despite the cancellation.
-        client = SandboxClient(APIClient())
-        deadline = time.time() + 30
-        while (
-            status := await asyncio.to_thread(lambda: client.get(sandbox_id).status)
-        ) != "TERMINATED" and time.time() < deadline:
-            await asyncio.sleep(2)
-        assert status == "TERMINATED", (
-            f"sandbox {sandbox_id} leaked after a cancelled teardown (status={status})"
-        )
+        # Provider-side truth, checked in-process (at exit the atexit backstop would
+        # mask the result): the resource is actually gone.
+        if runtime_type == "subprocess":
+            assert not Path(f"/tmp/{name}").exists(), "workdir leaked"
+        elif runtime_type == "docker":
+            leaked = subprocess.run(
+                ["docker", "ps", "-aq", "--filter", f"name={name}"],
+                capture_output=True,
+                text=True,
+                timeout=30,
+            ).stdout.strip()
+            assert not leaked, f"container {name} leaked after a cancelled teardown"
+        else:
+            from prime_sandboxes import SandboxClient
+            from prime_sandboxes.core import APIClient
+
+            client = SandboxClient(APIClient())
+            deadline = time.time() + 30
+            while (
+                status := await asyncio.to_thread(lambda: client.get(descriptor).status)
+            ) != "TERMINATED" and time.time() < deadline:
+                await asyncio.sleep(2)
+            assert status == "TERMINATED", (
+                f"sandbox {descriptor} leaked after a cancelled teardown ({status})"
+            )
     finally:
-        # A failing run must not keep paying for the sandbox (delete of a terminated one 404s).
+        # A failing run must not keep the resource around (cleanup is idempotent).
         with contextlib.suppress(Exception):
-            SandboxClient(APIClient()).delete(sandbox_id)
+            runtime.cleanup()

From 85cde88e3e043f7807ebbe0273b01859bbc16e40 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 8 Jul 2026 03:07:43 +0000
Subject: [PATCH 26/27] fix: report an aborted drain honestly, not as finished
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 'all in-flight teardowns finished' line fired whenever _STOPPING
emptied — including when a second Ctrl-C aborted teardowns mid-drain
(this branch makes run_shielded propagate user aborts immediately).
Track the abort and report 'teardown drain aborted' instead, so the
user who just aborted isn't told everything finished.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/runtimes/base.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py
index d2384de298..a1d3d46b3f 100644
--- a/verifiers/v1/runtimes/base.py
+++ b/verifiers/v1/runtimes/base.py
@@ -180,13 +180,25 @@ def interrupted() -> None:
 
         global _drain_reported
         _STOPPING[self] = time.time()
+        aborted = False
         try:
             await run_shielded(self.teardown(), interrupted=interrupted)
+        except (KeyboardInterrupt, SystemExit):
+            aborted = (
+                True  # a user abort cut the drain short (run_shielded propagates it)
+            )
+            raise
         finally:
             _STOPPING.pop(self, None)
             if _drain_reported and not _STOPPING:
                 _drain_reported = False
-                logger.warning("all in-flight teardowns finished")
+                if aborted:
+                    logger.warning(
+                        "teardown drain aborted — leftover resources are freed "
+                        "best-effort at exit"
+                    )
+                else:
+                    logger.warning("all in-flight teardowns finished")
 
     async def teardown(self) -> None:
         """Free the provisioned resource, off the event loop. Override only for teardown

From b2a917eea5e1c8984e91c169b240ef9191e35c83 Mon Sep 17 00:00:00 2001
From: rasdani <73563550+rasdani@users.noreply.github.com>
Date: Wed, 8 Jul 2026 03:50:07 +0000
Subject: [PATCH 27/27] fix: surface warnings under the --rich dashboard
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

setup_logging(console=False) left loguru with no sink at all when no
log_file was given (validate), silently discarding the teardown-drain
warnings this branch adds — Ctrl-C still looked like a hang in the
default validate UX. Add a WARNING-level sink through the dashboard's
own rich Console: an active Live renders prints above the live region
cleanly, and eval's rich mode gets on-screen warnings too instead of
file-only. Verified under a pty: the warning renders above the frame,
no tearing, INFO stays hidden.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 verifiers/v1/utils/logging.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/verifiers/v1/utils/logging.py b/verifiers/v1/utils/logging.py
index 9ee900a17e..8cf487792f 100644
--- a/verifiers/v1/utils/logging.py
+++ b/verifiers/v1/utils/logging.py
@@ -12,6 +12,7 @@
 from pathlib import Path
 
 from loguru import logger
+from rich import get_console
 
 LIBRARY_LOGGER = "verifiers.v1"
 FORMAT = (
@@ -41,11 +42,21 @@ def setup_logging(
 ) -> None:
     """Route the library's stdlib logs through loguru, to stderr (when `console`) and/or a
     `log_file`. `console=False` (used under the `--rich` dashboard, which owns the screen)
-    keeps logs off the terminal while still writing them to `log_file`."""
+    keeps routine logs off the terminal while still writing them to `log_file` — but
+    warnings still surface, printed through the dashboard's own rich Console so an active
+    `Live` renders them above the live region instead of tearing the frame (a Ctrl-C
+    teardown drain must not look like a hang)."""
     lvl = level.upper()
     logger.remove()
     if console:
         logger.add(sys.stderr, level=lvl, format=FORMAT)
+    else:
+        rich_console = get_console()
+        logger.add(
+            lambda m: rich_console.print(m, end="", markup=False, highlight=False),
+            level="WARNING",
+            format=FORMAT,
+        )
     if log_file is not None:
         Path(log_file).parent.mkdir(parents=True, exist_ok=True)
         logger.add(log_file, level=lvl, format=FORMAT)