Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/agent_env/bundle/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from agent_env.store.image_store.oci_registry_credentials import names_registry
from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies
from agent_env.task_step.task_step import TaskStep, attach_base_fields, dependencies
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents

from ._fs import fold, relative, show
Expand Down Expand Up @@ -562,7 +562,7 @@ def build_step(step: dict) -> TaskStep:
"""The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes
never changes."""
step = copy.deepcopy(step)
return attach_retry_config(get_task_step_registry()[step["type"]].from_dict(step), step)
return attach_base_fields(get_task_step_registry()[step["type"]].from_dict(step), step)


def _nfc(name: str) -> str:
Expand Down
15 changes: 7 additions & 8 deletions src/agent_env/cli/task/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,13 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]:
raise SystemExit(1)
kwargs = {k: v for k, v in step_dict.items() if k != "type"}
kwargs.setdefault("version", None)
# retry_config lives on the TaskStep base but most step __init__s don't
# accept it; construct without it and hydrate (as Task.from_dict does),
# so retry_config on any step type doesn't crash create. put() below
# handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_retry_config

ctor_kwargs = {k: v for k, v in kwargs.items() if k != "retry_config"}
step = attach_retry_config(step_cls(**ctor_kwargs), step_dict)
# retry_config and step_role live on the TaskStep base but most step
# __init__s don't accept them; construct without them and hydrate (as
# Task.from_dict does). put() below handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_base_fields

ctor_kwargs = {k: v for k, v in kwargs.items() if k not in ("retry_config", "step_role")}
step = attach_base_fields(step_cls(**ctor_kwargs), step_dict)
built.append((kwargs, step))
return built

Expand Down
5 changes: 3 additions & 2 deletions src/agent_env/task/episode/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from agent_env.task.store import new_instance_id
from agent_env.task.task import Task, substitute_seed_in_steps
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_step import TaskStep, attach_base_fields
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step


Expand Down Expand Up @@ -100,5 +100,6 @@ def _in_episode(step: TaskStep, as_run: TaskStep, sandboxed: dict[str, bool], tu
depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error,
)
if as_run.type == "prompt_agent" and as_run.agent_name in sandboxed and turn_timeout is not None:
return PromptAgentTaskStep.from_dict({**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)})
data = {**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)}
return attach_base_fields(PromptAgentTaskStep.from_dict(data), data)
return step
4 changes: 2 additions & 2 deletions src/agent_env/task/episode/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from agent_env.task.episode.run import _episode_of
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_step import TaskStep, attach_base_fields
from agent_env.task_step.task_steps.deploy_agent import bound_deployment, live_mcp_url
from agent_env.task_step.task_steps.install_agent import PROXY_PARAM

Expand Down Expand Up @@ -38,7 +38,7 @@ def to_dict(self) -> dict[str, Any]:

@classmethod
def from_dict(cls, data: dict[str, Any]) -> _EpisodeAgentStep:
step = get_task_step_registry()[data["step"]["type"]].from_dict(data["step"])
step = attach_base_fields(get_task_step_registry()[data["step"]["type"]].from_dict(data["step"]), data["step"])
return cls(**cls._base_from_dict(data), step=step, sandboxed=data["sandboxed"])

async def execute(self, context: TaskStepContext) -> TaskStepContext:
Expand Down
203 changes: 203 additions & 0 deletions src/agent_env/task/outcome.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
"""What a step failure means for its run: the role the failed step played, why it failed, and whether the completed
run is degraded.

A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what
the run lost. Each ``context.metadata.failed_steps`` entry carries them, and a completed run's ``quality`` sums them up.
"""

from __future__ import annotations

import re
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass, field
from enum import StrEnum
from typing import TYPE_CHECKING, Any

import httpx

from agent_env.task_step.context import VerifierNotPassingError
from agent_env.task_step.task_step import StepRole

if TYPE_CHECKING:
from agent_env.task_step.task_step import TaskStep

__all__ = ["FailedStep", "FailureCause", "RunQuality", "StepRole", "classify_failure", "run_quality"]


class FailureCause(StrEnum):
KEY_BUDGET = "key_budget"
KEY_AUTH = "key_auth"
TRANSIENT = "transient"
# The grader ran and the run failed its check: the grade exists, so nothing was lost.
VERDICT = "verdict"
UNKNOWN = "unknown"


# Each pattern matches only wording that the model proxy, the HTTP client or the model SDK writes. An error's text also
# carries env responses, tool output and model text, where a bare "401" or "status 500" says nothing about our key or
# our infra. Checked in order: a key that is out of budget or rejected fails every retry, so it wins over a timeout.
_CAUSE_PATTERNS: tuple[tuple[FailureCause, re.Pattern[str]], ...] = (
(FailureCause.KEY_BUDGET, re.compile(
r"Budget has been exceeded!|ExceededBudget:|ExceededTokenBudget|BudgetExceededError"
r"|Budget exceeded for \S+: spent \$"
)),
(FailureCause.KEY_AUTH, re.compile(
r"Authentication Error(?:,| -|:)|Invalid proxy server token|token_not_found_in_db|litellm\.AuthenticationError"
)),
(FailureCause.TRANSIENT, re.compile(
r"\bAPI Error: (?:Request rejected \()?(?:5\d\d|429)\b|\bError code: (?:5\d\d|429)\b"
r'|\bAPI Error: \{"error":\{[^\n]*"code":"(?:5\d\d|429)"\}\}|All connection attempts failed'
r"|unexpected status (?:408|429|5\d\d)\b|Server error '5\d\d |Client error '429 |\brawError=429\b"
r"|Server disconnected without sending a response|Request timed out"
r"|\b(?:litellm|openai)\.(?:RateLimitError|APIConnectionError|APITimeoutError|Timeout|InternalServerError"
r"|ServiceUnavailableError)\b"
r"|OOMKilled|exit(?:ed)? (?:with )?code 137"
)),
)

# A failure of one of these types is one the A2A client or the model SDK retries, whatever its message says.
_TRANSIENT_TYPES = (TimeoutError, ConnectionError, httpx.TransportError)


def _chain(exc: BaseException) -> Iterator[BaseException]:
"""``exc`` and every exception it was explicitly raised from, each once.

Only ``__cause__`` (``raise ... from``) and an exception group's members count. ``__context__`` is set for any
exception raised inside an ``except``, including one the code had already recovered from, so it can't say why.
"""
seen: set[int] = set()
stack: list[BaseException | None] = [exc]
while stack:
current = stack.pop()
if current is None or id(current) in seen:
continue
seen.add(id(current))
yield current
stack.append(current.__cause__)
if isinstance(current, BaseExceptionGroup):
stack.extend(current.exceptions)


def classify_failure(exc: BaseException) -> FailureCause:
"""Why ``exc`` happened, read from the types and messages of ``exc`` and the exceptions it was raised from.

Matching text is the only option for most causes: a judge or agent error crosses A2A and the model proxy as a
message string, so its original type is gone by the time a step raises.
"""
if isinstance(exc, VerifierNotPassingError):
return FailureCause.VERDICT
chain = list(_chain(exc))
text = "\n".join(f"{type(e).__name__}: {e}" for e in chain)
for cause, pattern in _CAUSE_PATTERNS:
if pattern.search(text):
return cause
if any(isinstance(e, _TRANSIENT_TYPES) for e in chain):
return FailureCause.TRANSIENT
return FailureCause.UNKNOWN


@dataclass(frozen=True)
class FailedStep:
"""One ``context.metadata.failed_steps`` entry."""

step_id: str
step_type: str
error: str
error_type: str
started_at_utc: str
duration_seconds: float
is_fatal: bool
retried: bool = False
step_role: StepRole = StepRole.OTHER
cause: FailureCause = FailureCause.UNKNOWN
# Set once a later action (e.g. a regrade) repaired what this failure lost.
resolved_by: str | None = None

@classmethod
def from_exception(
cls, step: TaskStep, exc: BaseException, *, started_at_utc: str, duration_seconds: float, is_fatal: bool,
) -> FailedStep:
return cls(
step_id=step.id,
step_type=step.type,
error=str(exc),
error_type=type(exc).__name__,
started_at_utc=started_at_utc,
duration_seconds=duration_seconds,
is_fatal=is_fatal,
step_role=getattr(step, "step_role", StepRole.OTHER),
cause=classify_failure(exc),
)

@classmethod
def from_dict(cls, data: Mapping[str, Any]) -> FailedStep:
"""An entry as stored. One written before ``step_role`` and ``cause`` existed, or holding a value this version
does not know, reads as ``other`` and ``unknown``."""
return cls(
step_id=data.get("step_id", ""),
step_type=data.get("step_type", ""),
error=data.get("error", ""),
error_type=data.get("error_type", ""),
started_at_utc=data.get("started_at_utc", ""),
duration_seconds=data.get("duration_seconds", 0.0),
is_fatal=bool(data.get("is_fatal", True)),
retried=bool(data.get("retried", False)),
step_role=StepRole(data["step_role"]) if data.get("step_role") in _ROLES else StepRole.OTHER,
cause=FailureCause(data["cause"]) if data.get("cause") in _CAUSES else FailureCause.UNKNOWN,
resolved_by=data.get("resolved_by"),
)

def to_dict(self) -> dict[str, Any]:
"""The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and
``resolved_by`` appear only once set, as ``retried`` always has."""
out: dict[str, Any] = {
"step_id": self.step_id, "step_type": self.step_type,
"error": self.error, "error_type": self.error_type,
"started_at_utc": self.started_at_utc, "duration_seconds": self.duration_seconds,
"is_fatal": self.is_fatal,
"step_role": self.step_role.value, "cause": self.cause.value,
}
if self.retried:
out["retried"] = True
if self.resolved_by is not None:
out["resolved_by"] = self.resolved_by
return out


_ROLES = frozenset(StepRole)
_CAUSES = frozenset(FailureCause)
# Most severe first: a run with no grade is worse than one missing its agent's turn, which is worse than one missing
# evidence.
_SEVERITY = (StepRole.GRADER, StepRole.AGENT, StepRole.EVIDENCE, StepRole.OTHER)


@dataclass(frozen=True)
class RunQuality:
"""How much of a completed run is usable. ``reasons`` holds the roles of the steps whose failure the run tolerated
and nothing has repaired, most severe first; a run with none is clean."""

reasons: tuple[StepRole, ...] = ()
# Derived, never passed, so it cannot disagree with ``reasons``; a field so ``asdict`` matches the stored form.
degraded: bool = field(init=False)

def __post_init__(self) -> None:
object.__setattr__(self, "degraded", bool(self.reasons))

@classmethod
def from_dict(cls, data: Mapping[str, Any] | None) -> RunQuality:
"""``quality`` as stored. A run completed before ``quality`` existed has none and reads as clean."""
roles = {StepRole(r) for r in (data or {}).get("reasons", ()) if r in _ROLES}
return cls(tuple(r for r in _SEVERITY if r in roles))

def to_dict(self) -> dict[str, Any]:
return {"degraded": self.degraded, "reasons": [r.value for r in self.reasons]}


def run_quality(failed_steps: Iterable[FailedStep]) -> RunQuality:
"""A fatal failure fails the run instead, a retried or resolved one lost nothing, and a failing verdict is a grade
the run has, not one it lost."""
open_roles = {
f.step_role for f in failed_steps
if not f.is_fatal and not f.retried and f.resolved_by is None and f.cause != FailureCause.VERDICT
}
return RunQuality(tuple(r for r in _SEVERITY if r in open_roles))
11 changes: 11 additions & 0 deletions src/agent_env/task/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
commit_ordered,
replay_context,
)
from agent_env.task.outcome import FailedStep, RunQuality, run_quality
from agent_env.task_step.context import TaskStepContext

if TYPE_CHECKING:
Expand Down Expand Up @@ -184,6 +185,8 @@ class TaskInstance:
# carries the epoch it was dispatched under and no-ops when older than the stored one,
# fencing a writer from a rolled-back attempt. Missing (pre-field docs) reads as 0.
attempt_epoch: int = 0
# Stamped with ``status="completed"``; clean until then, and on a run completed before the field existed.
quality: RunQuality = field(default_factory=RunQuality)

@classmethod
def from_dict(cls, data: dict) -> TaskInstance:
Expand All @@ -202,6 +205,7 @@ def from_dict(cls, data: dict) -> TaskInstance:
completed_steps=[TaskStepResult.from_dict(e) for e in data.get("completed_steps", [])],
step_attempt_failures=data.get("step_attempt_failures", []),
attempt_epoch=data.get("attempt_epoch", 0),
quality=RunQuality.from_dict(data.get("quality")),
)


Expand Down Expand Up @@ -717,6 +721,13 @@ def _mutate(doc: dict) -> UpdateSpec | None:
if len(completed) >= total_steps and doc.get("status") not in ("failed", "cancelled"):
set_fields["status"] = "completed"
set_fields["completed_at_utc"] = completed_at_utc
# In the same write as the status, so no reader sees a completed run without its quality.
after = {"context": copy.deepcopy(doc.get("context"))}
_apply_forward(after, ops, move_to_end=rerecorded)
failed_steps = _get_path(after, "context.metadata.failed_steps") or []
set_fields["quality"] = run_quality(
FailedStep.from_dict(f) for f in failed_steps if isinstance(f, dict)
).to_dict()
return UpdateSpec(set=set_fields, unset=set(ops.unsets))

result = compare_and_swap(
Expand Down
18 changes: 8 additions & 10 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from agent_env.store.document_store import UnknownDocumentTypeError
from agent_env.store.routing import run_scope
from agent_env.task_step.preflight import pinned
from agent_env.task.outcome import FailedStep
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
Expand Down Expand Up @@ -130,12 +131,12 @@ def _sub(value):
return {k: _sub(v) for k, v in value.items()}
return value

from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

out: list["TaskStep"] = []
for step in steps:
data = _sub(step.to_dict())
seeded = attach_retry_config(type(step).from_dict(data), data)
seeded = attach_base_fields(type(step).from_dict(data), data)
if (keep := getattr(seeded, "seeded_from", None)) is not None: # a step that keeps something as written
keep(step)
out.append(seeded)
Expand Down Expand Up @@ -463,9 +464,9 @@ def from_dict(cls, data: dict) -> "Task":
f"{_registration.failure_note(_registration.TASK_STEPS, step_type)}", *problems]),
unmet,
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

steps.append(attach_retry_config(step_cls.from_dict(step_dict), step_dict))
steps.append(attach_base_fields(step_cls.from_dict(step_dict), step_dict))
return cls(
id=data["id"],
version=data.get("version"),
Expand Down Expand Up @@ -684,12 +685,9 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None:
except BaseException as exc:
duration = time.monotonic() - started_mono
fatal = getattr(step, "fail_task_on_error", True)
context.metadata.setdefault("failed_steps", []).append({
"step_id": step.id, "step_type": step.type,
"error": str(exc), "error_type": type(exc).__name__,
"started_at_utc": started_utc, "duration_seconds": duration,
"is_fatal": fatal,
})
context.metadata.setdefault("failed_steps", []).append(FailedStep.from_exception(
step, exc, started_at_utc=started_utc, duration_seconds=duration, is_fatal=fatal,
).to_dict())
if not fatal:
await _heartbeat_step(step, pre_context, _store.TaskStepStatus.FAILURE, dispatch_epoch)
raise
Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/task_step/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ def _deserialize(self, doc: dict) -> TaskStep:
f"Unknown task step type: {task_step_type}"
f"{_registration.failure_note(_registration.TASK_STEPS, task_step_type)}"
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

return attach_retry_config(cls.from_dict(doc), doc)
return attach_base_fields(cls.from_dict(doc), doc)

def execute_query(self, query: TaskStepQuery) -> list[TaskStep]:
filt, sort = to_document_query(query)
Expand Down
Loading
Loading