Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/agent_env/bundle/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from agent_env.store.image_store.oci_registry_credentials import names_registry
from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies
from agent_env.task_step.task_step import TaskStep, attach_base_fields, dependencies
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents

from ._fs import fold, relative, show
Expand Down Expand Up @@ -562,7 +562,7 @@ def build_step(step: dict) -> TaskStep:
"""The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes
never changes."""
step = copy.deepcopy(step)
return attach_retry_config(get_task_step_registry()[step["type"]].from_dict(step), step)
return attach_base_fields(get_task_step_registry()[step["type"]].from_dict(step), step)


def _nfc(name: str) -> str:
Expand Down
15 changes: 7 additions & 8 deletions src/agent_env/cli/task/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,13 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]:
raise SystemExit(1)
kwargs = {k: v for k, v in step_dict.items() if k != "type"}
kwargs.setdefault("version", None)
# retry_config lives on the TaskStep base but most step __init__s don't
# accept it; construct without it and hydrate (as Task.from_dict does),
# so retry_config on any step type doesn't crash create. put() below
# handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_retry_config

ctor_kwargs = {k: v for k, v in kwargs.items() if k != "retry_config"}
step = attach_retry_config(step_cls(**ctor_kwargs), step_dict)
# retry_config and step_role live on the TaskStep base but most step
# __init__s don't accept them; construct without them and hydrate (as
# Task.from_dict does). put() below handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_base_fields

ctor_kwargs = {k: v for k, v in kwargs.items() if k not in ("retry_config", "step_role")}
step = attach_base_fields(step_cls(**ctor_kwargs), step_dict)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
built.append((kwargs, step))
return built

Expand Down
5 changes: 3 additions & 2 deletions src/agent_env/task/episode/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from agent_env.task.store import new_instance_id
from agent_env.task.task import Task, substitute_seed_in_steps
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_step import TaskStep, attach_base_fields
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step


Expand Down Expand Up @@ -100,5 +100,6 @@ def _in_episode(step: TaskStep, as_run: TaskStep, sandboxed: dict[str, bool], tu
depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error,
)
if as_run.type == "prompt_agent" and as_run.agent_name in sandboxed and turn_timeout is not None:
return PromptAgentTaskStep.from_dict({**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)})
data = {**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)}
return attach_base_fields(PromptAgentTaskStep.from_dict(data), data)
return step
4 changes: 2 additions & 2 deletions src/agent_env/task/episode/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from agent_env.task.episode.run import _episode_of
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_step import TaskStep, attach_base_fields
from agent_env.task_step.task_steps.deploy_agent import bound_deployment, live_mcp_url
from agent_env.task_step.task_steps.install_agent import PROXY_PARAM

Expand Down Expand Up @@ -38,7 +38,7 @@ def to_dict(self) -> dict[str, Any]:

@classmethod
def from_dict(cls, data: dict[str, Any]) -> _EpisodeAgentStep:
step = get_task_step_registry()[data["step"]["type"]].from_dict(data["step"])
step = attach_base_fields(get_task_step_registry()[data["step"]["type"]].from_dict(data["step"]), data["step"])
return cls(**cls._base_from_dict(data), step=step, sandboxed=data["sandboxed"])

async def execute(self, context: TaskStepContext) -> TaskStepContext:
Expand Down
146 changes: 146 additions & 0 deletions src/agent_env/task/outcome.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
"""What a step failure means for its run: the role the failed step played and why it failed.

A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what
the run lost. Each ``context.metadata.failed_steps`` entry carries them.
"""

from __future__ import annotations

import re
from collections.abc import Iterator
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING, Any

import httpx

from agent_env.task_step.context import VerifierNotPassingError
from agent_env.task_step.task_step import StepRole

if TYPE_CHECKING:
from agent_env.task_step.task_step import TaskStep

__all__ = ["FailedStep", "FailureCause", "StepRole", "classify_failure"]


class FailureCause(StrEnum):
KEY_BUDGET = "key_budget"
KEY_AUTH = "key_auth"
TRANSIENT = "transient"
# The grader ran and the run failed its check: the grade exists, so nothing was lost.
VERDICT = "verdict"
UNKNOWN = "unknown"


# Each pattern matches only wording that the model proxy, the HTTP client or the model SDK writes. An error's text also
# carries env responses, tool output and model text, where a bare "401" or "status 500" says nothing about our key or
# our infra. Checked in order: a key that is out of budget or rejected fails every retry, so it wins over a timeout.
_CAUSE_PATTERNS: tuple[tuple[FailureCause, re.Pattern[str]], ...] = (
(FailureCause.KEY_BUDGET, re.compile(
r"Budget has been exceeded!|ExceededBudget:|ExceededTokenBudget|BudgetExceededError"
r"|Budget exceeded for \S+: spent \$"
)),
(FailureCause.KEY_AUTH, re.compile(
r"Authentication Error(?:,| -|:)|Invalid proxy server token|token_not_found_in_db|litellm\.AuthenticationError"
)),
(FailureCause.TRANSIENT, re.compile(
r"\bAPI Error: (?:Request rejected \()?(?:5\d\d|429)\b|\bError code: (?:5\d\d|429)\b"
r'|\bAPI Error: \{"error":\{[^\n]*"code":"(?:5\d\d|429)"\}\}|All connection attempts failed'
r"|unexpected status (?:408|429|5\d\d)\b|Server error '5\d\d |Client error '429 |\brawError=429\b"
r"|Server disconnected without sending a response|Request timed out"
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
r"|\b(?:litellm|openai)\.(?:RateLimitError|APIConnectionError|APITimeoutError|Timeout|InternalServerError"
r"|ServiceUnavailableError)\b"
r"|OOMKilled|exit(?:ed)? (?:with )?code 137"
)),
)

# A failure of one of these types is one the A2A client or the model SDK retries, whatever its message says.
_TRANSIENT_TYPES = (TimeoutError, ConnectionError, httpx.TransportError)


def _chain(exc: BaseException) -> Iterator[BaseException]:
"""``exc`` and every exception it was explicitly raised from, each once.

Only ``__cause__`` (``raise ... from``) and an exception group's members count. ``__context__`` is set for any
exception raised inside an ``except``, including one the code had already recovered from, so it can't say why.
"""
seen: set[int] = set()
stack: list[BaseException | None] = [exc]
while stack:
current = stack.pop()
if current is None or id(current) in seen:
continue
seen.add(id(current))
yield current
stack.append(current.__cause__)
if isinstance(current, BaseExceptionGroup):
stack.extend(current.exceptions)


def classify_failure(exc: BaseException) -> FailureCause:
"""Why ``exc`` happened, read from the types and messages of ``exc`` and the exceptions it was raised from.

Matching text is the only option for most causes: a judge or agent error crosses A2A and the model proxy as a
message string, so its original type is gone by the time a step raises.
"""
if isinstance(exc, VerifierNotPassingError):
return FailureCause.VERDICT
chain = list(_chain(exc))
text = "\n".join(f"{type(e).__name__}: {e}" for e in chain)
for cause, pattern in _CAUSE_PATTERNS:
if pattern.search(text):
return cause
if any(isinstance(e, _TRANSIENT_TYPES) for e in chain):
return FailureCause.TRANSIENT
return FailureCause.UNKNOWN


@dataclass(frozen=True)
class FailedStep:
"""One ``context.metadata.failed_steps`` entry."""

step_id: str
step_type: str
error: str
error_type: str
started_at_utc: str
duration_seconds: float
is_fatal: bool
retried: bool = False
step_role: StepRole = StepRole.OTHER
cause: FailureCause = FailureCause.UNKNOWN
# Set once a later action (e.g. a regrade) repaired what this failure lost.
resolved_by: str | None = None

@classmethod
def from_exception(
cls, step: TaskStep, exc: BaseException, *, started_at_utc: str, duration_seconds: float, is_fatal: bool,
) -> FailedStep:
return cls(
step_id=step.id,
step_type=step.type,
error=str(exc),
error_type=type(exc).__name__,
started_at_utc=started_at_utc,
duration_seconds=duration_seconds,
is_fatal=is_fatal,
step_role=getattr(step, "step_role", StepRole.OTHER),
cause=classify_failure(exc),
)

def to_dict(self) -> dict[str, Any]:
"""The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and
``resolved_by`` appear only once set, as ``retried`` always has."""
out: dict[str, Any] = {
"step_id": self.step_id, "step_type": self.step_type,
"error": self.error, "error_type": self.error_type,
"started_at_utc": self.started_at_utc, "duration_seconds": self.duration_seconds,
"is_fatal": self.is_fatal,
"step_role": self.step_role.value, "cause": self.cause.value,
}
if self.retried:
out["retried"] = True
if self.resolved_by is not None:
out["resolved_by"] = self.resolved_by
return out

18 changes: 8 additions & 10 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from agent_env.store.document_store import UnknownDocumentTypeError
from agent_env.store.routing import run_scope
from agent_env.task_step.preflight import pinned
from agent_env.task.outcome import FailedStep
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
Expand Down Expand Up @@ -130,12 +131,12 @@ def _sub(value):
return {k: _sub(v) for k, v in value.items()}
return value

from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

out: list["TaskStep"] = []
for step in steps:
data = _sub(step.to_dict())
seeded = attach_retry_config(type(step).from_dict(data), data)
seeded = attach_base_fields(type(step).from_dict(data), data)
if (keep := getattr(seeded, "seeded_from", None)) is not None: # a step that keeps something as written
keep(step)
out.append(seeded)
Expand Down Expand Up @@ -463,9 +464,9 @@ def from_dict(cls, data: dict) -> "Task":
f"{_registration.failure_note(_registration.TASK_STEPS, step_type)}", *problems]),
unmet,
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

steps.append(attach_retry_config(step_cls.from_dict(step_dict), step_dict))
steps.append(attach_base_fields(step_cls.from_dict(step_dict), step_dict))
return cls(
id=data["id"],
version=data.get("version"),
Expand Down Expand Up @@ -684,12 +685,9 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None:
except BaseException as exc:
duration = time.monotonic() - started_mono
fatal = getattr(step, "fail_task_on_error", True)
context.metadata.setdefault("failed_steps", []).append({
"step_id": step.id, "step_type": step.type,
"error": str(exc), "error_type": type(exc).__name__,
"started_at_utc": started_utc, "duration_seconds": duration,
"is_fatal": fatal,
})
context.metadata.setdefault("failed_steps", []).append(FailedStep.from_exception(
step, exc, started_at_utc=started_utc, duration_seconds=duration, is_fatal=fatal,
).to_dict())
if not fatal:
await _heartbeat_step(step, pre_context, _store.TaskStepStatus.FAILURE, dispatch_epoch)
raise
Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/task_step/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ def _deserialize(self, doc: dict) -> TaskStep:
f"Unknown task step type: {task_step_type}"
f"{_registration.failure_note(_registration.TASK_STEPS, task_step_type)}"
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

return attach_retry_config(cls.from_dict(doc), doc)
return attach_base_fields(cls.from_dict(doc), doc)

def execute_query(self, query: TaskStepQuery) -> list[TaskStep]:
filt, sort = to_document_query(query)
Expand Down
34 changes: 24 additions & 10 deletions src/agent_env/task_step/task_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self

from agent_env.task_step.context import TaskStepContext
Expand Down Expand Up @@ -40,13 +41,23 @@ def dependencies(raw: Any) -> list[TaskStepDependency] | None:
return [dep if isinstance(dep, TaskStepDependency) else TaskStepDependency.from_dict(dep) for dep in raw]


def attach_retry_config(step: "TaskStep", data: dict[str, Any]) -> "TaskStep":
"""Hydrate ``retry_config`` after ``step_cls.from_dict`` at the generic
parse sites, so the field survives round-trips on step types whose
``__init__`` doesn't accept it. No-op if already set or absent."""
class StepRole(StrEnum):
"""What a step contributes to a run, so a tolerated failure says what the run lost."""

GRADER = "grader"
EVIDENCE = "evidence"
AGENT = "agent"
OTHER = "other"


def attach_base_fields(step: "TaskStep", data: dict[str, Any]) -> "TaskStep":
"""Hydrate the base fields most step ``__init__``s don't accept (``retry_config``, ``step_role``) after
``step_cls.from_dict`` at the generic parse sites, so they survive round-trips on every step type."""
raw = data.get("retry_config")
if raw is not None and getattr(step, "retry_config", None) is None:
step.retry_config = RetryConfig.from_dict(raw)
if data.get("step_role") is not None:
step.step_role = StepRole(data["step_role"])
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
return step


Expand Down Expand Up @@ -97,6 +108,9 @@ class TaskStep(ABC):
# Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A
# subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs.
field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER}
# A step type declares its role; one step overrides it in config, e.g. a run_code step that grades. Not
# ``role``: deploy_agent and modify_env_tool_access already use that name for their own config.
step_role: StepRole = StepRole.OTHER

def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
Expand Down Expand Up @@ -163,6 +177,8 @@ def to_dict(self) -> dict[str, Any]:
base["depends_on"] = [dataclasses.asdict(d) for d in self.depends_on]
if self.retry_config is not None:
base["retry_config"] = dataclasses.asdict(self.retry_config)
if self.step_role != type(self).step_role:
base["step_role"] = self.step_role.value
return base

@classmethod
Expand All @@ -188,13 +204,11 @@ def put(cls, **kwargs: Any) -> Self:
from .store import get_task_step_store

kwargs.setdefault("version", None)
# Most step __init__s don't accept retry_config (it lives on the base but
# subclasses don't forward it); construct without it, then hydrate so a
# retry_config on any step type round-trips through the step store.
retry_config_raw = kwargs.pop("retry_config", None)
# Most step __init__s don't accept the base-only fields; construct without
# them, then hydrate so they round-trip through the step store on any type.
base_fields = {k: kwargs.pop(k) for k in ("retry_config", "step_role") if k in kwargs}
instance = cls(**kwargs)
if retry_config_raw is not None:
attach_retry_config(instance, {"retry_config": retry_config_raw})
attach_base_fields(instance, base_fields)
return get_task_step_store().put_document(instance)

@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)

Expand All @@ -17,6 +17,7 @@

class VerifyA2AAgentCardStep(TaskStep):
type: ClassVar[str] = "verify_a2a_agent_card"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),)

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@
from agent_env.a2a_agent import A2AAgent
from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)


class VerifyA2AAgentConfigIdentityStep(TaskStep):
type: ClassVar[str] = "verify_a2a_agent_config_identity"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),)

def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True):
Expand Down
Loading
Loading