diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index c5dcf930c..4a5fa6ea8 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -34,7 +34,7 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies +from agent_env.task_step.task_step import TaskStep, attach_base_fields, dependencies from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from ._fs import fold, relative, show @@ -562,7 +562,7 @@ def build_step(step: dict) -> TaskStep: """The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes never changes.""" step = copy.deepcopy(step) - return attach_retry_config(get_task_step_registry()[step["type"]].from_dict(step), step) + return attach_base_fields(get_task_step_registry()[step["type"]].from_dict(step), step) def _nfc(name: str) -> str: diff --git a/src/agent_env/cli/task/create.py b/src/agent_env/cli/task/create.py index 8bc96849b..4475219da 100644 --- a/src/agent_env/cli/task/create.py +++ b/src/agent_env/cli/task/create.py @@ -74,14 +74,13 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]: raise SystemExit(1) kwargs = {k: v for k, v in step_dict.items() if k != "type"} kwargs.setdefault("version", None) - # retry_config lives on the TaskStep base but most step __init__s don't - # accept it; construct without it and hydrate (as Task.from_dict does), - # so retry_config on any step type doesn't crash create. put() below - # handles the kwargs form the same way. - from agent_env.task_step.task_step import attach_retry_config - - ctor_kwargs = {k: v for k, v in kwargs.items() if k != "retry_config"} - step = attach_retry_config(step_cls(**ctor_kwargs), step_dict) + # retry_config and step_role live on the TaskStep base but most step + # __init__s don't accept them; construct without them and hydrate (as + # Task.from_dict does). put() below handles the kwargs form the same way. + from agent_env.task_step.task_step import attach_base_fields + + ctor_kwargs = {k: v for k, v in kwargs.items() if k not in ("retry_config", "step_role")} + step = attach_base_fields(step_cls(**ctor_kwargs), step_dict) built.append((kwargs, step)) return built diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 0b9dc6593..4de2b033d 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -10,7 +10,7 @@ from agent_env.task.store import new_instance_id from agent_env.task.task import Task, substitute_seed_in_steps from agent_env.task_step.context import TaskStepContext, sandbox_overrides -from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_step import TaskStep, attach_base_fields from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step @@ -100,5 +100,6 @@ def _in_episode(step: TaskStep, as_run: TaskStep, sandboxed: dict[str, bool], tu depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error, ) if as_run.type == "prompt_agent" and as_run.agent_name in sandboxed and turn_timeout is not None: - return PromptAgentTaskStep.from_dict({**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)}) + data = {**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)} + return attach_base_fields(PromptAgentTaskStep.from_dict(data), data) return step diff --git a/src/agent_env/task/episode/steps.py b/src/agent_env/task/episode/steps.py index 9bcc85816..a7dc469a8 100644 --- a/src/agent_env/task/episode/steps.py +++ b/src/agent_env/task/episode/steps.py @@ -9,7 +9,7 @@ from agent_env.task.episode.run import _episode_of from agent_env.task_step.context import TaskStepContext from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_step import TaskStep, attach_base_fields from agent_env.task_step.task_steps.deploy_agent import bound_deployment, live_mcp_url from agent_env.task_step.task_steps.install_agent import PROXY_PARAM @@ -38,7 +38,7 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: dict[str, Any]) -> _EpisodeAgentStep: - step = get_task_step_registry()[data["step"]["type"]].from_dict(data["step"]) + step = attach_base_fields(get_task_step_registry()[data["step"]["type"]].from_dict(data["step"]), data["step"]) return cls(**cls._base_from_dict(data), step=step, sandboxed=data["sandboxed"]) async def execute(self, context: TaskStepContext) -> TaskStepContext: diff --git a/src/agent_env/task/outcome.py b/src/agent_env/task/outcome.py new file mode 100644 index 000000000..8baabcfea --- /dev/null +++ b/src/agent_env/task/outcome.py @@ -0,0 +1,203 @@ +"""What a step failure means for its run: the role the failed step played, why it failed, and whether the completed +run is degraded. + +A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what +the run lost. Each ``context.metadata.failed_steps`` entry carries them, and a completed run's ``quality`` sums them up. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass, field +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +import httpx + +from agent_env.task_step.context import VerifierNotPassingError +from agent_env.task_step.task_step import StepRole + +if TYPE_CHECKING: + from agent_env.task_step.task_step import TaskStep + +__all__ = ["FailedStep", "FailureCause", "RunQuality", "StepRole", "classify_failure", "run_quality"] + + +class FailureCause(StrEnum): + KEY_BUDGET = "key_budget" + KEY_AUTH = "key_auth" + TRANSIENT = "transient" + # The grader ran and the run failed its check: the grade exists, so nothing was lost. + VERDICT = "verdict" + UNKNOWN = "unknown" + + +# Each pattern matches only wording that the model proxy, the HTTP client or the model SDK writes. An error's text also +# carries env responses, tool output and model text, where a bare "401" or "status 500" says nothing about our key or +# our infra. Checked in order: a key that is out of budget or rejected fails every retry, so it wins over a timeout. +_CAUSE_PATTERNS: tuple[tuple[FailureCause, re.Pattern[str]], ...] = ( + (FailureCause.KEY_BUDGET, re.compile( + r"Budget has been exceeded!|ExceededBudget:|ExceededTokenBudget|BudgetExceededError" + r"|Budget exceeded for \S+: spent \$" + )), + (FailureCause.KEY_AUTH, re.compile( + r"Authentication Error(?:,| -|:)|Invalid proxy server token|token_not_found_in_db|litellm\.AuthenticationError" + )), + (FailureCause.TRANSIENT, re.compile( + r"\bAPI Error: (?:Request rejected \()?(?:5\d\d|429)\b|\bError code: (?:5\d\d|429)\b" + r'|\bAPI Error: \{"error":\{[^\n]*"code":"(?:5\d\d|429)"\}\}|All connection attempts failed' + r"|unexpected status (?:408|429|5\d\d)\b|Server error '5\d\d |Client error '429 |\brawError=429\b" + r"|Server disconnected without sending a response|Request timed out" + r"|\b(?:litellm|openai)\.(?:RateLimitError|APIConnectionError|APITimeoutError|Timeout|InternalServerError" + r"|ServiceUnavailableError)\b" + r"|OOMKilled|exit(?:ed)? (?:with )?code 137" + )), +) + +# A failure of one of these types is one the A2A client or the model SDK retries, whatever its message says. +_TRANSIENT_TYPES = (TimeoutError, ConnectionError, httpx.TransportError) + + +def _chain(exc: BaseException) -> Iterator[BaseException]: + """``exc`` and every exception it was explicitly raised from, each once. + + Only ``__cause__`` (``raise ... from``) and an exception group's members count. ``__context__`` is set for any + exception raised inside an ``except``, including one the code had already recovered from, so it can't say why. + """ + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + yield current + stack.append(current.__cause__) + if isinstance(current, BaseExceptionGroup): + stack.extend(current.exceptions) + + +def classify_failure(exc: BaseException) -> FailureCause: + """Why ``exc`` happened, read from the types and messages of ``exc`` and the exceptions it was raised from. + + Matching text is the only option for most causes: a judge or agent error crosses A2A and the model proxy as a + message string, so its original type is gone by the time a step raises. + """ + if isinstance(exc, VerifierNotPassingError): + return FailureCause.VERDICT + chain = list(_chain(exc)) + text = "\n".join(f"{type(e).__name__}: {e}" for e in chain) + for cause, pattern in _CAUSE_PATTERNS: + if pattern.search(text): + return cause + if any(isinstance(e, _TRANSIENT_TYPES) for e in chain): + return FailureCause.TRANSIENT + return FailureCause.UNKNOWN + + +@dataclass(frozen=True) +class FailedStep: + """One ``context.metadata.failed_steps`` entry.""" + + step_id: str + step_type: str + error: str + error_type: str + started_at_utc: str + duration_seconds: float + is_fatal: bool + retried: bool = False + step_role: StepRole = StepRole.OTHER + cause: FailureCause = FailureCause.UNKNOWN + # Set once a later action (e.g. a regrade) repaired what this failure lost. + resolved_by: str | None = None + + @classmethod + def from_exception( + cls, step: TaskStep, exc: BaseException, *, started_at_utc: str, duration_seconds: float, is_fatal: bool, + ) -> FailedStep: + return cls( + step_id=step.id, + step_type=step.type, + error=str(exc), + error_type=type(exc).__name__, + started_at_utc=started_at_utc, + duration_seconds=duration_seconds, + is_fatal=is_fatal, + step_role=getattr(step, "step_role", StepRole.OTHER), + cause=classify_failure(exc), + ) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> FailedStep: + """An entry as stored. One written before ``step_role`` and ``cause`` existed, or holding a value this version + does not know, reads as ``other`` and ``unknown``.""" + return cls( + step_id=data.get("step_id", ""), + step_type=data.get("step_type", ""), + error=data.get("error", ""), + error_type=data.get("error_type", ""), + started_at_utc=data.get("started_at_utc", ""), + duration_seconds=data.get("duration_seconds", 0.0), + is_fatal=bool(data.get("is_fatal", True)), + retried=bool(data.get("retried", False)), + step_role=StepRole(data["step_role"]) if data.get("step_role") in _ROLES else StepRole.OTHER, + cause=FailureCause(data["cause"]) if data.get("cause") in _CAUSES else FailureCause.UNKNOWN, + resolved_by=data.get("resolved_by"), + ) + + def to_dict(self) -> dict[str, Any]: + """The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and + ``resolved_by`` appear only once set, as ``retried`` always has.""" + out: dict[str, Any] = { + "step_id": self.step_id, "step_type": self.step_type, + "error": self.error, "error_type": self.error_type, + "started_at_utc": self.started_at_utc, "duration_seconds": self.duration_seconds, + "is_fatal": self.is_fatal, + "step_role": self.step_role.value, "cause": self.cause.value, + } + if self.retried: + out["retried"] = True + if self.resolved_by is not None: + out["resolved_by"] = self.resolved_by + return out + + +_ROLES = frozenset(StepRole) +_CAUSES = frozenset(FailureCause) +# Most severe first: a run with no grade is worse than one missing its agent's turn, which is worse than one missing +# evidence. +_SEVERITY = (StepRole.GRADER, StepRole.AGENT, StepRole.EVIDENCE, StepRole.OTHER) + + +@dataclass(frozen=True) +class RunQuality: + """How much of a completed run is usable. ``reasons`` holds the roles of the steps whose failure the run tolerated + and nothing has repaired, most severe first; a run with none is clean.""" + + reasons: tuple[StepRole, ...] = () + # Derived, never passed, so it cannot disagree with ``reasons``; a field so ``asdict`` matches the stored form. + degraded: bool = field(init=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "degraded", bool(self.reasons)) + + @classmethod + def from_dict(cls, data: Mapping[str, Any] | None) -> RunQuality: + """``quality`` as stored. A run completed before ``quality`` existed has none and reads as clean.""" + roles = {StepRole(r) for r in (data or {}).get("reasons", ()) if r in _ROLES} + return cls(tuple(r for r in _SEVERITY if r in roles)) + + def to_dict(self) -> dict[str, Any]: + return {"degraded": self.degraded, "reasons": [r.value for r in self.reasons]} + + +def run_quality(failed_steps: Iterable[FailedStep]) -> RunQuality: + """A fatal failure fails the run instead, a retried or resolved one lost nothing, and a failing verdict is a grade + the run has, not one it lost.""" + open_roles = { + f.step_role for f in failed_steps + if not f.is_fatal and not f.retried and f.resolved_by is None and f.cause != FailureCause.VERDICT + } + return RunQuality(tuple(r for r in _SEVERITY if r in open_roles)) diff --git a/src/agent_env/task/store.py b/src/agent_env/task/store.py index 72a1b1fbb..17acc0140 100644 --- a/src/agent_env/task/store.py +++ b/src/agent_env/task/store.py @@ -28,6 +28,7 @@ commit_ordered, replay_context, ) +from agent_env.task.outcome import FailedStep, RunQuality, run_quality from agent_env.task_step.context import TaskStepContext if TYPE_CHECKING: @@ -184,6 +185,8 @@ class TaskInstance: # carries the epoch it was dispatched under and no-ops when older than the stored one, # fencing a writer from a rolled-back attempt. Missing (pre-field docs) reads as 0. attempt_epoch: int = 0 + # Stamped with ``status="completed"``; clean until then, and on a run completed before the field existed. + quality: RunQuality = field(default_factory=RunQuality) @classmethod def from_dict(cls, data: dict) -> TaskInstance: @@ -202,6 +205,7 @@ def from_dict(cls, data: dict) -> TaskInstance: completed_steps=[TaskStepResult.from_dict(e) for e in data.get("completed_steps", [])], step_attempt_failures=data.get("step_attempt_failures", []), attempt_epoch=data.get("attempt_epoch", 0), + quality=RunQuality.from_dict(data.get("quality")), ) @@ -717,6 +721,13 @@ def _mutate(doc: dict) -> UpdateSpec | None: if len(completed) >= total_steps and doc.get("status") not in ("failed", "cancelled"): set_fields["status"] = "completed" set_fields["completed_at_utc"] = completed_at_utc + # In the same write as the status, so no reader sees a completed run without its quality. + after = {"context": copy.deepcopy(doc.get("context"))} + _apply_forward(after, ops, move_to_end=rerecorded) + failed_steps = _get_path(after, "context.metadata.failed_steps") or [] + set_fields["quality"] = run_quality( + FailedStep.from_dict(f) for f in failed_steps if isinstance(f, dict) + ).to_dict() return UpdateSpec(set=set_fields, unset=set(ops.unsets)) result = compare_and_swap( diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index d356c1bdc..94379e45c 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -24,6 +24,7 @@ from agent_env.store.document_store import UnknownDocumentTypeError from agent_env.store.routing import run_scope from agent_env.task_step.preflight import pinned +from agent_env.task.outcome import FailedStep from agent_env.task.teardown import env_record_ids from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops @@ -130,12 +131,12 @@ def _sub(value): return {k: _sub(v) for k, v in value.items()} return value - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields out: list["TaskStep"] = [] for step in steps: data = _sub(step.to_dict()) - seeded = attach_retry_config(type(step).from_dict(data), data) + seeded = attach_base_fields(type(step).from_dict(data), data) if (keep := getattr(seeded, "seeded_from", None)) is not None: # a step that keeps something as written keep(step) out.append(seeded) @@ -463,9 +464,9 @@ def from_dict(cls, data: dict) -> "Task": f"{_registration.failure_note(_registration.TASK_STEPS, step_type)}", *problems]), unmet, ) - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields - steps.append(attach_retry_config(step_cls.from_dict(step_dict), step_dict)) + steps.append(attach_base_fields(step_cls.from_dict(step_dict), step_dict)) return cls( id=data["id"], version=data.get("version"), @@ -684,12 +685,9 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None: except BaseException as exc: duration = time.monotonic() - started_mono fatal = getattr(step, "fail_task_on_error", True) - context.metadata.setdefault("failed_steps", []).append({ - "step_id": step.id, "step_type": step.type, - "error": str(exc), "error_type": type(exc).__name__, - "started_at_utc": started_utc, "duration_seconds": duration, - "is_fatal": fatal, - }) + context.metadata.setdefault("failed_steps", []).append(FailedStep.from_exception( + step, exc, started_at_utc=started_utc, duration_seconds=duration, is_fatal=fatal, + ).to_dict()) if not fatal: await _heartbeat_step(step, pre_context, _store.TaskStepStatus.FAILURE, dispatch_epoch) raise diff --git a/src/agent_env/task_step/store.py b/src/agent_env/task_step/store.py index cf40652a4..db775b6fb 100644 --- a/src/agent_env/task_step/store.py +++ b/src/agent_env/task_step/store.py @@ -96,9 +96,9 @@ def _deserialize(self, doc: dict) -> TaskStep: f"Unknown task step type: {task_step_type}" f"{_registration.failure_note(_registration.TASK_STEPS, task_step_type)}" ) - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields - return attach_retry_config(cls.from_dict(doc), doc) + return attach_base_fields(cls.from_dict(doc), doc) def execute_query(self, query: TaskStepQuery) -> list[TaskStep]: filt, sort = to_document_query(query) diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index c26c6dc6c..1540d5ee4 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass +from enum import StrEnum from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agent_env.task_step.context import TaskStepContext @@ -40,13 +41,23 @@ def dependencies(raw: Any) -> list[TaskStepDependency] | None: return [dep if isinstance(dep, TaskStepDependency) else TaskStepDependency.from_dict(dep) for dep in raw] -def attach_retry_config(step: "TaskStep", data: dict[str, Any]) -> "TaskStep": - """Hydrate ``retry_config`` after ``step_cls.from_dict`` at the generic - parse sites, so the field survives round-trips on step types whose - ``__init__`` doesn't accept it. No-op if already set or absent.""" +class StepRole(StrEnum): + """What a step contributes to a run, so a tolerated failure says what the run lost.""" + + GRADER = "grader" + EVIDENCE = "evidence" + AGENT = "agent" + OTHER = "other" + + +def attach_base_fields(step: "TaskStep", data: dict[str, Any]) -> "TaskStep": + """Hydrate the base fields most step ``__init__``s don't accept (``retry_config``, ``step_role``) after + ``step_cls.from_dict`` at the generic parse sites, so they survive round-trips on every step type.""" raw = data.get("retry_config") if raw is not None and getattr(step, "retry_config", None) is None: step.retry_config = RetryConfig.from_dict(raw) + if data.get("step_role") is not None: + step.step_role = StepRole(data["step_role"]) return step @@ -97,6 +108,9 @@ class TaskStep(ABC): # Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A # subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs. field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER} + # A step type declares its role; one step overrides it in config, e.g. a run_code step that grades. Not + # ``role``: deploy_agent and modify_env_tool_access already use that name for their own config. + step_role: StepRole = StepRole.OTHER def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -163,6 +177,8 @@ def to_dict(self) -> dict[str, Any]: base["depends_on"] = [dataclasses.asdict(d) for d in self.depends_on] if self.retry_config is not None: base["retry_config"] = dataclasses.asdict(self.retry_config) + if self.step_role != type(self).step_role: + base["step_role"] = self.step_role.value return base @classmethod @@ -188,13 +204,11 @@ def put(cls, **kwargs: Any) -> Self: from .store import get_task_step_store kwargs.setdefault("version", None) - # Most step __init__s don't accept retry_config (it lives on the base but - # subclasses don't forward it); construct without it, then hydrate so a - # retry_config on any step type round-trips through the step store. - retry_config_raw = kwargs.pop("retry_config", None) + # Most step __init__s don't accept the base-only fields; construct without + # them, then hydrate so they round-trip through the step store on any type. + base_fields = {k: kwargs.pop(k) for k in ("retry_config", "step_role") if k in kwargs} instance = cls(**kwargs) - if retry_config_raw is not None: - attach_retry_config(instance, {"retry_config": retry_config_raw}) + attach_base_fields(instance, base_fields) return get_task_step_store().put_document(instance) @classmethod diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py index a0d3e4a74..9c69f346e 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -17,6 +17,7 @@ class VerifyA2AAgentCardStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_card" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py index 5ecd0429d..a74e1e308 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py @@ -10,13 +10,14 @@ from agent_env.a2a_agent import A2AAgent from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2AAgentConfigIdentityStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_config_identity" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py index 7842e83d4..6e9989d0a 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py @@ -12,7 +12,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -25,6 +25,7 @@ class VerifyA2AAgentMCPStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_mcp" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py index b3aa1d457..a6ef9eca6 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py @@ -17,7 +17,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.entity.refs import EntityRef from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep @@ -32,6 +32,7 @@ class VerifyA2ABuiltinToolsStep(TaskStep): type: ClassVar[str] = "verify_a2a_builtin_tools" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py index 7615fe69d..0426c0c24 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py @@ -7,13 +7,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyCoreA2AProtocolStep(TaskStep): type: ClassVar[str] = "verify_a2a_core_protocol" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py index 71c04bd03..3f71d735f 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py @@ -14,13 +14,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2AInstallStep(TaskStep): type: ClassVar[str] = "verify_a2a_install" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py index 4024a9bbb..497b7bca4 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py @@ -31,7 +31,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -177,6 +177,7 @@ class VerifyA2AModalitiesStep(TaskStep): """ type: ClassVar[str] = "verify_a2a_modalities" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py index bba35f04b..849317c37 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py @@ -9,7 +9,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -22,6 +22,7 @@ class VerifyA2APeerAgentsStep(TaskStep): to use peer_send_message to relay an `expected_token` from the peer.""" type: ClassVar[str] = "verify_a2a_peer_agents" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int], peer_prompt_id: str, expected_token: str, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py index 4096ac42d..05de33aa5 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py @@ -28,13 +28,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ARoleStep(TaskStep): type: ClassVar[str] = "verify_a2a_role" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py index 4b800fdd0..cd1d896f7 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py @@ -19,13 +19,14 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ASkillConfigStep(TaskStep): type: ClassVar[str] = "verify_a2a_skill_config" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py index 434f60b47..a2a43cc1c 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -28,6 +28,7 @@ class VerifyA2ASnapshotStep(TaskStep): """ type: ClassVar[str] = "verify_a2a_snapshot" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py index 70daf95b8..06ec35b2a 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py @@ -34,13 +34,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ASystemPromptStep(TaskStep): type: ClassVar[str] = "verify_a2a_system_prompt" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py index d1f0dea80..755cbb0ce 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py @@ -15,13 +15,14 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.task_step.snapshot_utils.agent_state_capture import trajectory_object_url from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ATrajectoryStep(TaskStep): type: ClassVar[str] = "verify_a2a_trajectory" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/collect_artifacts.py b/src/agent_env/task_step/task_steps/collect_artifacts.py index b021ba003..9492edee0 100644 --- a/src/agent_env/task_step/task_steps/collect_artifacts.py +++ b/src/agent_env/task_step/task_steps/collect_artifacts.py @@ -86,7 +86,7 @@ from agent_env.store.ids import derive_path_id, is_local_id, validate_local_id from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container from agent_env.task_step.thread_work import finish_on_thread @@ -227,6 +227,7 @@ def _upload_file(store, file_path: str, artifact_id: str, version: int, object_n class CollectArtifactsTaskStep(TaskStep): type: ClassVar[str] = "collect_artifacts" + step_role = StepRole.EVIDENCE entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py index 63bd1e7c2..fc32eace9 100644 --- a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py +++ b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -20,6 +20,7 @@ class VerifyEnvironmentCardStep(TaskStep): type: ClassVar[str] = "verify_env_card" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py index 20a92ff0b..c6358ac2b 100644 --- a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py +++ b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py @@ -11,7 +11,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -27,6 +27,7 @@ class VerifyCoreEnvironmentProtocolStep(TaskStep): type: ClassVar[str] = "verify_env_core_protocol" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py b/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py index b0a015aef..6264b0c9d 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py @@ -15,7 +15,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -51,6 +51,7 @@ def aggregate_gate(verifications: dict, required_gates) -> dict: class ValidationGateAggregatorStep(TaskStep): type: ClassVar[str] = "validation_gate_aggregator" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py index 72ba54990..1c1f8693a 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py @@ -8,13 +8,14 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyMCPEnvAssessmentStep(TaskStep): type: ClassVar[str] = "verify_mcp_env_assessment" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py index 1a7df4f06..8366ac898 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py @@ -11,7 +11,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -36,6 +36,7 @@ def resolve_defs_ref(param_schema: dict, schema: dict) -> dict: class VerifyMCPToolSchemaTaskStep(TaskStep): type: ClassVar[str] = "verify_mcp_tool_schema" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py index 9787c3ae5..f9c69919d 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py @@ -29,7 +29,7 @@ from agent_env.env import legacy_protocol from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import ( resolve_defs_ref, ) @@ -378,6 +378,7 @@ def _summarize(findings: list[dict], limit: int = 10) -> str: class VerifySpecConformanceTaskStep(TaskStep): type: ClassVar[str] = "verify_spec_conformance" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py b/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py index 541e31a40..2e5035dae 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py @@ -14,13 +14,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class CombineUniverseVerdictsStep(TaskStep): type: ClassVar[str] = "combine_universe_verdicts" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("universe_artifact_id", version_field="universe_artifact_version"), diff --git a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py index 585c2a945..7f3e1cbcb 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py @@ -14,7 +14,7 @@ from agent_env.store.ids import derive_id, fs_safe, is_local_id from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from .universe_comparison import classify_issues, compare_dicts, normalize @@ -26,6 +26,7 @@ class VerifyUniverseLoadExportRoundtripStep(TaskStep): type: ClassVar[str] = "verify_universe_load_export_roundtrip" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("universe_artifact_id", version_field="universe_artifact_version", artifact_type="environment_universe"), diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index e94f6e1b3..4bf92cd28 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -38,7 +38,7 @@ from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope -from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.notification_delivery import delivered_parts, labels_zone, next_message, take_due from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions @@ -155,6 +155,7 @@ def _file_uris(parts: list[dict]) -> list[str]: class PromptAgentTaskStep(TaskStep): type: ClassVar[str] = "prompt_agent" + step_role = StepRole.AGENT DEFAULT_USER_AGENT_NAME: ClassVar[str] = "human_agent" entity_refs = ( EntityRef.env("snapshot_config.env_id"), diff --git a/src/agent_env/task_step/task_steps/snapshot_agent_state.py b/src/agent_env/task_step/task_steps/snapshot_agent_state.py index b491029d6..1326c9ac8 100644 --- a/src/agent_env/task_step/task_steps/snapshot_agent_state.py +++ b/src/agent_env/task_step/task_steps/snapshot_agent_state.py @@ -10,7 +10,7 @@ from agent_env.store.ids import derive_id, is_local_id, validate_local_id from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef, RefRole -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.snapshot_utils import agent_state_capture from agent_env.task_step.snapshot_utils.snapshot_series import rollout_base @@ -49,6 +49,7 @@ class SnapshotAgentStateTaskStep(TaskStep): """ type: ClassVar[str] = "snapshot_agent_state" + step_role = StepRole.EVIDENCE entity_refs = ( EntityRef.artifact("artifact_id", role=RefRole.OUTPUT, artifact_type="file_artifact_universe"), EntityRef.env("env_id"), diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index d42d70943..64284ea91 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -35,7 +35,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef, RefRole from agent_env.entity.scopes import FieldScope -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency if TYPE_CHECKING: from agent_env.env.legacy_protocol import BackingServer @@ -184,6 +184,7 @@ class SnapshotEnvTaskStep(TaskStep): """ type: ClassVar[str] = "snapshot_env" + step_role = StepRole.EVIDENCE entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("snapshot_id", role=RefRole.OUTPUT, artifact_type="environment_universe"), diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 7d868f330..0c2c78672 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -22,7 +22,7 @@ from typing import ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score logger = logging.getLogger(__name__) @@ -35,6 +35,7 @@ class AgentPromptResponseVerifierTaskStep(TaskStep): type: ClassVar[str] = "agent_prompt_response_verifier" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py index 8d663a82d..ccb4541ab 100644 --- a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +++ b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py @@ -16,7 +16,7 @@ from typing import ClassVar, Optional from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score logger = logging.getLogger(__name__) @@ -24,6 +24,7 @@ class AggregateVerifiersTaskStep(TaskStep): type: ClassVar[str] = "aggregate_verifiers" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py index 5e45774af..346486531 100644 --- a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py @@ -15,7 +15,7 @@ from agent_env.config import get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import load_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -32,6 +32,7 @@ class EnvOutcomeVerifierTaskStep(TaskStep): control token, so its calls reach the env's framework-only routes as written.""" type: ClassVar[str] = "env_outcome_verifier" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("file_artifact_id", version_field="file_artifact_version", artifact_type="file"), @@ -76,8 +77,6 @@ def put(cls, *, verify_script_file_path: Optional[str] = None, **kwargs) -> "Env """Store the step. With ``verify_script_file_path`` the script is uploaded as ``-verifier-script`` first; otherwise the kwargs name an existing ``file_artifact_id`` (the shape ``agent-env task create`` builds from JSON).""" - from agent_env.task_step.store import get_task_step_store - if verify_script_file_path is not None: from agent_env.artifact import FileArtifact @@ -87,8 +86,7 @@ def put(cls, *, verify_script_file_path: Optional[str] = None, **kwargs) -> "Env file_path=verify_script_file_path, ) kwargs.update(file_artifact_id=artifact.id, file_artifact_version=artifact.version) - kwargs.setdefault("version", None) - return get_task_step_store().put_document(cls(**kwargs)) + return super().put(**kwargs) @classmethod def from_dict(cls, data: dict) -> EnvOutcomeVerifierTaskStep: diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 71f7bf944..fd0e8c487 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -47,7 +47,7 @@ from agent_env.entity.scopes import FieldScope from agent_env.store.base import NotFoundError from agent_env.store.ids import is_local_id -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import Run, load_run_context, render_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( @@ -449,6 +449,7 @@ def _as_of_turn(prompt_response: PromptResponse, turn: Optional[int]) -> PromptR class RubricsVerifierTaskStep(TaskStep): type: ClassVar[str] = "rubrics_verifier" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("judge_a2a_agent_id"),) context_paths = ("context_facts_for_judge{}",) field_scopes = {**TaskStep.field_scopes, **dict.fromkeys( diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index ab8cb2e96..1b15b316b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -51,7 +51,7 @@ ) from agent_env.store.ids import derive_id, is_local_id, key_segment, validate_local_id from agent_env.task_step.context import StepContextError, TaskStepContext, dual_keyed -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.run_code import _DriverTarget logger = logging.getLogger(__name__) @@ -87,6 +87,7 @@ class RunContainerUnitTestsVerifierTaskStep(TaskStep): type: ClassVar[str] = "run_container_unit_tests_verifier" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index 5d5776b34..b8656d63b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -36,7 +36,7 @@ get_sandbox_provider, ) from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -58,6 +58,7 @@ def _outcome(passed: bool, justification: str) -> dict: class VerifySandboxTaskStep(TaskStep): type: ClassVar[str] = "verify_sandbox" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/tst/unit/task/episode/spec_test.py b/tst/unit/task/episode/spec_test.py index 62641e54c..b23aff047 100644 --- a/tst/unit/task/episode/spec_test.py +++ b/tst/unit/task/episode/spec_test.py @@ -115,6 +115,17 @@ def test_the_outside_agents_deploy_runs_with_a_proxy_unseeded_and_round_trips(): assert _EpisodeAgentStep.from_dict(agent.to_dict()).to_dict() == agent.to_dict() +def test_the_outside_agents_wrapped_deploy_keeps_its_configured_role_and_retry_through_a_round_trip(): + task = _portsim() + agent = EpisodeSpec.from_task(task, seed=SEED)._copy.steps[2] + data = agent.to_dict() + data["step"] = {**data["step"], "step_role": "grader", "retry_config": {"max_retries": 2, "retry_from_step_id": data["step"]["id"]}} + + wrapped = _EpisodeAgentStep.from_dict(data).step + + assert (wrapped.step_role, wrapped.retry_config.max_retries) == ("grader", 2) + + def test_without_a_turn_timeout_a_turns_deadline_is_its_prompts_own(): task = _portsim(play={**PLAY, "timeout_seconds": 30}) @@ -131,6 +142,12 @@ def test_a_turn_timeout_replaces_every_outside_prompts_own_in_whole_seconds(turn assert copy.steps[3].to_dict() == {**task.steps[3].to_dict(), "timeout_seconds": deadline} +def test_a_turn_timeout_keeps_a_prompts_configured_role(): + task = _portsim(play={**PLAY, "step_role": "grader"}) + + assert EpisodeSpec.from_task(task, turn_timeout=90)._copy.steps[3].step_role == "grader" + + def test_an_inside_agent_keeps_its_own_steps(): copy = EpisodeSpec.from_task(_portsim(BOB, ASK_BOB), trainer_agents=["bob"])._copy diff --git a/tst/unit/task/outcome_test.py b/tst/unit/task/outcome_test.py new file mode 100644 index 000000000..367d44850 --- /dev/null +++ b/tst/unit/task/outcome_test.py @@ -0,0 +1,172 @@ +"""A failed step's entry in ``context.metadata.failed_steps`` says what role the step played and why it failed.""" + +from __future__ import annotations + +import inspect + +import httpx +import pytest + +from agent_env.config import set_document_store +from agent_env.store import LocalSqliteDocumentStore +from agent_env.task.outcome import FailureCause, StepRole, classify_failure +from agent_env.task.store import TASK_INSTANCES_COLLECTION, TaskInstanceStore, set_task_instance_store +from agent_env.task.task import Task +from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.run_code import RunCodeTaskStep +from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep +from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep, RubricsVerifierTaskStep + +BUDGET_400 = ( + "Judge A2A task failed (state=failed): API Error: 400 Budget exceeded for team-a: " + "spent $100.54 of $100.00 per 15d" +) + + +def _chained(outer: BaseException, inner: BaseException, *, explicit: bool = True) -> BaseException: + try: + try: + raise inner + except BaseException as e: + if explicit: + raise outer from e + raise outer + except BaseException as caught: + return caught + + +@pytest.mark.parametrize(("exc", "cause"), [ + (RuntimeError(BUDGET_400), FailureCause.KEY_BUDGET), + (Exception("litellm.BudgetExceededError: Budget has been exceeded! Current cost: 50.54, Max budget: 50.0"), + FailureCause.KEY_BUDGET), + (_chained(RuntimeError("LLM judge failed after 3 attempts"), RuntimeError(BUDGET_400)), FailureCause.KEY_BUDGET), + (_chained(TimeoutError("judge timed out"), RuntimeError(BUDGET_400)), FailureCause.KEY_BUDGET), + (RuntimeError("Error code: 401 - {'error': {'message': 'Authentication Error, Invalid proxy server token passed. " + "valid_token=None.', 'type': 'auth_error', 'code': '401'}}"), FailureCause.KEY_AUTH), + (RuntimeError("Authentication Error: token_not_found_in_db"), FailureCause.KEY_AUTH), + (RuntimeError("litellm.AuthenticationError: OpenAIException - Incorrect API key provided: sk-...abcd"), + FailureCause.KEY_AUTH), + (RuntimeError("Server disconnected without sending a response."), FailureCause.TRANSIENT), + (TimeoutError(), FailureCause.TRANSIENT), + (ConnectionResetError(104, "Connection reset by peer"), FailureCause.TRANSIENT), + (RuntimeError("API Error: 503 Service Unavailable"), FailureCause.TRANSIENT), + (RuntimeError("Server error '502 Bad Gateway' for url 'https://judge.example/v1/chat'"), FailureCause.TRANSIENT), + (RuntimeError("sandbox command exited with code 137 (OOMKilled)"), FailureCause.TRANSIENT), + (RuntimeError("Error code: 429 - Too Many Requests"), FailureCause.TRANSIENT), + (httpx.ConnectError("All connection attempts failed"), FailureCause.TRANSIENT), + (RuntimeError('[API Error: {"error":{"message":"LiteLLM Rate Limit Handler for rate limit type = key.",' + '"type":"None","param":"None","code":"429"}}]'), FailureCause.TRANSIENT), + (RuntimeError(r'''[API Error: {"error":{"message":"b'{\\n \"error\": {\\n \"code\": 503,\\n \"status\": ''' + r'''\"UNAVAILABLE\"\\n }\\n}\\n'","type":"None","param":"None","code":"503"}}]'''), + FailureCause.TRANSIENT), + (RuntimeError('[API Error: {"detail":"gemini upstream request failed: All connection attempts failed"}]'), + FailureCause.TRANSIENT), + (RuntimeError('[API Error: {"error":{"message":"Budget has been exceeded! Key=pub-xxxx Spend=1 Budget=1",' + '"type":"budget_exceeded","param":null,"code":"400"}}]'), FailureCause.KEY_BUDGET), + (_chained(RuntimeError("agent config not sent"), httpx.ConnectError("[Errno -3] Temporary failure in name resolution")), + FailureCause.TRANSIENT), + (RuntimeError("Authentication Error - Expired Key. Key Expiry time 2026-11-01 and current time 2026-11-17"), + FailureCause.KEY_AUTH), + (RuntimeError("claude CLI exited with code 1: API Error: 400 Budget exceeded for service_identity:litellm-x: " + "spent $50,008.81 of $50,000.00 per 15d (key: x)"), FailureCause.KEY_BUDGET), + (RuntimeError("API Error: Request rejected (429) ยท Rate limited. Retry in 1s."), + FailureCause.TRANSIENT), + (RuntimeError("codex CLI stalled (attempt 1: unexpected status 408 Request Timeout: request body arrived too slowly)"), + FailureCause.TRANSIENT), + (VerifierNotPassingError("verifier 'grade' scored 0.5: rows not met: done"), FailureCause.VERDICT), + # Text from an env, a tool, a script or a model mentions codes and words the proxy also uses; none of it is ours. + (RuntimeError("tool get_account returned HTTP 401 Unauthorized from env MCP server"), FailureCause.UNKNOWN), + (RuntimeError("run_code 'reload': exited 1: urllib.error.HTTPError: HTTP Error 401: Unauthorized"), + FailureCause.UNKNOWN), + (ValueError("rubric row 401 has no criteria"), FailureCause.UNKNOWN), + (ValueError("invalid token in template: {{seed}"), FailureCause.UNKNOWN), + (AssertionError("expected status 500 to not be returned by create_order"), FailureCause.UNKNOWN), + (RuntimeError("judge said: user exceeded the budget for travel"), FailureCause.UNKNOWN), + (RuntimeError("review 'review' timed out after 10800s awaiting a decision"), FailureCause.UNKNOWN), + # An error raised while handling another, without ``from``, is not caused by it. + (_chained(ValueError("rubric item 3 has no criteria"), TimeoutError(), explicit=False), FailureCause.UNKNOWN), + (_chained(RuntimeError("LLM judge failed after 3 attempts"), RuntimeError(BUDGET_400), explicit=False), + FailureCause.UNKNOWN), + (ValueError("rubric item 3 has no criteria"), FailureCause.UNKNOWN), + (ValueError("judge returned invalid JSON: Expecting value: line 1 column 1"), FailureCause.UNKNOWN), +]) +def test_classify_failure(exc, cause): + assert classify_failure(exc) == cause + + +def test_step_types_declare_their_role(): + assert RubricsVerifierTaskStep.step_role == StepRole.GRADER + assert AgentPromptResponseVerifierTaskStep.step_role == StepRole.GRADER + assert SnapshotEnvTaskStep.step_role == StepRole.EVIDENCE + assert SnapshotAgentStateTaskStep.step_role == StepRole.EVIDENCE + assert CollectArtifactsTaskStep.step_role == StepRole.EVIDENCE + assert PromptAgentTaskStep.step_role == StepRole.AGENT + assert RunCodeTaskStep.step_role == StepRole.OTHER + + +def test_every_step_type_that_writes_verifications_is_a_grader(): + writers = { + name for name, cls in get_task_step_registry().items() + if '"verifications"' in inspect.getsource(inspect.getmodule(cls)) + } + assert writers, "the scan found no verifier, so it no longer reads the steps' source" + assert {name for name in writers if get_task_step_registry()[name].step_role != StepRole.GRADER} == set() + + +def _run_code(**extra) -> dict: + return {"id": "verify", "type": "run_code", "script_artifact_id": "s", "entrypoint": "main.py", **extra} + + +def test_a_step_can_declare_its_role_in_config(): + task = Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code(step_role="grader")]}) + assert task.steps[0].step_role == StepRole.GRADER + assert task.to_dict()["steps"][0]["step_role"] == "grader" + + +def test_a_step_without_a_role_override_serializes_as_before(): + task = Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code()]}) + assert "step_role" not in task.to_dict()["steps"][0] + + +def test_an_unknown_role_is_refused_when_the_task_loads(): + with pytest.raises(ValueError, match="judge"): + Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code(step_role="judge")]}) + + +class _OutOfBudgetVerifier(AgentPromptResponseVerifierTaskStep): + async def execute(self, context): + raise RuntimeError(BUDGET_400) + + +@pytest.fixture +def store(tmp_path): + doc = LocalSqliteDocumentStore(str(tmp_path / "instances.db")) + doc.ensure_index(TASK_INSTANCES_COLLECTION, ["instance_id"], unique=True) + set_document_store(doc) + s = TaskInstanceStore() + set_task_instance_store(s) + try: + yield s + finally: + set_task_instance_store(None) + + +@pytest.mark.asyncio +async def test_a_tolerated_grader_failure_records_its_role_and_cause(store): + grader = _OutOfBudgetVerifier("grade", None, prompt_id="p", fail_task_on_error=False) + ctx = TaskStepContext() + result = await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + [entry] = result.metadata["failed_steps"] + assert entry == { + "step_id": "grade", "step_type": "agent_prompt_response_verifier", + "error": BUDGET_400, "error_type": "RuntimeError", + "started_at_utc": entry["started_at_utc"], "duration_seconds": entry["duration_seconds"], + "is_fatal": False, "step_role": "grader", "cause": "key_budget", + } + assert store.get(ctx.instance_id).status == "completed" + assert store.get(ctx.instance_id).context["metadata"]["failed_steps"] == [entry] diff --git a/tst/unit/task/run_quality_test.py b/tst/unit/task/run_quality_test.py new file mode 100644 index 000000000..5f8881a75 --- /dev/null +++ b/tst/unit/task/run_quality_test.py @@ -0,0 +1,106 @@ +"""A completed run carries ``quality``: degraded when it tolerated a step failure that nothing repaired.""" + +from __future__ import annotations + +import pytest + +from agent_env.config import set_document_store +from agent_env.store import LocalSqliteDocumentStore +from agent_env.store.document_store import Filter +from agent_env.task.outcome import FailedStep, FailureCause, RunQuality, StepRole, run_quality +from agent_env.task.store import TASK_INSTANCES_COLLECTION, TaskInstance, TaskInstanceStore, set_task_instance_store +from agent_env.task.task import Task +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import RetryConfig +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep + + +def _failed(role: StepRole, **fields) -> FailedStep: + return FailedStep(**{ + "step_id": role.value, "step_type": "t", "error": "boom", "error_type": "RuntimeError", + "started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False, + "step_role": role, **fields, + }) + + +def test_a_run_with_no_failures_is_clean(): + assert run_quality([]) == RunQuality() + assert RunQuality().to_dict() == {"degraded": False, "reasons": []} + + +def test_open_tolerated_failures_degrade_the_run_most_severe_first(): + quality = run_quality([_failed(StepRole.EVIDENCE), _failed(StepRole.GRADER), _failed(StepRole.EVIDENCE)]) + assert quality.to_dict() == {"degraded": True, "reasons": ["grader", "evidence"]} + + +@pytest.mark.parametrize("fields", [{"is_fatal": True}, {"retried": True}, {"resolved_by": "regrade:2026-01-02"}]) +def test_a_fatal_retried_or_resolved_failure_does_not_degrade_the_run(fields): + assert not run_quality([_failed(StepRole.GRADER, **fields)]).degraded + + +def test_a_tolerated_failing_verdict_does_not_degrade_the_run(): + assert not run_quality([_failed(StepRole.GRADER, cause=FailureCause.VERDICT)]).degraded + + +def test_an_entry_written_before_roles_existed_degrades_as_other(): + legacy = {"step_id": "s", "step_type": "run_code", "error": "boom", "error_type": "RuntimeError", + "started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False} + assert run_quality([FailedStep.from_dict(legacy)]).to_dict() == {"degraded": True, "reasons": ["other"]} + + +def test_a_run_completed_before_quality_existed_reads_as_clean(): + doc = {"instance_id": "i", "task_id": "t", "task_version": 1, "status": "completed", + "current_step": 1, "total_steps": 1} + assert TaskInstance.from_dict(doc).quality == RunQuality() + + +class _FailingVerifier(AgentPromptResponseVerifierTaskStep): + def __init__(self, *args, fail_times: int, **kwargs): + super().__init__(*args, **kwargs) + self.fail_times = fail_times + + async def execute(self, context): + if self.fail_times: + self.fail_times -= 1 + raise RuntimeError("Judge A2A task failed: API Error: 503 Service Unavailable") + return context + + +@pytest.fixture +def store(tmp_path): + doc = LocalSqliteDocumentStore(str(tmp_path / "instances.db")) + doc.ensure_index(TASK_INSTANCES_COLLECTION, ["instance_id"], unique=True) + set_document_store(doc) + s = TaskInstanceStore() + set_task_instance_store(s) + try: + yield s + finally: + set_task_instance_store(None) + + +def _stored_quality(store, instance_id): + return store._doc_store.find_one(TASK_INSTANCES_COLLECTION, Filter.of(instance_id=instance_id))["quality"] + + +@pytest.mark.asyncio +async def test_completing_with_a_tolerated_grader_failure_stamps_a_degraded_quality(store): + grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1) + ctx = TaskStepContext() + await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + assert _stored_quality(store, ctx.instance_id) == {"degraded": True, "reasons": ["grader"]} + instance = store.get(ctx.instance_id) + assert instance.status == "completed" + assert instance.quality == RunQuality((StepRole.GRADER,)) + + +@pytest.mark.asyncio +async def test_completing_after_a_successful_retry_stamps_a_clean_quality(store): + grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1) + grader.retry_config = RetryConfig(retry_from_step_id="grade", max_retries=1) + ctx = TaskStepContext() + result = await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + assert [f.get("retried") for f in result.metadata["failed_steps"]] == [True] + assert _stored_quality(store, ctx.instance_id) == {"degraded": False, "reasons": []} diff --git a/tst/unit/task_step/retry_test.py b/tst/unit/task_step/retry_test.py index 15be8c585..62e33bcbd 100644 --- a/tst/unit/task_step/retry_test.py +++ b/tst/unit/task_step/retry_test.py @@ -812,7 +812,7 @@ def test_retry_config_from_dict_validates(): def test_retry_config_round_trips_on_non_prompt_step(): - # deploy_env's __init__ does not accept retry_config; attach_retry_config must + # deploy_env's __init__ does not accept retry_config; attach_base_fields must # still hydrate it on the generic parse path, and it must round-trip out. doc = { "id": "t", "type": "task", "version": 1, diff --git a/tst/unit/task_step/verifiers/test_env_outcome_verifier_step.py b/tst/unit/task_step/verifiers/test_env_outcome_verifier_step.py index c9fb2ac0a..5cc8c45f0 100644 --- a/tst/unit/task_step/verifiers/test_env_outcome_verifier_step.py +++ b/tst/unit/task_step/verifiers/test_env_outcome_verifier_step.py @@ -41,6 +41,19 @@ def put_document(self, instance): assert step.score_aggregator is ScoreAggregator.ALL_PASS and step.version is None +def test_put_keeps_a_configured_role_and_retry(monkeypatch): + from agent_env.task_step import store as step_store + + class _Store: + def put_document(self, instance): + return instance + monkeypatch.setattr(step_store, "get_task_step_store", lambda: _Store()) + step = EnvOutcomeVerifierTaskStep.put(id="g", env_id="e", file_artifact_id="f", step_role="other", + retry_config={"retry_from_step_id": "g"}) + + assert step.step_role == "other" and step.retry_config.retry_from_step_id == "g" + + def test_put_with_a_script_path_uploads_then_stores(monkeypatch, tmp_path): from agent_env import artifact as artifact_mod from agent_env.task_step import store as step_store