Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
057bf26
feat(verifiers): the base fails a grade its verifier can't trust, and…
earakely-scale Nov 18, 2026
b3eda3c
test(verifiers): record the legacy entries run_container, verify_sand…
earakely-scale Nov 18, 2026
4dd27d8
test(verifiers): the rubrics family's legacy entries, recorded before…
earakely-scale Nov 18, 2026
84a0b5b
feat(verifiers): run_container, verify_sandbox, env_outcome and aggre…
earakely-scale Nov 18, 2026
f57fb71
feat(verifiers): rubrics_verifier, always_on_agent_verifier and user_…
earakely-scale Nov 18, 2026
14e4bdd
fix(verifiers): a run naming the step's own judge model overrides not…
earakely-scale Nov 18, 2026
4cb7caf
Merge branch 'edgararakelyan/g2-rubrics' into edgararakelyan/fd-5922-…
earakely-scale Nov 18, 2026
5280797
chore(verifiers): every verifier golden file under tst/data, and AGEN…
earakely-scale Nov 18, 2026
7f75904
Merge origin/main into the G2 branch
earakely-scale Nov 18, 2026
06a0e63
Merge remote-tracking branch 'origin/main' into edgararakelyan/fd-592…
earakely-scale Nov 18, 2026
e36c681
test(verifiers): a run's stored grades, a timed-out unit test run's f…
earakely-scale Nov 18, 2026
628c0dd
fix(verifiers): settings that can't be compacted are recorded, a judg…
earakely-scale Nov 18, 2026
31bcd31
test(verifiers): the reward test makes its reward's folder, as no ste…
earakely-scale Nov 18, 2026
083a466
refactor(verifiers): the rubrics judge and its settings read a run's …
earakely-scale Nov 19, 2026
d73736d
test(verifiers): one golden harness for every verifier, which re-reco…
earakely-scale Nov 19, 2026
2023092
Merge remote-tracking branch 'origin/main' into edgararakelyan/fd-592…
earakely-scale Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/agent_env/cli/task/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ def _format_verifier_output(vid: str, vdata: dict) -> list[str]:
else:
lines.append(f" {path}: {str(parsed)[:600]}")
results = vdata.get("results")
if results is not None:
if results:
lines.append(f"verification (verifier_id={vid}):")
lines.append(_format_verification_results(results))
score = vdata.get("score")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,33 +16,33 @@
from typing import ClassVar, Optional

from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency
from agent_env.task_step.task_step import TaskStepDependency
from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score
from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep

logger = logging.getLogger(__name__)


class AggregateVerifiersTaskStep(TaskStep):
class AggregateVerifiersTaskStep(VerifierTaskStep):
type: ClassVar[str] = "aggregate_verifiers"
step_role = StepRole.GRADER
entity_refs = ()
DEFAULT_SCORE_AGGREGATOR: ClassVar[ScoreAggregator] = ScoreAggregator.WEIGHTED_AVERAGE

def __init__(
self,
id: str,
version: Optional[int],
verifier_ids: Optional[list[str]] = None,
score_aggregator: Optional[ScoreAggregator] = None,
score_aggregator: Optional[ScoreAggregator | str] = None,
verifier_id: Optional[str] = None,
depends_on: Optional[list[TaskStepDependency]] = None,
fail_task_on_error: bool = True,
min_passing_score: Optional[float] = None,
):
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error)
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error,
verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator,
min_passing_score=min_passing_score)
self.verifier_ids = list(verifier_ids or [])
if isinstance(score_aggregator, str):
score_aggregator = ScoreAggregator(score_aggregator)
self.score_aggregator = score_aggregator or ScoreAggregator.WEIGHTED_AVERAGE
self.verifier_id = verifier_id or uuid.uuid4().hex

def to_dict(self) -> dict:
base = super().to_dict()
Expand All @@ -53,15 +53,9 @@ def to_dict(self) -> dict:

@classmethod
def from_dict(cls, data: dict) -> "AggregateVerifiersTaskStep":
raw_agg = data.get("score_aggregator")
return cls(
**cls._base_from_dict(data),
verifier_ids=data.get("verifier_ids"),
score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None,
verifier_id=data.get("verifier_id"),
)
return cls(**cls._base_from_dict(data), verifier_ids=data.get("verifier_ids"))

async def execute(self, context: TaskStepContext) -> TaskStepContext:
async def verify(self, context: TaskStepContext) -> Verification:
all_verifications = context.metadata.get("verifications", {})
missing = [vid for vid in self.verifier_ids if vid not in all_verifications]
if missing:
Expand All @@ -77,15 +71,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:

non_skipped = [r for r in merged_results if not r.get("skipped")]
score = aggregate_score(non_skipped, self.score_aggregator)

if "verifications" not in context.metadata:
context.metadata["verifications"] = {}
context.metadata["verifications"][self.verifier_id] = {
"score": score,
"source_verifier_ids": list(self.verifier_ids),
}
logger.info(
f"AggregateVerifiers '{self.verifier_id}': merged {len(self.verifier_ids)} verifier(s), "
f"{len(non_skipped)}/{len(merged_results)} criteria evaluated, score={score}"
)
return context
return Verification(score=score, output={"source_verifier_ids": list(self.verifier_ids)})
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
)
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import (
RUN_CONTEXT_PARTS,
RubricsVerdict,
RubricsVerifierTaskStep,
StagedTrajectory,
)
Expand Down Expand Up @@ -233,35 +234,31 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]:
f"'{prompts[0].id}', which isn't an always_on step; grade it with rubrics_verifier"]
return []

async def execute(self, context: TaskStepContext) -> TaskStepContext:
async def _judged(self, context: TaskStepContext) -> RubricsVerdict:
"""rubrics_verifier's verdict, refused for a run whose overrides cut the prompt or take the wakes from its judge;
then, for a staged run, what each judge call read of it (``judge_reads``); for a run with agent labels (the PSI
harness's), how many failed criteria the judge traced to a compaction (``context_loss``); and for every run,
the assistant's harness as the run records it (``assistant_harness``)."""
if "grade_up_to_turn" in self.step_param_overrides(context):
raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's "
"prompt, whose conversation turns are not its wakes.")
if self.use_trajectory and (context.metadata.get("user_overrides") or {}).get("use_agent_judge") is False:
raise RuntimeError(f"Verifier '{self.verifier_id}': use_trajectory needs the agent judge; this run's "
"use_agent_judge=False would grade without the wakes it was set to read.")
return await super().execute(context)

async def _grade(self, context: TaskStepContext) -> TaskStepContext:
"""rubrics_verifier's grade; then, for a staged run, what each judge call read of it (``judge_reads``); for a
run with agent labels (the PSI harness's), how many failed criteria the judge traced to a compaction (``context_loss``); and
for every run, the assistant's harness as the run records it (``assistant_harness``)."""
token = _CONTEXT_LOSS_RUN.set(False)
try:
context = await super()._grade(context)
verdict = await super()._judged(context)
asked = _CONTEXT_LOSS_RUN.get()
finally:
_CONTEXT_LOSS_RUN.reset(token)
entry = (context.metadata.get("verifications") or {}).get(self.verifier_id)
if entry is None:
return context
if "trajectory_staging" in entry:
entry["trajectory_staging"]["judge_reads"] = await self._judge_reads(entry)
output = dict(verdict.output)
if "trajectory_staging" in output:
output["trajectory_staging"]["judge_reads"] = await self._judge_reads(output)
if asked:
entry["context_loss"] = context_loss(entry.get("results") or [])
if harness := await self._assistant_harness(context, staged="trajectory_staging" in entry):
entry["assistant_harness"] = harness
return context
output["context_loss"] = context_loss(verdict.criteria)
if harness := await self._assistant_harness(context, staged="trajectory_staging" in output):
output["assistant_harness"] = harness
return dataclasses.replace(verdict, output=output)

def _spec(self) -> JudgeOutputFormatSpec:
"""rubrics_verifier's format; for a run with agent labels (``_CONTEXT_LOSS_RUN``), its rows also take ``context_loss`` and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,30 @@
import os
import tempfile
import uuid
from collections.abc import Sized
from typing import ClassVar, Optional

from agent_env.config import get_config
from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency
from agent_env.task_step.task_step import TaskStepDependency
from agent_env.task_step.task_steps.verifiers.run_context import load_run_context
from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score
from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep

__all__ = ["EnvOutcomeVerifierTaskStep", "ScoreAggregator", "aggregate_score"]


logger = logging.getLogger(__name__)


class EnvOutcomeVerifierTaskStep(TaskStep):
class EnvOutcomeVerifierTaskStep(VerifierTaskStep):
"""Runs the artifact's ``verify(mcp_url)``; a ``run`` parameter also gets the ``Run`` of ``prompt_id``, a
``trajectory`` parameter that prompt's agent trajectory, and an ``env_url`` parameter the env's address, which the
protocol client takes as it is: like every step, ``verify`` runs under ``Task.run``'s grant of each deployed env's
control token, so its calls reach the env's framework-only routes as written."""

type: ClassVar[str] = "env_outcome_verifier"
step_role = StepRole.GRADER
entity_refs = (
EntityRef.env("env_id"),
EntityRef.artifact("file_artifact_id", version_field="file_artifact_version", artifact_type="file"),
Expand All @@ -45,21 +46,20 @@ def __init__(
env_id: str,
file_artifact_id: str,
file_artifact_version: Optional[int] = None,
score_aggregator: Optional[ScoreAggregator] = None,
score_aggregator: Optional[ScoreAggregator | str] = None,
verifier_id: Optional[str] = None,
prompt_id: Optional[str] = None,
depends_on: Optional[list[TaskStepDependency]] = None,
fail_task_on_error: bool = True,
min_passing_score: Optional[float] = None,
):
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error)
self.verifier_id = verifier_id or uuid.uuid4().hex
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error,
verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator,
min_passing_score=min_passing_score)
self.prompt_id = prompt_id
self.env_id = env_id
self.file_artifact_id = file_artifact_id
self.file_artifact_version = file_artifact_version
if isinstance(score_aggregator, str):
score_aggregator = ScoreAggregator(score_aggregator)
self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS

def to_dict(self) -> dict:
base = super().to_dict()
Expand Down Expand Up @@ -90,18 +90,15 @@ def put(cls, *, verify_script_file_path: Optional[str] = None, **kwargs) -> "Env

@classmethod
def from_dict(cls, data: dict) -> EnvOutcomeVerifierTaskStep:
raw = data.get("score_aggregator")
return cls(
**cls._base_from_dict(data),
env_id=data["env_id"],
file_artifact_id=data["file_artifact_id"],
file_artifact_version=data.get("file_artifact_version"),
score_aggregator=ScoreAggregator(raw) if raw else None,
verifier_id=data.get("verifier_id"),
prompt_id=data.get("prompt_id"),
)

async def execute(self, context: TaskStepContext) -> TaskStepContext:
async def verify(self, context: TaskStepContext) -> Verification:
from agent_env.artifact import FileArtifact

deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None)
Expand Down Expand Up @@ -129,20 +126,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
if "trajectory" in parameters:
keywords["trajectory"] = await asyncio.to_thread(_trajectory, self._graded_prompt(context))
results = await module.verify(deployed.mcp_url, **keywords)

score = aggregate_score(results, self.score_aggregator)
if "verifications" not in context.metadata:
context.metadata["verifications"] = {}
context.metadata["verifications"][self.verifier_id] = {
"results": results,
"score": score,
}
logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} criteria, score={score}")
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)

return context
if not isinstance(results, Sized):
# Scores 0 with the rows as returned, and fails its step.
return Verification(criteria=results, raise_after=TypeError(
f"verify() in artifact '{self.file_artifact_id}' returned {results!r}, not a list of criterion rows"))
score = aggregate_score(results, self.score_aggregator)
logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} criteria, score={score}")
return Verification(criteria=results, score=score)

def _graded_prompt(self, context: TaskStepContext) -> Optional[PromptResponse]:
if self.prompt_id is None:
Expand Down
Loading
Loading