Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/scripts/check_plugin_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
BASES = (
"agent_env.env.env.Env",
"agent_env.task_step.task_step.TaskStep",
"agent_env.task_step.task_steps.verifiers.verifier.VerifierTaskStep",
"agent_env.artifact.artifact.Artifact",
"agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider",
"agent_env.providers.env_state.env_state_provider.EnvStateProvider",
Expand All @@ -47,6 +48,9 @@
"agent_env.runner.store.ensure_indexes", "agent_env.runner.store.insert_run", "agent_env.runner.store.get_run",
"agent_env.runner.store.list_runs", "agent_env.runner.store.count_runs", "agent_env.runner.store.active_runs",
"agent_env.runner.store.set_running", "agent_env.runner.store.mark_terminal",
# What a verifier step returns, scores its rows with and writes.
"agent_env.task_step.task_steps.verifiers.Verification", "agent_env.task_step.task_steps.verifiers.Grade", "agent_env.task_step.task_steps.verifiers.VerifierStatus",
"agent_env.task_step.task_steps.verifiers.ScoreAggregator", "agent_env.task_step.task_steps.verifiers.aggregate_score",
# What a verify script is handed of the run it grades.
"agent_env.task_step.task_steps.verifiers.Run", "agent_env.task_step.task_steps.verifiers.Turn",
"agent_env.task_step.task_steps.verifiers.Message", "agent_env.task_step.task_steps.verifiers.Firing",
Expand Down
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

13 changes: 12 additions & 1 deletion packages/agentenv-protocol/tests/test_a2a_agent_clock.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,17 @@ def _status(image, **status) -> None:
(image / "status.json").write_text(json.dumps(status))


EARLIER_STATUS_AGE_NS = 1_000_000_000


def _status_from_before(image, **status) -> None:
"""A status the follower wrote for an earlier config, ``EARLIER_STATUS_AGE_NS`` before this one: a file's mtime on
Linux moves in ticks of a few milliseconds, so a config written right after a status can share its mtime."""
_status(image, **status)
before = time.time_ns() - EARLIER_STATUS_AGE_NS
os.utime(image / "status.json", ns=(before, before))


def _follows_the_next_config(image, **status) -> threading.Thread:
"""A follower that, as the real one, writes its status once it reads a config other than the one it had."""
config = image / "follower.json"
Expand Down Expand Up @@ -140,7 +151,7 @@ def test_a_zone_that_isnt_one_is_refused(image):
{"url": "http://gw/clock/watch", "name": "agent-solver"}, # this one, under the token the config replaced
])
def test_what_the_follower_showed_of_its_config_before_never_passes_for_the_new_one(image, before):
_status(image, connected=True, version="b:3", armed=True, **before)
_status_from_before(image, connected=True, version="b:3", armed=True, **before)
with _app() as client:
answer = client.post("/ext/clock/sync-time", json=REQUEST)

Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/bundle/ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from __future__ import annotations

import hashlib
import json
import stat
from collections.abc import Callable, Iterable, Iterator, Mapping
from contextlib import contextmanager, nullcontext
Expand All @@ -31,6 +30,7 @@
from agent_env.store.ids import local_namespace
from agent_env.store.local_state import holding_locks
from agent_env.task.store import TASKS_COLLECTION
from agent_env.utils.canonical import canonical_json
from agent_env.utils.docker.context import shared_files

from .authoring import build_context_files
Expand Down Expand Up @@ -326,7 +326,7 @@ def _diff(before: Mapping[str, str], after: Mapping[str, str]) -> tuple[list[str


def _canonical(value: Any) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_scalar).encode()
return canonical_json(value, default=_scalar)


def _scalar(value: Any) -> str:
Expand Down
14 changes: 5 additions & 9 deletions src/agent_env/bundle/requires.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum
from importlib.metadata import distribution, packages_distributions
from importlib.metadata import packages_distributions
from pathlib import Path
from typing import TYPE_CHECKING

Expand Down Expand Up @@ -136,15 +136,11 @@ def _shipping(self, cls: type | None) -> str | None:
return None
if self._packages is None:
self._packages = packages_distributions()
module = cls.__module__
names = list(dict.fromkeys(self._packages.get(module.partition(".")[0], ())))
if len(names) > 1: # a namespace package: the one whose files hold the module
files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"}
names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}]
if len(names) != 1:
name = _requirements.distribution_holding(cls.__module__, self._packages)
if name is None:
return None
version = _requirements.installed(canonicalize_name(names[0]))
return f"{names[0]} {version}" if version else names[0]
version = _requirements.installed(canonicalize_name(name))
return f"{name} {version}" if version else name


def uses(resolved: ResolvedBundle, providers: Providers | None = None) -> list[Use]:
Expand Down
12 changes: 11 additions & 1 deletion src/agent_env/plugins/_requirements.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import functools
import shlex
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Mapping, Sequence
from importlib.metadata import PackageNotFoundError, distribution, version
from typing import Any, Optional

Expand All @@ -30,6 +30,16 @@ def installed(name: str) -> Optional[str]:
return None


def distribution_holding(module: str, packages: Mapping[str, Sequence[str]]) -> Optional[str]:
"""The one installed distribution whose files hold ``module``, given ``packages`` (``packages_distributions()``);
None when none or several do, as for a module on the path or an editable install."""
names = list(dict.fromkeys(packages.get(module.partition(".")[0], ())))
if len(names) > 1: # a namespace package: the one whose files hold the module
files = {f"{module.replace('.', '/')}.py", f"{module.replace('.', '/')}/__init__.py"}
names = [name for name in names if files & {str(file) for file in distribution(name).files or ()}]
return names[0] if len(names) == 1 else None


def incompatibility(dist: Any) -> Optional[str]:
"""Why ``dist``'s requirements on agent-env exclude the installed versions, or None.

Expand Down
12 changes: 8 additions & 4 deletions src/agent_env/task_step/task_steps/verifiers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
"""Verifier task steps: the steps that score a run and record a row under
``context.metadata["verifications"]``, plus the helpers only they use (score
aggregation, judge output formats, trajectory compaction and frame selection for
the LLM judge)."""
"""Verifier task steps: the steps that grade a run, the ``VerifierTaskStep`` base they move onto, which writes each one's
named grade under ``context.metadata["grades"]`` beside its ``verifications`` entry, and the helpers only they use
(score aggregation, judge output formats, trajectory compaction and frame selection for the LLM judge)."""

from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score
from agent_env.task_step.task_steps.verifiers.verifier import Grade, Verification, VerifierStatus, VerifierTaskStep
from agent_env.task_step.task_steps.verifiers.run_context import EnvClock, Firing, Message, Run, RunRecordUnavailable, Turn
from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier
Expand All @@ -18,13 +18,17 @@
"EnvClock",
"EnvOutcomeVerifierTaskStep",
"Firing",
"Grade",
"Message",
"RubricsVerifierTaskStep",
"Run",
"RunContainerUnitTestsVerifierTaskStep",
"RunRecordUnavailable",
"ScoreAggregator",
"Turn",
"Verification",
"VerifierStatus",
"VerifierTaskStep",
"VerifySandboxTaskStep",
"aggregate_score",
"is_rubrics_verifier",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
response_regex_present) raise ValueError. A verifier that can't actually
verify anything is a misconfiguration, not a runtime decision.

Result rows merge `{**criterion, **outcome}` and aggregated score is written
to `context.metadata["verifications"][verifier_id]`, matching the shape used
by `VerifySandboxTaskStep` and `RubricsVerifierTaskStep`.
Result rows merge `{**criterion, **outcome}`. On the `VerifierTaskStep` base, which
aggregates them into the step's grade and writes the
`context.metadata["verifications"][verifier_id]` entry, in the shape used by
`VerifySandboxTaskStep` and `RubricsVerifierTaskStep`.
"""
from __future__ import annotations

Expand All @@ -22,8 +23,9 @@
from typing import ClassVar, Optional

from agent_env.task_step.context import StepContextError, TaskStepContext
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency
from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score
from agent_env.task_step.task_step import TaskStepDependency
from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator
from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep

logger = logging.getLogger(__name__)

Expand All @@ -33,9 +35,8 @@
}


class AgentPromptResponseVerifierTaskStep(TaskStep):
class AgentPromptResponseVerifierTaskStep(VerifierTaskStep):
type: ClassVar[str] = "agent_prompt_response_verifier"
step_role = StepRole.GRADER
entity_refs = ()

def __init__(
Expand All @@ -44,18 +45,17 @@ def __init__(
version: Optional[int],
prompt_id: str,
criteria: Optional[list[dict]] = None,
score_aggregator: Optional[ScoreAggregator] = None,
score_aggregator: Optional[ScoreAggregator | str] = None,
verifier_id: Optional[str] = None,
depends_on: Optional[list[TaskStepDependency]] = None,
fail_task_on_error: bool = True,
min_passing_score: Optional[float] = None,
):
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error)
super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error,
verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator,
min_passing_score=min_passing_score)
self.prompt_id = prompt_id
self.criteria = list(criteria or [])
if isinstance(score_aggregator, str):
score_aggregator = ScoreAggregator(score_aggregator)
self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS
self.verifier_id = verifier_id or uuid.uuid4().hex

def to_dict(self) -> dict:
base = super().to_dict()
Expand All @@ -67,16 +67,9 @@ def to_dict(self) -> dict:

@classmethod
def from_dict(cls, data: dict) -> "AgentPromptResponseVerifierTaskStep":
raw_agg = data.get("score_aggregator")
return cls(
**cls._base_from_dict(data),
prompt_id=data["prompt_id"],
criteria=data.get("criteria"),
score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None,
verifier_id=data.get("verifier_id"),
)
return cls(**cls._base_from_dict(data), prompt_id=data["prompt_id"], criteria=data.get("criteria"))

async def execute(self, context: TaskStepContext) -> TaskStepContext:
async def verify(self, context: TaskStepContext) -> Verification:
prompt_response = next(
(pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id),
None,
Expand All @@ -88,16 +81,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
logger.warning(
f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}"
)
context.metadata.setdefault("verifications", {})[self.verifier_id] = {
"results": [{
"id": "prompt_error",
"score": 0,
"result": False,
"message": f"Skipped: prompt had error_type={prompt_response.error_type}",
}],
return Verification(criteria=[{
"id": "prompt_error",
"score": 0,
}
return context
"result": False,
"message": f"Skipped: prompt had error_type={prompt_response.error_type}",
}], score=0)

response = prompt_response.response or ""

Expand All @@ -117,18 +106,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
"result": bool(outcome["passed"]),
"justification": outcome["justification"],
})

score = aggregate_score(results, self.score_aggregator)

context.metadata.setdefault("verifications", {})[self.verifier_id] = {
"results": results,
"score": score,
}
logger.info(
f"Verification '{self.verifier_id}': "
f"{len(results)} criteria evaluated, score={score}"
)
return context
return Verification(criteria=results)

def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict:
rtype = criterion["type"]
Expand Down
Loading
Loading