Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion src/agent_env/bundle/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
parse_bundle,
root_problems,
segment_problem,
task_file_body,
)
from agent_env.bundle.plan import STORE_GETTERS, check_bundle
from agent_env.bundle.register import REGISTRATIONS
Expand Down Expand Up @@ -278,7 +279,8 @@ def add_task(self, task: Task) -> str:
self._ref(site, f"{where}: {site.path}", pins)
self._default_agents(step, cls.entity_refs or (), where)
sync_context_paths(stored, steps)
self.files[path] = json.dumps(steps, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
self.files[path] = json.dumps(task_file_body(steps, task.headline_verifiers), indent=2, ensure_ascii=False,
allow_nan=False) + "\n"
self.tasks.append(ImportedTask(path, task.id, task.version, len(steps)))
self.declared += task.requires or ()
return name
Expand Down
5 changes: 5 additions & 0 deletions src/agent_env/bundle/ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ def digest(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Digest |
inputs["published"] = published.pinned
if write.requires:
inputs["requires"] = list(write.requires)
if write.kind is BundleKind.TASK and write.source.entry.headline_verifiers:
inputs["headline_verifiers"] = list(write.source.entry.headline_verifiers)
if self.pins_references or write.kind not in (BundleKind.TASK, BundleKind.EVAL):
# A document records the versions of what it references (an env its images, a universe its
# environments), so one written anew means it must be written again. A task or eval a run writes names
Expand Down Expand Up @@ -301,6 +303,9 @@ def _changes(row: Mapping[str, Any], digest: Digest) -> list[str]:
reasons.append("config changed")
if before.get("requires") != after.get("requires"):
reasons.append(f"requires {', '.join(after['requires'])}" if after.get("requires") else "requires nothing")
if before.get("headline_verifiers") != after.get("headline_verifiers"):
reasons.append(f"headline_verifiers {', '.join(after['headline_verifiers'])}" if after.get("headline_verifiers")
else "no headline_verifiers")
if before.get("published") != after.get("published"):
reasons.append(f"published as {after['published'].partition('@')[0]}" if after.get("published")
else "not found published")
Expand Down
5 changes: 3 additions & 2 deletions src/agent_env/bundle/materialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None:
for write in tasks:
through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps,
requires=write.requires,
entity_ids=built[write.id].entity_ids).version)
entity_ids=built[write.id].entity_ids,
headline_verifiers=built[write.id].headline_verifiers).version)
for write in evals:
through_ledger(write, lambda: _write_eval(plan, write, written))
return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left))
Expand Down Expand Up @@ -455,7 +456,7 @@ def _build(write: Write, plan: Plan, written: Mapping[tuple[str, str], Materiali
naming the one it reads."""
steps = write.source.config if written is None else pinned_steps(plan, write, written)
return Task(id=write.id, version=None, steps=[build_step(step) for step in steps],
entity_ids=write.source.entity_ids)
entity_ids=write.source.entity_ids, headline_verifiers=write.source.entry.headline_verifiers)


def pinned_steps(plan: Plan, write: Write, written: Mapping[tuple[str, str], Materialized]) -> list[dict]:
Expand Down
39 changes: 33 additions & 6 deletions src/agent_env/bundle/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
import unicodedata
from collections import Counter, defaultdict
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from enum import Enum
from pathlib import Path
from typing import Any, NoReturn
from typing import Any, NoReturn, Sequence

from packaging.requirements import InvalidRequirement, Requirement

Expand Down Expand Up @@ -65,9 +65,32 @@ def store(self) -> str:
CONFIG_FILES = {BundleKind.ENV: "env.toml", BundleKind.AGENT: "agent.toml", BundleKind.ARTIFACT: "artifact.toml"}
_DEFAULT_TYPE = {BundleKind.ENV: "mcp_server", BundleKind.AGENT: "a2a_agent"}
_EVAL_KEYS = ("tasks", "type", "id")
_TASK_KEYS = ("steps", "headline_verifiers")
_FILE_SUFFIX = {BundleKind.TASK: ".json", BundleKind.EVAL: ".toml"}


def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]:
"""A task file's steps and ``headline_verifiers``: a bare array of steps, or ``{"headline_verifiers": [...],
"steps": [...]}``. Raises ValueError saying what's wrong with any other shape."""
steps, headlines = data, None
if isinstance(data, dict):
if unknown := sorted(set(data) - set(_TASK_KEYS)):
raise ValueError(f"unknown key{'s' * (len(unknown) > 1)} {', '.join(map(repr, unknown))}; a task file's "
f"object takes {' and '.join(_TASK_KEYS)}")
steps, headlines = data.get("steps"), data.get("headline_verifiers")
if not (isinstance(steps, list) and all(isinstance(step, dict) for step in steps)) or not (
headlines is None or isinstance(headlines, list) and all(isinstance(h, str) for h in headlines)):
raise ValueError("a task file must be a JSON array of step objects, or an object whose steps is one and whose "
"headline_verifiers lists step ids")
return steps, headlines


def task_file_body(steps: list[dict], headlines: Sequence[str] | None) -> list[dict] | dict[str, Any]:
"""A task file holding ``steps`` and ``headlines``, as ``task_file_parts`` reads it back: the bare array when there
are no headlines."""
return {"headline_verifiers": list(headlines), "steps": steps} if headlines else steps


@dataclass(frozen=True)
class BundleEntry:
kind: BundleKind
Expand All @@ -77,6 +100,8 @@ class BundleEntry:
type: str # declared, else the kind's default; for artifacts, inferred from the directory
# The toml as written ({} when absent), a task's steps, None for skills.
config: Any = field(compare=False, repr=False)
# A task file's headline_verifiers, None when it names none.
headline_verifiers: tuple[str, ...] | None = field(default=None, compare=False)


@dataclass(frozen=True)
Expand Down Expand Up @@ -315,10 +340,12 @@ def _entry(self, kind: BundleKind, name: str, path: Path) -> BundleEntry:
return read.get(kind, self._artifact)(name, path)

def _task(self, name: str, path: Path) -> BundleEntry:
steps = self._read_json(path)
if not (isinstance(steps, list) and all(isinstance(step, dict) for step in steps)):
self._skip(path, "a task file must be a JSON array of step objects")
return self._make(BundleKind.TASK, name, path, "task", steps)
try:
steps, headlines = task_file_parts(self._read_json(path))
except ValueError as e:
self._skip(path, str(e))
entry = self._make(BundleKind.TASK, name, path, "task", steps)
return replace(entry, headline_verifiers=tuple(headlines) if headlines else None)

def _eval(self, name: str, path: Path) -> BundleEntry:
config = self._read_toml(path)
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/bundle/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -1140,7 +1140,8 @@ def _build_tasks(self, closure: set[_Key]) -> None:
self.steps[key] = steps
if not failed:
try:
Task(id=source.entry.id, version=None, steps=steps)
Task(id=source.entry.id, version=None, steps=steps,
headline_verifiers=source.entry.headline_verifiers)
except ValueError as e:
self.problems.append(f"{self._path(source.entry)}: {e}")

Expand Down
25 changes: 15 additions & 10 deletions src/agent_env/cli/task/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,32 @@ def create(filepath: Path, task_id: str, skip_validation: bool):
"""Create a task from a JSON file containing step definitions.

\b
The JSON file should contain a list of step dicts, e.g.:
The JSON file holds a list of step dicts, as a bundle's task file does, e.g.:
[
{"id": "step-1", "type": "deploy_env", "env_id": "my-env"},
{"id": "step-2", "type": "prompt_agent", "prompt": "Hello"}
]
or an object with them as "steps" and the step ids of its reward as "headline_verifiers".
"""
from agent_env.bundle.parse import task_file_parts
from agent_env.task import Task

click.echo(f"Reading steps from {filepath}...")
with open(filepath) as f:
steps = json.load(f)

if not isinstance(steps, list):
click.echo("Error: JSON file must contain a list of step dicts", err=True)
data = json.load(f)
try:
steps, headlines = task_file_parts(data)
except ValueError as e:
click.echo(f"Error: {e}", err=True)
raise SystemExit(1)

built = _build_steps(steps)
_report_preflight(
Task(id=task_id, version=None, steps=[s for _, s in built]),
skip_validation,
)
try:
checked = Task(id=task_id, version=None, steps=[s for _, s in built], headline_verifiers=headlines)
except ValueError as e:
click.echo(f"Error: {e}", err=True)
raise SystemExit(1)
_report_preflight(checked, skip_validation)

task_steps = []
for i, (kwargs, step) in enumerate(built):
Expand All @@ -49,7 +54,7 @@ def create(filepath: Path, task_id: str, skip_validation: bool):
task_steps.append(saved)

click.echo(f"Creating task '{task_id}' with {len(task_steps)} steps...")
task = Task.put(id=task_id, steps=task_steps)
task = Task.put(id=task_id, steps=task_steps, headline_verifiers=headlines)
click.echo(f"Created task: id={task.id} version={task.version} steps={len(task.steps)}")


Expand Down
7 changes: 4 additions & 3 deletions src/agent_env/cli/task/get.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
@click.option("--id", "task_id", required=True, help="Task id")
@click.option("--version", "task_version", default=None, type=int, help="Task version (defaults to latest)")
def get(task_id: str, task_version: int | None):
"""Get a task and output its step definitions as JSON."""
"""Get a task and output its step definitions as JSON, as task create takes them: an object with its steps and
headline_verifiers when it has them."""
from agent_env.bundle.parse import task_file_body
from agent_env.task import Task

task = Task.get(task_id, task_version)
steps = task.to_dict()["steps"]
click.echo(json.dumps(steps, indent=2))
click.echo(json.dumps(task_file_body(task.to_dict()["steps"], task.headline_verifiers), indent=2))
3 changes: 2 additions & 1 deletion src/agent_env/task/episode/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def from_task(
raise EpisodeError(f"task {task.id!r} can't run as an episode: " + "; ".join(found))
sandboxed = {name: works_in_sandbox(seeded, name) for name in outside}
steps = [_in_episode(step, as_run, sandboxed, turn_timeout) for step, as_run in zip(task.steps, seeded)]
return cls(copy=Task(id=task.id, version=task.version, steps=steps), trainer_agents=tuple(outside), seed=seed)
copy = Task(id=task.id, version=task.version, steps=steps, headline_verifiers=task.headline_verifiers)
return cls(copy=copy, trainer_agents=tuple(outside), seed=seed)

@property
def prompts(self) -> tuple[tuple[str, str | None], ...]:
Expand Down
22 changes: 22 additions & 0 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,8 @@ class Task:
registration records (``agent-env bundle register``); None when nothing recorded it.
``entity_ids`` maps each id a bundle rewrote in the task's steps, as the task names it, to the id the bundle gave
it; ``run`` puts it in ``context.metadata`` for ``run_entity_id``. None for a task stored as written.
``headline_verifiers`` names, by step id, the grades that are the task's reward: one is the reward and the run's
pass or fail, several a multi-reward over them; None, or empty, a multi-reward over every grade.
"""

type: ClassVar[str] = "task"
Expand All @@ -366,14 +368,31 @@ def __init__(
steps: list[TaskStep] | None = None,
requires: Sequence[str] | None = None,
entity_ids: Mapping[str, str] | None = None,
headline_verifiers: Sequence[str] | None = None,
):
self.id = id
self.version = version
self.steps: list[TaskStep] = steps or []
self.requires: list[str] | None = list(requires) if requires is not None else None
self.entity_ids: dict[str, str] | None = dict(entity_ids) if entity_ids else None
self.headline_verifiers: list[str] | None = list(headline_verifiers) if headline_verifiers else None
self._validate_dag(self.steps)
self._validate_pairings(self.steps)
self._validate_headlines(self.headline_verifiers, self.steps)

@staticmethod
def _validate_headlines(headlines: list[str] | None, steps: list[TaskStep]) -> None:
"""Refuse a headline that isn't a step id of this task, or that is named twice. Whether the step it names is a
verifier is the task lint's to say."""
if headlines is None:
return
ids = {step.id for step in steps}
unknown = [h for h in headlines if not isinstance(h, str) or h not in ids]
if unknown:
raise ValueError(f"headline_verifiers names no step in this task: {unknown!r} (its steps: {sorted(ids)})")
if len(set(headlines)) != len(headlines):
twice = sorted({h for h in headlines if headlines.count(h) > 1})
raise ValueError(f"headline_verifiers names {twice!r} more than once")

@staticmethod
def _validate_pairings(steps: list[TaskStep]) -> None:
Expand Down Expand Up @@ -446,6 +465,8 @@ def to_dict(self) -> dict:
data["requires"] = list(self.requires)
if self.entity_ids:
data["entity_ids"] = dict(self.entity_ids)
if self.headline_verifiers:
data["headline_verifiers"] = list(self.headline_verifiers)
return data

@classmethod
Expand Down Expand Up @@ -473,6 +494,7 @@ def from_dict(cls, data: dict) -> "Task":
steps=steps,
requires=data["requires"] if isinstance(data.get("requires"), list) else None,
entity_ids=data["entity_ids"] if isinstance(data.get("entity_ids"), dict) else None,
headline_verifiers=data["headline_verifiers"] if isinstance(data.get("headline_verifiers"), list) else None,
)

async def run(
Expand Down
16 changes: 16 additions & 0 deletions tst/integration/cli/run_bundle_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,22 @@ def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bund
assert [(result["criterion"], result["result"]) for result in results] == [("greets", False), ("check passes", False)]


def test_a_task_file_naming_its_headline_verifiers_runs_and_its_stored_task_keeps_them(bundle, quiet_logs):
steps = json.loads((bundle / "tasks/hello.json").read_text())
(bundle / "tasks/hello.json").write_text(json.dumps({"headline_verifiers": ["hello"], "steps": steps}))

assert "tasks/hello.json v1: passed (hello: 1)" in _run(bundle)
assert "tasks/hello.json: v1, unchanged" in _run(bundle)
with namespace_routing():
assert Task.get("@local/~/my-hello/hello").headline_verifiers == ["hello"]

(bundle / "tasks/hello.json").write_text(json.dumps(steps))
assert "tasks/hello.json: v2 (no headline_verifiers)" in _run(bundle)
with namespace_routing():
assert Task.get("@local/~/my-hello/hello").headline_verifiers is None
assert Task.get("@local/~/my-hello/hello", 1).headline_verifiers == ["hello"]


@skip_without_gnu_stat()
def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs):
steps = json.loads((bundle / "tasks/hello.json").read_text())
Expand Down
6 changes: 6 additions & 0 deletions tst/unit/bundle/_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
CLAUDE_CODE_IMAGE = next(image for image in AGENT_IMAGES if image.name == "agentenv-claude-code")
# The digest the registry fixture pins every tag to.
DIGEST = "sha256:" + "a" * 64
# A task's steps with two verifiers, the second an aggregate of the first, for headline_verifiers to name.
GRADED_STEPS = (
{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"},
{"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", "criteria": []},
{"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests"]},
)


def layout(root, files):
Expand Down
14 changes: 13 additions & 1 deletion tst/unit/bundle/importer_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from agent_env.a2a_agent import A2AAgent
from agent_env.artifact import FileArtifact
from agent_env.artifact.artifacts.skill import SkillArtifact
from agent_env.bundle import BundleError, parse_bundle
from agent_env.bundle import BundleError, BundleKind, parse_bundle
from agent_env.bundle import importer as importer_module
from agent_env.bundle.importer import ImportedEval, import_eval, import_task, task_images
from agent_env.bundle.installed import InstalledBundle
Expand Down Expand Up @@ -101,6 +101,18 @@ def test_a_task_imports_as_its_stored_steps_naming_each_entity_it_uses_by_the_fo
assert imported.pinned == ()


def test_a_task_naming_headline_verifiers_imports_as_the_object_form_its_folder_reads_back(store):
Task.put(id="triage", steps=Task.get("triage").steps, headline_verifiers=["judge"])

root = import_task("triage", store / "triage").root

written = json.loads((root / "tasks/triage.json").read_text())
assert (written["headline_verifiers"], [step["id"] for step in written["steps"]]) == (
["judge"], [step["id"] for step in STEPS])
(task,) = [entry for entry in parse_bundle(root).entries if entry.kind is BundleKind.TASK]
assert task.headline_verifiers == ("judge",)


def test_an_id_a_step_of_the_task_writes_stays_unpinned_and_unchanged(store):
steps = _steps(import_task("triage", store / "triage").root, "triage")

Expand Down
35 changes: 34 additions & 1 deletion tst/unit/bundle/materialize_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@
from agent_env.task_step.task_step import TaskStep
from agent_env.utils.docker.context import BuildContext
from agent_env.env.published_images import EXAMPLE_IMAGES
from tst.unit.bundle._support import CLAUDE_CODE_IMAGE, DIGEST, RefusingStore, layout, local_builds, local_store, plan_of, shipped_agent
from tst.unit.bundle._support import (CLAUDE_CODE_IMAGE, DIGEST, GRADED_STEPS, RefusingStore, layout, local_builds,
local_store, plan_of, shipped_agent)

ROOT = "@local/~/triage"
LAYOUT = {
Expand Down Expand Up @@ -377,6 +378,38 @@ def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_d
assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"}


def _task_file(root, **fields):
steps = list(GRADED_STEPS)
(root / "tasks/t.json").write_text(json.dumps({**fields, "steps": steps} if fields else steps))


def test_a_task_files_headline_verifiers_are_written_and_changing_them_writes_a_new_version(bundle_dir):
_task_file(bundle_dir)
bare = _summary(_run(bundle_dir))[f"{ROOT}/t"]
_task_file(bundle_dir, headline_verifiers=[])
assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0], True, ())

_task_file(bundle_dir, headline_verifiers=["final"])
assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0] + 1, False, ("headline_verifiers final",))
assert Task.get(f"{ROOT}/t").headline_verifiers == ["final"]
assert _summary(_run(bundle_dir))[f"{ROOT}/t"] == (bare[0] + 1, True, ())

_task_file(bundle_dir, headline_verifiers=["tests", "final"])
assert _summary(_run(bundle_dir))[f"{ROOT}/t"][1:] == (False, ("headline_verifiers tests, final",))
assert Task.get(f"{ROOT}/t").headline_verifiers == ["tests", "final"]


@_RUN_OR_DRY_RUN
def test_a_headline_naming_no_step_is_refused_before_anything_is_written(bundle_dir, dry_run):
_task_file(bundle_dir, headline_verifiers=["finl"])

(problem,) = _problems(lambda: _run(bundle_dir, dry_run=dry_run))

assert problem.startswith("tasks/t.json: headline_verifiers names no step in this task: ['finl']")
with pytest.raises(NotFoundError):
Task.get(f"{ROOT}/t")


@pytest.mark.parametrize(("toml", "problem"), [
('image = "claude-image"\nowner = "me"\n',
"agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, metadata, type and id"),
Expand Down
Loading
Loading