Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions src/agent_env/bundle/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]:
headlines is None or isinstance(headlines, list) and all(isinstance(h, str) for h in headlines)):
raise ValueError("a task file must be a JSON array of step objects, or an object whose steps is one and whose "
"headline_verifiers lists step ids")
for index, step in enumerate(steps):
for key in ("id", "type"):
if isinstance(step.get(key), (list, dict)):
raise ValueError(f"step #{index}'s {key} is a JSON {'array' if isinstance(step[key], list) else 'object'}"
f"; it must be a string")
return steps, headlines


Expand Down
48 changes: 37 additions & 11 deletions src/agent_env/bundle/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
from agent_env.store.base import NotFoundError
from agent_env.store.ids import LOCAL_PREFIX, is_local_id
from agent_env.store.routing import namespace_routing_enabled
from agent_env.task.lint import Finding, lint, refuses
from agent_env.task.task import Task
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep
Expand Down Expand Up @@ -198,6 +199,9 @@ class Plan:
alias_uses: Mapping[EntityKind, Mapping[str, AliasUse]] = field(default_factory=dict)
agent: AgentOverride | None = None # the agent --agent names for the solver
variant: Varied | None = None # what --set changes, written under its variant's ids
# Each selected task's task lint findings that didn't refuse it, with its path: its warnings, and its errors too
# when the plan skipped lint.
lint_findings: tuple[tuple[str, Finding], ...] = ()

def bundle_of(self, entry: BundleEntry) -> Bundle:
"""The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's."""
Expand All @@ -210,14 +214,17 @@ def path(self, entry: BundleEntry) -> str:


def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: Sequence[str] = (),
installed: Installed | None = None, agent: str | None = None, default_agents: bool = True) -> Plan:
installed: Installed | None = None, agent: str | None = None, default_agents: bool = True,
skip_lint: bool = False) -> Plan:
"""Plan a run of ``tasks`` and ``evals``, each a name or id in this bundle. With neither, the
run is every eval or, in a bundle without evals, every task. ``installed`` lists the installed bundles whose
entities a reference pulls in (``installed_bundles``); without it, an id under one is a store id. ``agent`` names
the agent the solver runs as, as a task's reference would, leaving the tasks as written (``Plan.agent``). Without
``default_agents``, a step that leaves its agent unset pulls in no configured default, as for a task written to
run wherever another's default applies. Raises BundleError listing every problem found."""
return _Planner(resolved, installed, default_agents).plan(tasks, evals, agent)
run wherever another's default applies. With ``skip_lint``, the errors of task lint's own rules are findings it
keeps (``Plan.lint_findings``), not problems; a step's config problems stay problems. Raises BundleError listing
every problem found."""
return _Planner(resolved, installed, default_agents, skip_lint).plan(tasks, evals, agent)


def plan_entities(resolved: ResolvedBundle, ids: Sequence[str], *, installed: Installed | None = None) -> Plan:
Expand All @@ -244,10 +251,14 @@ class _Pullable:


class _Planner:
def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True):
def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True,
skip_lint: bool = False):
self.resolved = resolved
self.default_agents = default_agents
self.skip_lint = skip_lint
self.problems: list[str] = []
self.lint_findings: list[tuple[str, Finding]] = []
self.lint_refused = 0
self.nodes: dict[_Key, ResolvedEntry | BundleImage] = {}
self.rank: dict[_Key, int] = {}
self.edges: dict[_Key, list[_Key]] = {}
Expand Down Expand Up @@ -575,19 +586,23 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr
store_refs, store_latest = self._check_store_refs(closure)
if override is not None:
override = replace(override, version=store_latest.get((EntityKind.AGENT, override.agent_id)))
self._build_tasks(closure)
self._build_tasks(closure, linted=True)
self._check_files(closure)
self._check_cli_loads(closure)
self._check_env_types(closure)
self._check_multi_names(closure, store_latest)
self._check_environment_names(closure, store_latest)
self._check_loaded_environments(closure, store_latest)
if self.problems:
raise NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems)
refused = NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems)
if self.lint_refused:
refused.add_note(f"task lint found {self.lint_refused} of these; --skip-lint runs the tasks anyway")
raise refused
writes = self._ordered(closure)
return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest,
dict(self.origins), self._published(writes),
{kind: dict(uses) for kind, uses in self.alias_uses.items()}, override)
{kind: dict(uses) for kind, uses in self.alias_uses.items()}, override,
lint_findings=tuple(self.lint_findings))

def check(self) -> None:
"""The checks a run makes before it reads a store. A bundle with nothing to run passes only when its
Expand Down Expand Up @@ -1122,8 +1137,12 @@ def _store_entity(self, ref: Reference, store_latest: dict[tuple[EntityKind, str
except (NotFoundError, ValueError, KeyError, TypeError):
return None # reported by the store ref check

def _build_tasks(self, closure: set[_Key]) -> None:
"""Build each task's steps the way a write will, so a bad step fails before anything is written."""
def _build_tasks(self, closure: set[_Key], *, linted: bool = False) -> None:
"""Build each task's steps the way a write will, so a bad step fails before anything is written. ``linted``,
for a run or registration, adds task lint: its errors are problems, unless the plan skips lint, and the rest
``lint_findings``. The
store-free check leaves it out, since a folder ``bundle init`` writes leaves user-scoped fields for its runner
to fill in."""
registry = get_task_step_registry()
for key in sorted(closure, key=self.rank.__getitem__):
source = self.nodes[key]
Expand All @@ -1140,10 +1159,17 @@ def _build_tasks(self, closure: set[_Key]) -> None:
self.steps[key] = steps
if not failed:
try:
Task(id=source.entry.id, version=None, steps=steps,
headline_verifiers=source.entry.headline_verifiers)
task = Task(id=source.entry.id, version=None, steps=steps,
headline_verifiers=source.entry.headline_verifiers)
except ValueError as e:
self.problems.append(f"{self._path(source.entry)}: {e}")
continue
for finding in lint(task, source.config, unpinned=False) if linted else ():
if refuses(finding, skip_lint=self.skip_lint):
self.problems.append(f"{self._path(source.entry)}: {finding.text}")
self.lint_refused += not refuses(finding, skip_lint=True)
else:
self.lint_findings.append((self._path(source.entry), finding))

# Order

Expand Down
16 changes: 10 additions & 6 deletions src/agent_env/bundle/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def register_bundle(
on_progress: Callable[[str], None] | None = None,
sandbox: str | None = None,
emulate: bool = True,
skip_lint: bool = False,
) -> Registration:
"""Register the bundle at ``root``: ``tasks`` and ``evals`` by name, or with neither every task and eval it has,
or every entity when it has none, with what they reference. ``prefix`` opens each entry's registry id, the folder's
Expand All @@ -91,7 +92,8 @@ def register_bundle(
run of the folder would (``declaring_for``). ``sandbox`` is the provider its tasks deploy on, as a run's is: each
computer env composes only the forms its deploys there boot, and without it every form its base declares. Either
way it composes on the env's provider, emulating an arch no link of that one runs natively unless ``emulate`` is
False, which leaves it skipped.
False, which leaves it skipped. ``skip_lint`` registers tasks task lint's own rules
refuse (``Registration.plan.lint_findings``).

It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is
written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse. So does
Expand All @@ -101,7 +103,7 @@ def register_bundle(
with namespace_routing():
bundle = parse_bundle(Path(root))
resolved = resolve_bundle(bundle)
plan = _plan(resolved, tasks, evals)
plan = _plan(resolved, tasks, evals, skip_lint)
ids, problems = registry_ids(plan, _prefix(bundle, prefix))
if problems:
raise BundleError(problems)
Expand Down Expand Up @@ -130,15 +132,16 @@ def registered_line(plan: Plan, done: Materialized) -> str:
return f"{_label(plan, done.write)} -> {done.write.kind.store} {done.write.id} v{done.version} ({why})"


def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str]) -> Plan:
def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str], skip_lint: bool = False) -> Plan:
"""The plan of ``tasks`` and ``evals``, or of every task and eval, or of every entity of a bundle with none."""
entries = resolved.bundle.entries
if not tasks and not evals:
tasks = [entry.name for entry in entries if entry.kind is BundleKind.TASK]
evals = [entry.name for entry in entries if entry.kind is BundleKind.EVAL]
if not tasks and not evals:
return plan_entities(resolved, [entry.id for entry in entries], installed=installed_bundles)
return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False)
return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False,
skip_lint=skip_lint)


def _prefix(bundle: Bundle, prefix: str | None) -> str:
Expand Down Expand Up @@ -241,8 +244,9 @@ def _slug(name: str) -> str:


# The fields of a Plan that hold no ``@local`` id, so a translated plan keeps them: the store's latest versions, by store
# id, and the agent ``--agent`` names, which a registration plans without.
_KEPT = frozenset({"store_latest", "agent", "variant"})
# id, the agent ``--agent`` names, which a registration plans without, and task lint's warnings, which name the bundle's
# files.
_KEPT = frozenset({"store_latest", "agent", "variant", "lint_findings"})


def translated(plan: Plan, ids: Mapping[str, str]) -> Plan:
Expand Down
25 changes: 16 additions & 9 deletions src/agent_env/bundle/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from agent_env.store.routing import namespace_routing, run_scope
from agent_env.task.store import record_task_cancelled
from agent_env.task.interrupts import Interrupts
from agent_env.task.lint import said
from agent_env.utils.progress import run_progress
from agent_env.task.task import Task, recorded_scores
from agent_env.task.teardown import TeardownReport, attempt_epoch, record_teardown, teardown_run
Expand Down Expand Up @@ -225,6 +226,7 @@ def run_bundle(
emulate: bool = True,
source: HubSource | None = None,
variant: Variant | None = None,
skip_lint: bool = False,
) -> BundleRun:
"""Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run
each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the
Expand All @@ -237,7 +239,8 @@ def run_bundle(
(``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come
from its path. ``source`` says where a Hub bundle came from (``HubBundle.source``), which each run's context
records under ``metadata["bundle_source"]``. ``variant`` sets env keys for this run (``with_variant``), writing
what it changes under ids of their own and leaving the bundle as written.
what it changes under ids of their own and leaving the bundle as written. ``skip_lint`` runs tasks task lint's own
rules refuse, saying why (``plan_bundle``).

Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone.
``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``.
Expand All @@ -264,7 +267,7 @@ def run_bundle(
with namespace_routing():
prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say,
declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing,
variant=variant)
variant=variant, skip_lint=skip_lint)
plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks]
with Interrupts() as interrupts:
overrides = _overrides(sandbox, user_model, judge_model)
Expand Down Expand Up @@ -352,6 +355,7 @@ def dry_run_bundle(
id_root: str | None = None,
emulate: bool = True,
variant: Variant | None = None,
skip_lint: bool = False,
) -> DryRun:
"""Check what ``run_bundle`` with these arguments checks before its first task runs, and say what it would
write and run, writing and running nothing: no entity, ledger row, lock or instance. Each write's version is
Expand All @@ -365,7 +369,7 @@ def dry_run_bundle(
composing = _composing(sandbox, emulate)
say = _progress(on_progress)
with namespace_routing():
plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant)
plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant, skip_lint=skip_lint)
_say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True))
materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)),
composing=composing, sandbox=sandbox,
Expand Down Expand Up @@ -408,9 +412,9 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu
def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None,
agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring,
composing: Composing = Composing(), *, solver_endpoint: bool = False,
variant: Variant | None = None) -> PreparedBundle:
variant: Variant | None = None, skip_lint: bool = False) -> PreparedBundle:
plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played,
solver_endpoint=solver_endpoint, variant=variant)
solver_endpoint=solver_endpoint, variant=variant, skip_lint=skip_lint)
_say_plan(plan, say, declaring)
materialization = _written_with_infra(plan, preflight, say, composing, sandbox)
to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan))
Expand All @@ -419,14 +423,15 @@ def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandb

def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_root: str | None,
sandbox: str | None, agent: str | None = None, *, played: bool = False,
solver_endpoint: bool = False, variant: Variant | None = None) -> tuple[Plan, Preflight]:
solver_endpoint: bool = False, variant: Variant | None = None,
skip_lint: bool = False) -> tuple[Plan, Preflight]:
bundle = parse_bundle(Path(root), id_root=id_root)
varied = with_variant(bundle, variant) if variant is not None else None
if varied is not None:
tasks, evals = varied.selected(tasks, BundleKind.TASK), varied.selected(evals, BundleKind.EVAL)
resolved = resolve_bundle(varied.bundle if varied is not None else bundle)
plan = plan_bundle(without_played_agents(resolved) if played else resolved, tasks=tasks, evals=evals,
installed=installed_bundles, agent=agent)
installed=installed_bundles, agent=agent, skip_lint=skip_lint)
refuse_store_tasks(plan)
if varied is not None:
plan = replace(plan, variant=varied)
Expand Down Expand Up @@ -484,8 +489,8 @@ def refuse_store_tasks(plan: Plan) -> None:

def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> None:
"""Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, the
ids what --set changes is written under, and what plugin packages the bundle uses that its bundle.toml doesn't
require, as ``declaring`` says (``declare``)."""
ids what --set changes is written under, what plugin packages the bundle uses that its bundle.toml doesn't
require, as ``declaring`` says (``declare``), and its tasks' lint findings, each kind once."""
for use in (use for uses in plan.alias_uses.values() for use in uses.values()):
say(_aliased(use))
if plan.agent is not None:
Expand All @@ -498,6 +503,8 @@ def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> N
say(f"{variant.typed}: {_changed(plan)} written under ids ending in ({variant.label})")
for line in declare(plan.bundle, declaring):
say(line)
for line in said(plan.lint_findings):
say(line)


def _changed(plan: Plan) -> str:
Expand Down
Loading
Loading