Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions src/agent_env/bundle/authoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class AuthoringContext:
entry: BundleEntry
#: How writing a computer env composes it.
composing: Composing = Composing()
#: Whether writing a computer env composes it at all: not one no task of the run or registration deploys.
compose: bool = True

@property
def id(self) -> str:
Expand Down
52 changes: 40 additions & 12 deletions src/agent_env/bundle/materialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
from .ledger import Ledger, locked_ids, materializing
from .parse import BundleError, BundleKind
from .plan import Plan, Write, artifact_writer, env_writer, unpinned_store_refs
from .resolve import BuiltImage, BundleImage, ReferencedImage, build_step
from .resolve import BuiltImage, BundleImage, ReferencedImage, ResolvedEntry, build_step

#: How much of each build's reason the problem naming a computer env no deploy can boot quotes: a failed build's reason
#: holds the build's error, which can run to pages.
Expand Down Expand Up @@ -112,10 +112,11 @@ def materialize(
Each write gets the version it would reuse or the store's next, which another run can take first. A step
reading what the run would write first isn't preflighted, since the store doesn't hold it yet.

Each computer env written is composed as ``composing`` says, and so is what one reused has left, as ``agent-env
env build`` does; ``on_progress``, when given, is told how each goes, a line at a time with its write. With
``bootable``, as a registration has it, a computer env written or reused whose compose left no build a deploy can
boot stops it once every entity is written, before any task or eval, raising BundleError naming each such env.
Each computer env written that the run or registration composes (``_composed_envs``) is composed as ``composing``
says, and so is what one reused has left, as ``agent-env env build`` does; any other is written uncomposed and
composes at its first deploy. ``on_progress``, when given, is told how each goes, a line at a time with its write.
With ``bootable``, as a registration has it, a computer env written or reused whose compose left no build a deploy
can boot stops it once every entity is written, before any task or eval, raising BundleError naming each such env.

``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each
task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is
Expand All @@ -130,6 +131,10 @@ def materialize(
evals = [write for write in plan.writes if write.kind is BundleKind.EVAL]
done: dict[tuple[str, str], Materialized] = {}
left: list[str] = []
composed = _composed_envs(plan)

def composes(write: Write) -> bool:
return composed is None or write.id in composed
unbootable: list[str] = []

def through_ledger(write: Write, write_fn: Callable[[], int]) -> None:
Expand All @@ -154,13 +159,13 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None:
through_ledger(write, lambda: _write_pushed_image(plan, write, on_build))
else:
through_ledger(write, lambda: _write_entity(plan, write, here, on_build,
_composing(composing, on_progress, write)))
_composing(composing, on_progress, write), composes(write)))
if here and done[_key(write)].reused:
_build_reused(plan, write, done[_key(write)].version, on_build)
if write.kind is BundleKind.ENV and done[_key(write)].reused and _compose_reused(
if write.kind is BundleKind.ENV and composes(write) and done[_key(write)].reused and _compose_reused(
write, done[_key(write)].version, _composing(composing, on_progress, write), dry_run):
left.append(write.id)
if bootable and not dry_run and write.kind is BundleKind.ENV and (
if bootable and not dry_run and write.kind is BundleKind.ENV and composes(write) and (
why := _unbootable(plan, write, done[_key(write)].version, composing)):
unbootable.append(why)
if unbootable:
Expand Down Expand Up @@ -192,15 +197,15 @@ def _composing(composing: Composing, on_progress: Callable[[Write, str], None] |


def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None,
composing: Composing) -> int:
composing: Composing, compose: bool = True) -> int:
if isinstance(write.source, ReferencedImage):
return _write_referenced_image(plan, write)
if isinstance(write.source, BuiltImage):
if write.id in plan.published:
return _write_published_image(plan, write)
return _write_build_context(plan, write, here, on_build)
if write.kind is BundleKind.ENV:
return _write_env(plan, write, composing)
return _write_env(plan, write, composing, compose)
return _WRITERS[write.kind](plan, write)


Expand Down Expand Up @@ -374,11 +379,34 @@ def _unbootable(plan: Plan, write: Write, version: int, composing: Composing) ->
f"{write.id} --version {version}{how}, and then register again")


def _write_env(plan: Plan, write: Write, composing: Composing = Composing()) -> int:
def _write_env(plan: Plan, write: Write, composing: Composing = Composing(), compose: bool = True) -> int:
entry = write.source.entry
cls = get_env_registry()[entry.type]
return cls.from_toml(_pinned(plan, write, cls.toml_refs),
AuthoringContext(plan.bundle_of(entry), entry, composing=composing)).version
AuthoringContext(plan.bundle_of(entry), entry, composing=composing, compose=compose)).version


def _composed_envs(plan: Plan) -> frozenset[str] | None:
"""The envs a run or registration of ``plan`` composes: each its tasks deploy, and each those hold but not a
computer env's own apps and driver, which run inside it; and each it pulls in from an installed bundle, whose one
write every bundle shares. None, every env, for a plan with no tasks, which writes its envs for others to deploy,
each alone."""
tasks = [write.source for write in plan.writes if write.kind is BundleKind.TASK]
if not tasks:
return None
envs = {write.id: write.source for write in plan.writes
if write.kind is BundleKind.ENV and isinstance(write.source, ResolvedEntry)}
named = [ref.id for task in tasks for ref in task.references if ref.kind is EntityKind.ENV]
deployed: set[str] = set()
while named:
if (env_id := named.pop()) in deployed:
continue
deployed.add(env_id)
source = envs.get(env_id)
cls = get_env_registry().get(source.entry.type) if source is not None else None
if source is not None and not (cls is not None and issubclass(cls, ComputerEnv)):
named += [ref.id for ref in source.references if ref.kind is EntityKind.ENV]
return frozenset(deployed | {env_id for env_id, source in envs.items() if source.entry in plan.origins})


def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any:
Expand Down
2 changes: 1 addition & 1 deletion src/agent_env/env/envs/computer/app_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv:
return cls.put(id=ctx.id, environment_name=fields["environment_name"], module=fields["module"],
files=universe, declaration=declaration, computer=computer, driver=driver,
build_settings=fields.get("build_settings"), settings=fields.get("settings"),
composing=ctx.composing)
composing=ctx.composing, compose=ctx.compose)
except ValueError as e:
ctx.refuse([str(e)])

Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/env/envs/computer/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv:
ComputerDriverEnv)
fields["apps"], fields["app_settings"] = _listed_apps(fields.get("apps", []), ctx)
try:
return cls.put(id=ctx.id, computer=base, driver=driver, composing=ctx.composing, **fields)
return cls.put(id=ctx.id, computer=base, driver=driver, composing=ctx.composing, compose=ctx.compose,
**fields)
except ValueError as e:
ctx.refuse([str(e)])

Expand Down
58 changes: 58 additions & 0 deletions tst/unit/bundle/computer_env_folder_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,64 @@ def test_a_computer_env_folder_that_declares_what_cant_compose_is_refused_naming
assert composed == []


#: The two app folders, each naming the computer and driver it deploys on alone.
APPS_ALONE = {path: text + 'computer = "ubuntu-24.04"\ndriver = "reference"\n' if path.endswith("env.toml") else text
for path, text in APPS.items()}


def _composed_ids(composed) -> list[str]:
return [call[0].rsplit("/", 1)[1] for call in composed]


@pytest.mark.parametrize("names, expected, apps", [
("desktop", ["desktop"], ("mail", "notes")), ("mail", ["mail"], ("mail",)),
], ids=["the-desktop", "an-app-alone"])
def test_a_run_composes_only_the_computer_envs_its_tasks_deploy_not_the_apps_on_one(tmp_path, composed, names,
expected, apps):
"""An app that names its own computer and driver is a computer too, which no task deploys while it's on a
desktop; a task that names it alone does deploy it."""
root = layout(tmp_path / "desk", {"envs/desktop/env.toml": WITH_APPS, **APPS_ALONE, "tasks/t.json": json.dumps([
{"id": "env", "type": "deploy_env", "env_id": names}])})

materialize(_plan(root), build_here=local_builds(_plan(root)))

id_root = _plan(root).bundle.bundle.id_root
assert _composed_ids(composed) == expected
assert all(Env.get(f"{id_root}/{app}").computer is not None for app in apps)


def test_a_computer_a_multi_env_holds_is_deployed_with_it_but_not_that_computers_apps(tmp_path, composed):
root = layout(tmp_path / "desk", {"envs/desktop/env.toml": WITH_APPS, **APPS_ALONE,
"envs/office/env.toml": 'type = "multi"\ncomputer_envs = ["desktop"]\n',
"tasks/t.json": json.dumps([{"id": "env", "type": "deploy_env",
"env_id": "office"}])})

materialize(_plan(root), build_here=local_builds(_plan(root)))

assert _composed_ids(composed) == ["desktop"]


def test_a_bundle_with_no_tasks_composes_every_computer_env_it_writes_each_deployable_alone(tmp_path, composed):
root = layout(tmp_path / "apps", APPS_ALONE)
resolved = resolve_bundle(parse_bundle(root))

materialize(plan_entities(resolved, [entry.id for entry in resolved.bundle.entries], installed=installed_bundles))

assert sorted(_composed_ids(composed)) == ["mail", "notes"]


def test_a_reused_app_no_task_deploys_is_not_composed_again_nor_said_to_be(tmp_path, composed):
root = layout(tmp_path / "desk", {"envs/desktop/env.toml": WITH_APPS, **APPS_ALONE, "tasks/t.json": json.dumps([
{"id": "env", "type": "deploy_env", "env_id": "desktop"}])})
materialize(_plan(root), build_here=local_builds(_plan(root)))
told = []

dry = materialize(_plan(root), dry_run=True,
on_progress=lambda write, line: told.append(write.id.rsplit("/", 1)[1]))

assert set(told) <= {"desktop"} and all(not name.endswith(("mail", "notes")) for name in dry.composing)


def _office(root, computers):
"""A bundle whose multi env ``office`` holds the computer env folders ``computers``, by folder."""
return layout(root, {**{f"envs/{folder}/env.toml": toml for folder, toml in computers.items()},
Expand Down
40 changes: 37 additions & 3 deletions tst/unit/bundle/register_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
from agent_env.task import Task
from agent_env.task_step.task_step import TaskStep
from tst.unit.bundle._support import DIGEST, layout
from tst.unit.bundle.computer_app_env_test import APP, TOML as APP_TOML
from tst.unit.bundle.computer_driver_env_test import LINUX

BOX = {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"}
# Not a credential: what a remote URL's user info holds in the tests that its name drops, joined in at run time.
Expand Down Expand Up @@ -543,9 +545,9 @@ def test_the_installed_chromium_registers_over_its_own_driver_though_the_store_h
assert Env.get("chromium", versions["chromium"]).driver.version == 3


def _desktop(home, monkeypatch, compose):
def _desktop(home, monkeypatch, compose, extra=None):
"""A bundle of a computer env on the shipped base and driver, registered into a shared store as the hub is, under
their aliases, ``compose`` standing in for the composer."""
their aliases, ``compose`` standing in for the composer; ``extra`` replaces or adds files."""
for module in (register_module, env_store_module):
monkeypatch.setattr(module, "shared_store", lambda store: True)
bases = {image.ref() for image in COMPUTER_IMAGES}
Expand All @@ -554,10 +556,28 @@ def _desktop(home, monkeypatch, compose):
lambda image, arches, read: computer_artifact_module._unread(arches, "no registry in a test"))
monkeypatch.setattr(composer_module, "compose", compose)
files = {"envs/desktop/env.toml": 'type = "computer"\ncomputer = "ubuntu-24.04"\ndriver = "reference"\n',
"tasks/t.json": json.dumps([{"id": "env", "type": "deploy_env", "env_id": "desktop"}])}
"tasks/t.json": json.dumps([{"id": "env", "type": "deploy_env", "env_id": "desktop"}]), **(extra or {})}
return layout(home / "desk", files)


def test_a_registration_composes_the_desktop_its_task_deploys_and_not_each_app_on_it_alone(home, composed, monkeypatch):
"""Each app names the computer and driver it deploys on alone, so each is a computer too, which no task deploys."""
apps = {f"envs/{name}/{file}": text for name in ("mail", "notes") for file, text in (
("env.toml", APP_TOML + 'computer = "ubuntu-24.04"\ndriver = "reference"\n'),
("app.py", APP.format(supports=LINUX, skill="skill.md", linux_skill="skill-linux.md")),
("requirements.txt", "agentenv-framework-protocol\n"), ("skill.md", "# Mail\n"),
("skill-linux.md", "# Mail on Linux\n"))}
composings = []
root = _desktop(home, monkeypatch, lambda env, pairs=None, composing=None: composings.append(env.id), {
**apps, "envs/desktop/env.toml": 'type = "computer"\ncomputer = "ubuntu-24.04"\ndriver = "reference"\n'
'apps = ["mail", "notes"]\n'})

registered = register_bundle(root)

assert composings == ["desk-desktop"]
assert {done.write.id for done in registered.materialization.writes} >= {"desk-mail", "desk-notes", "desk-desktop"}


def _link(containers=(), disks=()):
"""A sandbox provider that runs containers of ``containers`` and boots disks of ``disks`` natively."""
link = SimpleNamespace(native_platforms=lambda: frozenset(f"linux/{arch}" for arch in containers),
Expand Down Expand Up @@ -697,3 +717,17 @@ def dies_after_writing_docs(self, write, needs):

assert {done.write.id: (done.version, done.reused) for done in resumed.materialization.writes} == {
"triage-docs": (1, True), "triage-greeting": (1, False), "triage-t": (1, False)}


def test_an_installed_app_a_registration_pulls_in_is_composed_though_no_task_deploys_it_alone(home, composed,
monkeypatch):
"""Every bundle shares the shared store's one write of an installed bundle's env, and a local run can't compose the
store's env at its first deploy, so the registration that writes it leaves it deployable."""
composings = []
root = _desktop(home, monkeypatch, lambda env, pairs=None, composing=None: composings.append(env.id), {
"envs/desktop/env.toml": 'type = "computer"\ncomputer = "ubuntu-24.04"\ndriver = "reference"\n'
'apps = ["chromium"]\n'})

register_bundle(root)

assert sorted(composings) == ["chromium", "desk-desktop"]
Loading