diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 602e03060..058aa45f4 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -91,23 +91,35 @@ def import_task(task_id: str, dest: Path | str, *, version: int | None = None, ``dest``. Its steps' scoped fields are left out unless ``include_scopes`` names their scope. An image whose store id ``image_refs`` maps to a registry reference is written as that reference, which a run pulls, its tag pinned to the digest its registry serves now when it gives none, with its build context beside it.""" - importer = _Importer(Path(dest), include_scopes, progress, image_refs) task = Task.get(task_id, version) - importer.reserve_aliases([task]) + importer = _importing([task], Path(dest), include_scopes, progress, image_refs) importer.add_task(task) return importer.write(f"Imported from the store's task {task.id} v{task.version}.", {"task": task.id, "version": task.version}) +def _importing(tasks: Collection[Task], dest: Path | None, include_scopes: Collection[FieldScope], + progress: Callable[[str], None] | None, image_refs: Mapping[str, str] | None) -> _Importer: + """An importer for ``tasks`` whose folders take no installed alias the export names floating, since a folder of + that name would capture the reference in a run. A first pass writing nothing finds them, an entity the export + writes naming one as well as a step. It changes what it reads as it goes, so it takes copies of the tasks and the + importer reads the rest again.""" + scout = _Importer(None, include_scopes) + for task in tasks: + scout.add_task(copy.deepcopy(task)) + importer = _Importer(dest, include_scopes, progress, image_refs) + importer.reserved, importer.installed, importer.aliases = scout.floated, scout.installed, scout.aliases + return importer + + def task_images(task_id: str, *, version: int | None = None, include_scopes: Collection[FieldScope] = ()) -> tuple[DockerImageArtifact, ...]: """The images ``import_task`` writes for the stored task ``task_id``, at ``version`` or its latest, each once in the order it reaches them, found without downloading anything: what an export pushes to a registry before importing the task with ``image_refs`` naming them there. Every image is listed, however it's stored, since each is then given a reference; raises BundleError when the import would fail on anything else it reads.""" - importer = _Importer(None, include_scopes) task = Task.get(task_id, version) - importer.reserve_aliases([task]) + importer = _importing([task], None, include_scopes, None, None) importer.add_task(task) if importer.problems: raise BundleError(importer.problems) @@ -119,23 +131,24 @@ def import_eval(eval_id: str, dest: Path | str, *, version: int | None = None, progress: Callable[[str], None] | None = None) -> BundleImport: """Write the stored eval ``eval_id``, at ``version`` or its latest, the tasks it names and every entity they use as a bundle, leaving out their steps' scoped fields and writing ``image_refs`` as ``import_task`` does.""" - importer = _Importer(Path(dest), include_scopes, progress, image_refs) stored = Eval.get(eval_id, version) - tasks, seen = [], set() + tasks, seen, problems, duplicates = [], set(), [], [] for entry in stored.tasks: try: task = Task.get(entry.task_id, entry.task_version) except NotFoundError: at = f" v{entry.task_version}" if entry.task_version is not None else "" - importer.problems.append(f"eval {stored.id} v{stored.version} names task {entry.task_id}{at}, which isn't " - "in the store") + problems.append(f"eval {stored.id} v{stored.version} names task {entry.task_id}{at}, which isn't in the " + "store") continue if (task.id, task.version) in seen: - importer.duplicates.append(f"task {task.id} v{task.version}") + duplicates.append(f"task {task.id} v{task.version}") continue seen.add((task.id, task.version)) tasks.append(task) - importer.reserve_aliases(tasks) + importer = _importing(tasks, Path(dest), include_scopes, progress, image_refs) + importer.problems += problems + importer.duplicates += duplicates names = [importer.add_task(task) for task in tasks] if names: importer.add_eval(stored, names) @@ -165,6 +178,10 @@ def ref(self, entity: Any) -> str | dict: if (bundle := self.importer._installed_for(kind, entity.id)) is not None: self.importer.floating.append(Floating(self.folder, kind, entity.id, _from_bundle(bundle))) return entity.id + if (bundle := self.importer._registered_from(kind, entity)) is not None: + self.importer.floated.setdefault(kind, set()).add(fold(entity.id)) + self.importer.floating.append(Floating(self.folder, kind, entity.id, _registered_from_bundle(bundle))) + return entity.id local = self.importer.local(kind, entity, self.folder) return local if local is not None else {kind.value: entity.id, "version": entity.version} @@ -221,7 +238,8 @@ def __init__(self, dest: Path | None, include_scopes: Collection[FieldScope] = ( self.providers = Providers() self.installed: tuple[InstalledBundle, ...] | None = None self.aliases: dict[str, InstalledAliases] = {} # the installed bundles' aliases, by aliases.toml table - self.reserved: dict[EntityKind, set[str]] = {} # each kind's installed aliases a step names, folded + self.reserved: dict[EntityKind, set[str]] = {} # each kind's names no folder takes, folded (_importing) + self.floated: dict[EntityKind, set[str]] = {} # each kind's installed aliases written floating, folded if self.dest is not None: _check_destination(self.dest) @@ -470,6 +488,7 @@ def _ref(self, site: RefSite, where: str, pins: Mapping[str, set[int]]) -> None: self.problems.append(f"{where}: {entity}") return if (bundle := self._registered_from(kind, entity)) is not None: + self.floated.setdefault(kind, set()).add(fold(entity_id)) self._from_installed(site, where, bundle, _registered_from_bundle(bundle)) return if isinstance(site.key, str) and (why := _not_taken(site, entity)) is not None: @@ -509,6 +528,9 @@ def _folder(self, kind: EntityKind, entity: Any) -> tuple[BundleKind, str]: if fold(name) in self.taken[BundleKind.ARTIFACT.value]: raise Unwritable(f"a skill's folder takes its skill's name, {name!r}, which another artifact in the " "folder already has") + if fold(name) in self.reserved.get(EntityKind.ARTIFACT, set()): + raise Unwritable(f"a skill's folder takes its skill's name, {name!r}, which the export names as an " + "installed alias of an artifact, so the folder would take the alias's place in a run") self.taken[BundleKind.ARTIFACT.value].add(fold(name)) return BundleKind.SKILL, name bundle_kind = _FOLDERS[kind] @@ -565,17 +587,6 @@ def _registered_from(self, kind: EntityKind, entity: Any) -> InstalledBundle | N bundle=f"installed:{claim.bundle.id_root.removeprefix(LOCAL_PREFIX)}")) return claim.bundle if registered is not None else None - def reserve_aliases(self, tasks: Collection[Task]) -> None: - """Keep each installed alias a step of ``tasks`` names out of the folders' names, before any is given: such a - step may name it floating, and a folder of that name would capture it in a run. Reads no store.""" - registry = get_task_step_registry() - for task in tasks: - for step in task.to_dict()["steps"]: - for site in ref_sites(registry[step["type"]].entity_refs or (), step): - table = _ALIAS_TABLES.get(site.ref.kind) - if isinstance(site.value, str) and table is not None and self._aliases(table).target(site.value): - self.reserved.setdefault(site.ref.kind, set()).add(fold(site.value)) - def _aliases(self, table: str) -> InstalledAliases: """The aliases the installed bundles give in ``table``.""" if self.installed is None: @@ -615,8 +626,8 @@ def _default_agents(self, step: dict, refs: tuple[EntityRef, ...], where: str) - "default")) def _name(self, kind: str, entity_id: str) -> str: - """A name in the ``kind`` folder: the id's last segment made valid, numbered when taken, caselessly, or when a - step of the export names it as an installed alias (``reserve_aliases``).""" + """A name in the ``kind`` folder: the id's last segment made valid, numbered when taken, caselessly, or when the + export names it as an installed alias, floating (``_importing``).""" names = _names_of(BundleKind(kind)) aliases = self.reserved.get(_FOLDER_KINDS[kind], set()) if kind in _FOLDER_KINDS else set() base = _valid_name(entity_id.rsplit("/", 1)[-1]) or kind.removesuffix("s") diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index 3cdb043df..b46903ddc 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -13,20 +13,23 @@ from agent_env.a2a_agent import A2AAgent from agent_env.artifact import FileArtifact +from agent_env.artifact.artifacts.skill import SkillArtifact from agent_env.bundle import BundleError, parse_bundle from agent_env.bundle import importer as importer_module -from agent_env.bundle.importer import ImportedEval, import_eval, import_task +from agent_env.bundle.importer import ImportedEval, import_eval, import_task, task_images from agent_env.bundle.installed import InstalledBundle from agent_env.bundle.materialize import materialize from agent_env.bundle import register as register_module from agent_env.bundle.plan import plan_bundle -from agent_env.bundle.register import register_bundle +from agent_env.bundle.register import REGISTRATIONS, register_bundle from agent_env.bundle.requires import Provider from agent_env.bundle.resolve import resolve_bundle from agent_env.cli import cli from agent_env.config import get_config +from agent_env.env import store as env_store_module from agent_env.config.runtime import _DEFAULT_A2A_AGENT_ID as DEFAULT_AGENT, Config from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.env.envs.multi_env import MultiEnv from agent_env.eval import Eval from agent_env.eval.eval import EvalTask from agent_env.entity.scopes import FieldScope @@ -786,8 +789,8 @@ def test_an_agent_stored_under_an_installed_alias_by_anything_else_is_written_in assert "claude-code" not in [ref.id for ref in imported.floating] assert [(entity.folder, entity.id, entity.version) for entity in imported.written] == [ - ("agents/claude-code-2", "claude-code", 2 if registered_first else 1)] - assert _steps(imported.root, "own")[0]["a2a_agent_id"] == "claude-code-2" + ("agents/claude-code", "claude-code", 2 if registered_first else 1)] + assert _steps(imported.root, "own")[0]["a2a_agent_id"] == "claude-code" def test_a_folder_never_takes_an_alias_a_step_floats_so_each_version_keeps_its_own(registry, tmp_path): @@ -814,3 +817,76 @@ def unreachable(*args, **kwargs): imported = import_task("triage", store / "triage", include_scopes=[FieldScope.USER]) assert [entity.folder for entity in imported.written] == ["envs/crm-env", "artifacts/crm-data", "agents/solver"] + + +def test_an_env_another_names_by_its_registered_alias_floats_and_no_folder_takes_the_alias(store, tmp_path, + monkeypatch): + monkeypatch.setattr(env_store_module, "shared_store", lambda store: True) + mail = MCPServerEnv.put(id="mail", docker_image_artifact=context_image(tmp_path, "mail-image"), + environment_name="mail") + get_config().get_document_store().insert(REGISTRATIONS, { + "store": "env", "id": "mail", "version": mail.version, "status": "done", + "bundle": "installed:agentenv-framework/mail-mcp"}) + MCPServerEnv.put(id="team/mail", docker_image_artifact=context_image(tmp_path, "team-mail-image"), + environment_name="team-mail") + MultiEnv.put(id="office", mcp_server_envs=[mail]) + _task("both", [{"id": "team", "type": "deploy_env", "env_id": "team/mail"}, + {"id": "office", "type": "deploy_env", "env_id": "office"}]) + + imported = import_task("both", tmp_path / "imported") + + assert sorted(entity.folder for entity in imported.written) == ["envs/mail-2", "envs/office"] + assert [ref.where for ref in imported.floating if ref.id == "mail"] == ["envs/office"] + assert tomllib.loads((imported.root / "envs/office/env.toml").read_text())["mcp_server_envs"] == ["mail"] + assert _steps(imported.root, "both")[0]["env_id"] == "mail-2" + + +def _registered(kind, id, version, bundle="installed:agentenv-framework/agents"): + get_config().get_document_store().insert(REGISTRATIONS, {"store": kind, "id": id, "version": version, + "status": "done", "bundle": bundle}) + + +@pytest.mark.parametrize("bundle,kind,version", [("installed:someone/else", "agent", None), (None, "artifact", None), + (None, "agent", 7)]) +def test_a_ledger_row_of_another_bundle_kind_or_version_floats_nothing(registry, tmp_path, bundle, kind, version): + agent = A2AAgent.put(id="claude-code", docker_image_artifact=context_image(tmp_path, "own-image")) + _registered(kind, "claude-code", version or agent.version, bundle or "installed:agentenv-framework/agents") + _task("own", AGENT_NAMED_BY_ALIAS) + + imported = import_task("own", tmp_path / "imported", include_scopes=[FieldScope.USER]) + + assert "claude-code" not in [ref.id for ref in imported.floating] + assert [entity.folder for entity in imported.written] == ["agents/claude-code"] + + +@pytest.mark.parametrize("order", [("own", "registered"), ("registered", "own")]) +def test_an_evals_tasks_keep_a_version_off_the_alias_another_floats_whatever_their_order(registry, tmp_path, order): + registry("triage", AGENT_NAMED_BY_ALIAS) + A2AAgent.put(id="claude-code", docker_image_artifact=context_image(tmp_path, "own-image")) + _task("own", [{**AGENT_NAMED_BY_ALIAS[0], "a2a_agent_version": 2}]) + _task("registered", [{**AGENT_NAMED_BY_ALIAS[0], "a2a_agent_version": 1}]) + Eval.put(id="both", tasks=[EvalTask(task_id=name) for name in order]) + + imported = import_eval("both", tmp_path / "imported", include_scopes=[FieldScope.USER]) + + assert [(entity.folder, entity.version) for entity in imported.written] == [("agents/claude-code-2", 2)] + assert [_steps(imported.root, name)[0]["a2a_agent_id"] for name in ("own", "registered")] == [ + "claude-code-2", "claude-code"] + + +def test_a_skill_named_like_an_artifact_alias_the_export_floats_fails_the_import(store, tmp_path): + (tmp_path / "base.txt").write_text("base") + base = FileArtifact.put("windows-11", description="base", file_path=str(tmp_path / "base.txt")) + _registered("artifact", "windows-11", base.version, "installed:agentenv-framework/computers") + SkillArtifact.put("team/windows-11", skill_dir=layout(tmp_path / "skill", { + "SKILL.md": "---\nname: windows-11\ndescription: how to use it\n---\nUse it.\n"})) + _task("t", [{"id": "env", "type": "deploy_env", "env_id": "crm-env"}, + {"id": "load", "type": "load_artifact", "env_id": "crm-env", "artifacts": [{"id": "windows-11"}]}, + {"id": "skills", "type": "add_skills", "agent_name": "default-agent", + "skills": [{"skill_artifact_id": "team/windows-11"}]}]) + + for imports in (lambda: import_task("t", tmp_path / "imported"), lambda: task_images("t")): + with pytest.raises(BundleError) as refused: + imports() + assert ("a skill's folder takes its skill's name, 'windows-11', which the export names as an installed alias " + "of an artifact") in refused.value.problems[0] diff --git a/tst/unit/bundle/register_test.py b/tst/unit/bundle/register_test.py index 77f3996ba..216a0240f 100644 --- a/tst/unit/bundle/register_test.py +++ b/tst/unit/bundle/register_test.py @@ -24,10 +24,15 @@ from agent_env.env.envs.computer.utils.composed import composition from agent_env.env.published_images import COMPUTER_IMAGES from agent_env.bundle.register import REGISTRATIONS, _remote_name, register_bundle +from agent_env.bundle.importer import import_task +from agent_env.bundle.installed import installed_bundles +from agent_env.artifact.store import reset_artifact_store +from agent_env.config import configure +from agent_env.store.routing import namespace_routing from agent_env.cli import cli from agent_env.config import get_config from agent_env.config.runtime import Config -from agent_env.entity.refs import EntityRef, RefRole +from agent_env.entity.refs import EntityKind, EntityRef, RefRole from agent_env.env.env import Env from agent_env.eval import Eval from agent_env.bundle.parse import BundleKind, parse_bundle @@ -593,6 +598,33 @@ def compose(env, pairs=None, composing=None): assert result.exit_code == 1 and "has no build a deploy can boot (container/amd64 failed), so" in result.output +def test_a_desktop_on_the_shipped_base_and_driver_exports_naming_them_by_their_aliases(home, composed, monkeypatch): + """As a hub's computer task exports: the base and driver it registered from the installed bundles aren't copied, and + a fresh store plans the export from the installed bundles.""" + def compose(env, pairs=None, composing=None): + env.update_metadata({**env.metadata, "composed": {"container": { + "amd64": {"status": "ready", "image": "desktop", "composer": composition()}}}}) + + register_bundle(_desktop(home, monkeypatch, compose)) + imported = import_task("desk-t", home / "imported") + + assert sorted(entity.folder for entity in imported.written) == ["envs/desk-desktop"] + desktop = (imported.root / "envs/desk-desktop/env.toml").read_text() + assert 'computer = "ubuntu-24.04"' in desktop and 'driver = "reference"' in desktop + assert sorted((ref.where, ref.id) for ref in imported.floating) == [ + ("envs/desk-desktop", "reference"), ("envs/desk-desktop", "ubuntu-24.04")] + + monkeypatch.setenv("XDG_STATE_HOME", str(home / "elsewhere")) + configure() + reset_artifact_store() + with namespace_routing(): + plan = plan_bundle(resolve_bundle(parse_bundle(imported.root)), installed=installed_bundles) + assert {alias: use.target_id for alias, use in plan.alias_uses[EntityKind.ARTIFACT].items()} == { + "ubuntu-24.04": "@local/agentenv-framework/computers/ubuntu-24.04"} + assert {alias: use.target_id for alias, use in plan.alias_uses[EntityKind.ENV].items()} == { + "reference": "@local/agentenv-framework/drivers/reference"} + + def test_a_write_that_fails_leaves_no_pending_row(home, monkeypatch): def failing(*args, **kwargs): raise RuntimeError("the store went away")