From 68255778a6613ebdcb21e91002bbf787cafff93c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 17 Nov 2026 11:19:35 -0800 Subject: [PATCH 1/2] fix(sandbox): a VM logs in to the image store's registry before pulling an image by its digest A bundle's image, registered by its digest in a private image store such as ECR, failed to pull on a VM sandbox ("no basic auth credentials") and was built from its build context on every deploy, since only the pull by name logged in. The pull by digest now logs in to each registry the store holds credentials for first, as the pull by name does; a registry it holds none for, GHCR's published images, is left alone. Co-Authored-By: Claude Opus 5.5 --- .../providers/sandbox_providers/sandbox.py | 23 ++++++++++++------- .../sandbox_providers/vm_sandbox_test.py | 21 +++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index 0986f795c..a3c2209a9 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -360,8 +360,10 @@ async def _pull_or_build(self, artifacts: list) -> None: raise RuntimeError(f"Couldn't pull {why}. {e}") from e async def pull_published(self, artifacts: list) -> list[tuple[Any, str]]: - """Pull each image by its digest, concurrently, and tag it with its ``image_name``, which its containers start - from; return each the VM couldn't pull, with why, for it to build from its build context instead.""" + """Pull each image by its digest, concurrently, after logging in to each registry the image store holds + credentials for, as ``pull_images`` does, and tag it with its ``image_name``, which its containers start from; + return each the VM couldn't pull, with why, for it to build from its build context instead.""" + await self._log_in([artifact.pinned_name for artifact in artifacts]) results = await asyncio.gather(*(_pull_published(self, artifact) for artifact in artifacts), return_exceptions=True) unpulled = [] @@ -437,12 +439,7 @@ async def pull_images(self, image_names: list[str]) -> None: the failure back; the command a cancelled exec started keeps running on the VM until it ends or the VM does. A network policy restricting egress isn't widened for them: it must allow the registries itself.""" image_names = list(dict.fromkeys(image_names)) - store = get_config().get_image_store() - one_per_registry = {registry_host_from_ref(image_name): image_name for image_name in reversed(image_names)} - for image_name in one_per_registry.values(): - auth = await asyncio.to_thread(store.auth, image_name) - if auth is not None: - await self.docker_login(auth) + await self._log_in(image_names) pulls = [asyncio.ensure_future(_pull(self, image_name)) for image_name in image_names] try: await asyncio.gather(*pulls) @@ -452,6 +449,16 @@ async def pull_images(self, image_names: list[str]) -> None: await asyncio.gather(*pulls, return_exceptions=True) raise + async def _log_in(self, refs: list[str]) -> None: + """``docker login`` to each registry of ``refs`` the image store holds credentials for, once per registry, + since minting a login can be a network round trip; a registry it holds none for, a public one, is left alone.""" + store = get_config().get_image_store() + one_per_registry = {registry_host_from_ref(ref): ref for ref in reversed(refs)} + for ref in one_per_registry.values(): + auth = await asyncio.to_thread(store.auth, ref) + if auth is not None: + await self.docker_login(auth) + @staticmethod async def _signed_image_urls(artifacts: list) -> list[str | None]: """Each image tarball's signed URL, or None where the store cannot sign one. Signed diff --git a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py index cc149110a..0568283c1 100644 --- a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py @@ -422,6 +422,27 @@ async def test_a_published_image_is_pulled_by_its_digest_and_tagged_with_the_nam assert sandbox.scripts == [f"docker pull {PUBLIC}", f"docker pull {_PINNED}", f"docker tag {_PINNED} {_GATEWAY}"] +@pytest.mark.asyncio +async def test_an_image_in_a_registry_the_store_holds_is_pulled_by_its_digest_after_logging_in_to_it(registry): + """A bundle's image, registered by its digest in a private image store, pulls rather than being rebuilt on the VM; + the published one beside it, on a registry the store holds nothing for, logs in nowhere.""" + sandbox = _StdinVm() + private = "registry.example/team/items:src-" + "1" * 64 + pinned = f"{private}@{_GATEWAY_DIGEST}" + image = DockerImageArtifact(id="items", version=1, description="d", image_name=private, image_digest=_GATEWAY_DIGEST, + build_context_object_url="s3://bucket/items/build-context.tar.gz", + dockerfile_path="Dockerfile", source_digest="sha256:" + "1" * 64) + + await sandbox.load_docker_images([image, _published()]) + + assert sandbox.scripts[:3] == [ + "mktemp -d", + "docker login --username user --password-stdin registry.example < /tmp/tmp.Xq3/password", + "rm -rf /tmp/tmp.Xq3", + ] + assert sorted(sandbox.scripts[3:]) == sorted([f"docker pull {pinned}", f"docker tag {pinned} {private}", + f"docker pull {_PINNED}", f"docker tag {_PINNED} {_GATEWAY}"]) + @pytest.mark.asyncio async def test_a_published_image_the_vm_cant_pull_is_built_from_its_build_context_under_its_name(signing_store, no_waits): From 421097e61059683f3aeba12fcebd27117c36e539 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 17 Nov 2026 11:45:30 -0800 Subject: [PATCH 2/2] fix(sandbox): a registry a VM can't log in to falls back to building its images, as a failed pull by digest does Co-Authored-By: Claude Opus 5.5 --- .../providers/sandbox_providers/sandbox.py | 33 ++++++++++---- .../sandbox_providers/vm_sandbox_test.py | 45 ++++++++++++++++--- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index a3c2209a9..6b46f29e1 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -362,11 +362,17 @@ async def _pull_or_build(self, artifacts: list) -> None: async def pull_published(self, artifacts: list) -> list[tuple[Any, str]]: """Pull each image by its digest, concurrently, after logging in to each registry the image store holds credentials for, as ``pull_images`` does, and tag it with its ``image_name``, which its containers start from; - return each the VM couldn't pull, with why, for it to build from its build context instead.""" - await self._log_in([artifact.pinned_name for artifact in artifacts]) + return each the VM couldn't pull, with why, for it to build from its build context instead. An image of a + registry the VM couldn't log in to isn't pulled, and is returned with why.""" + refused = await self._log_in([artifact.pinned_name for artifact in artifacts], tolerate=True) + unpulled = [(artifact, refused[host]) for artifact in artifacts + if (host := registry_host_from_ref(artifact.pinned_name)) in refused] + for artifact, reason in unpulled: + logger.warning("Couldn't pull %s on %s, so it's built from its build context: %s", artifact.pinned_name, + self.sandbox_id, reason) + artifacts = [artifact for artifact in artifacts if registry_host_from_ref(artifact.pinned_name) not in refused] results = await asyncio.gather(*(_pull_published(self, artifact) for artifact in artifacts), return_exceptions=True) - unpulled = [] for artifact, result in zip(artifacts, results, strict=True): if isinstance(result, RuntimeError): reason = _last_output_line(str(result)) @@ -449,15 +455,24 @@ async def pull_images(self, image_names: list[str]) -> None: await asyncio.gather(*pulls, return_exceptions=True) raise - async def _log_in(self, refs: list[str]) -> None: + async def _log_in(self, refs: list[str], *, tolerate: bool = False) -> dict[str, str]: """``docker login`` to each registry of ``refs`` the image store holds credentials for, once per registry, - since minting a login can be a network round trip; a registry it holds none for, a public one, is left alone.""" + since minting a login can be a network round trip; a registry it holds none for, a public one, is left alone. + With ``tolerate``, each registry whose login failed is returned with why rather than raised, for a caller that + has another way to its images.""" store = get_config().get_image_store() one_per_registry = {registry_host_from_ref(ref): ref for ref in reversed(refs)} - for ref in one_per_registry.values(): - auth = await asyncio.to_thread(store.auth, ref) - if auth is not None: - await self.docker_login(auth) + refused = {} + for host, ref in one_per_registry.items(): + try: + auth = await asyncio.to_thread(store.auth, ref) + if auth is not None: + await self.docker_login(auth) + except Exception as e: + if not tolerate: + raise + refused[host] = f"logging in to {host} failed: {e}" + return refused @staticmethod async def _signed_image_urls(artifacts: list) -> list[str | None]: diff --git a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py index 0568283c1..c4c91a977 100644 --- a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py @@ -391,6 +391,17 @@ def _published() -> DockerImageArtifact: source_digest="sha256:" + "6" * 64) +_PRIVATE = "registry.example/team/items:src-" + "1" * 64 + + +def _private() -> DockerImageArtifact: + """A bundle's image as ``bundle register`` writes it: pushed to the store's own registry, by its digest, with its + build context beside it.""" + return DockerImageArtifact(id="items", version=1, description="d", image_name=_PRIVATE, image_digest=_GATEWAY_DIGEST, + build_context_object_url="s3://bucket/items/build-context.tar.gz", + dockerfile_path="Dockerfile", source_digest="sha256:" + "1" * 64) + + class _PinnedPullFailsVm(_RecordingVmSandbox): """Fails the first ``fails`` pulls by digest with ``error``, and every build when ``build_fails``.""" @@ -427,22 +438,42 @@ async def test_an_image_in_a_registry_the_store_holds_is_pulled_by_its_digest_af """A bundle's image, registered by its digest in a private image store, pulls rather than being rebuilt on the VM; the published one beside it, on a registry the store holds nothing for, logs in nowhere.""" sandbox = _StdinVm() - private = "registry.example/team/items:src-" + "1" * 64 - pinned = f"{private}@{_GATEWAY_DIGEST}" - image = DockerImageArtifact(id="items", version=1, description="d", image_name=private, image_digest=_GATEWAY_DIGEST, - build_context_object_url="s3://bucket/items/build-context.tar.gz", - dockerfile_path="Dockerfile", source_digest="sha256:" + "1" * 64) + pinned = f"{_PRIVATE}@{_GATEWAY_DIGEST}" - await sandbox.load_docker_images([image, _published()]) + await sandbox.load_docker_images([_private(), _published()]) assert sandbox.scripts[:3] == [ "mktemp -d", "docker login --username user --password-stdin registry.example < /tmp/tmp.Xq3/password", "rm -rf /tmp/tmp.Xq3", ] - assert sorted(sandbox.scripts[3:]) == sorted([f"docker pull {pinned}", f"docker tag {pinned} {private}", + assert sorted(sandbox.scripts[3:]) == sorted([f"docker pull {pinned}", f"docker tag {pinned} {_PRIVATE}", f"docker pull {_PINNED}", f"docker tag {_PINNED} {_GATEWAY}"]) + +class _RefusingRegistry(_Registry): + """Holds credentials for registry.example, which can't be minted.""" + + def auth(self, ref): + if ref.startswith("registry.example/"): + raise RuntimeError("the registry's credentials couldn't be minted") + return None + + +@pytest.mark.asyncio +async def test_an_image_whose_registry_cant_be_logged_in_to_is_built_from_its_build_context(signing_store): + """A pull by digest that fails falls back to building the image, so a login that fails does too, rather than + failing the deploy; the published image beside it is pulled as before.""" + get_config().set_image_store(_RefusingRegistry()) + sandbox = _RecordingVmSandbox() + + await sandbox.load_docker_images([_private(), _published()]) + + assert not any(script.startswith(f"docker pull {_PRIVATE}") for script in sandbox.scripts) + assert any(f"docker build --platform linux/amd64 -f Dockerfile -t {_PRIVATE} ." in script + for script in sandbox.scripts) + assert f"docker pull {_PINNED}" in sandbox.scripts and f"docker tag {_PINNED} {_GATEWAY}" in sandbox.scripts + @pytest.mark.asyncio async def test_a_published_image_the_vm_cant_pull_is_built_from_its_build_context_under_its_name(signing_store, no_waits):