From e8692ccb1ae418093a3cf1c7b78f35d281fed48e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 00:36:44 -0800 Subject: [PATCH 1/3] fix(computers): a compose on this machine tries a download again when its connection fails A compose of the clock desktop failed on main on 11-18 (01:18 UTC run) because downloading Chrome for Testing's zip from storage.googleapis.com failed once. The builder VM's helper already has curl retry every download 5 times, 1 s apart; this machine's builder tried once. It now does as the helper does, writing each attempt afresh, and still checks the sha256 of what it fetched. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 28 ++++++++++++------- tst/unit/env/envs/test_computer_builder.py | 27 ++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 9162a6b84..97d642f61 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -71,6 +71,8 @@ #: How long a builder VM may run, natively and when it emulates the arch, which runs some 10-20 times slower. VM_TIMEOUT_S, EMULATED_VM_TIMEOUT_S = 7200, 21600 _DOWNLOAD_TIMEOUT_S = 600 +#: How often a download that fails in transit is tried again, and how long apart: as the builder VM's helper has curl. +_DOWNLOAD_RETRIES, _DOWNLOAD_RETRY_S = 5, 1.0 _FETCHES_IN_FLIGHT = 8 _CHUNK = 1 << 20 @@ -315,17 +317,23 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: return json.loads((out / "result.json").read_text()) def download(self, url: str, sha256: str, path: str) -> None: - target, hashed = self.work / path, hashlib.sha256() + target = self.work / path target.parent.mkdir(parents=True, exist_ok=True) - try: - with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: - response.raise_for_status() - with target.open("wb") as file: - for chunk in response.iter_bytes(_CHUNK): - hashed.update(chunk) - file.write(chunk) - except httpx.HTTPError as e: - raise ResolveError(f"downloading {url} failed: {e}") from None + for attempt in range(_DOWNLOAD_RETRIES + 1): + hashed = hashlib.sha256() + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: + response.raise_for_status() + with target.open("wb") as file: + for chunk in response.iter_bytes(_CHUNK): + hashed.update(chunk) + file.write(chunk) + break + except httpx.HTTPError as e: + if attempt == _DOWNLOAD_RETRIES: + raise ResolveError(f"downloading {url} failed: {e}") from None + logger.warning(f"downloading {url} failed ({e}); trying again") + time.sleep(_DOWNLOAD_RETRY_S) if hashed.hexdigest() != sha256: raise ResolveError(f"{url} has the sha256 {hashed.hexdigest()}, not the {sha256} its download names") diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index f8f3475c0..329e7e3a1 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -3,6 +3,7 @@ or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says as it goes. Docker, the registry and the VMs are faked.""" +import contextlib import hashlib import io import json @@ -17,6 +18,7 @@ from pathlib import Path from types import SimpleNamespace +import httpx import pytest from click.testing import CliRunner @@ -230,6 +232,31 @@ def _resolving(tmp_path, monkeypatch, docker): image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") +def test_a_download_its_connection_drops_is_tried_again_and_one_that_never_comes_says_why(tmp_path, monkeypatch): + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64") + attempts, waits = [], [] + monkeypatch.setattr(builder.time, "sleep", waits.append) + + @contextlib.contextmanager + def stream(method, url, **kwargs): + attempts.append(url) + if url.endswith("/down") or len(attempts) < 3: + raise httpx.ConnectError("connection reset by peer") + yield httpx.Response(200, content=b"data", request=httpx.Request(method, url)) + + monkeypatch.setattr(builder.httpx, "stream", stream) + local.download("https://files.test/flaky", hashlib.sha256(b"data").hexdigest(), "fetched/flaky") + fetched, tries, waited = (tmp_path / "fetched" / "flaky").read_bytes(), len(attempts), list(waits) + attempts.clear() + with pytest.raises(builder.ResolveError, match="downloading https://files.test/down failed: connection reset"): + local.download("https://files.test/down", "0" * 64, "fetched/down") + + assert (fetched, tries, waited) == (b"data", 3, [1.0, 1.0]) + assert len(attempts) == builder._DOWNLOAD_RETRIES + 1 + + def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): listed = json.dumps([{"name": "mss", "version": "10.2.0"}, {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode() From 0f19cbed9b0e91f5ec5a64fde8c3ebd3766ef9be Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 00:36:45 -0800 Subject: [PATCH 2/3] test(docker): the prune test pulls its base before its builds, trying again after a failed pull Its build failed on main on 11-18 (16:10 UTC run) when the registry didn't answer BuildKit's lookup of its digest-pinned base mid-build. The fixture now pulls the base first, trying again after 10 s and 20 s, so the build takes it from this machine. Co-Authored-By: Claude Opus 5.5 --- tst/integration/utils/docker/prune_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tst/integration/utils/docker/prune_test.py b/tst/integration/utils/docker/prune_test.py index 862aa6e35..5bb746eac 100644 --- a/tst/integration/utils/docker/prune_test.py +++ b/tst/integration/utils/docker/prune_test.py @@ -5,6 +5,7 @@ import shutil import subprocess import tarfile +import time import uuid from pathlib import Path @@ -16,6 +17,8 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] BASE = "mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf" +#: How long to wait before pulling the base again, after each failed pull. +PULL_WAITS_S = (10, 20) DOCKERFILE = f"""FROM {BASE} AS build COPY tools/gen.sh /gen.sh RUN sh /gen.sh > /built.txt @@ -93,8 +96,14 @@ def _image_files(tag: str) -> dict[str, tuple[int, str]]: @pytest.fixture def docker(): + """The tags the test builds, removed after it; its base pulled first, tried again after a failed pull, so the build + takes it from here rather than ask the registry for it again mid-build, which failed a build on main.""" if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0: pytest.skip(missing_capability_reason("docker_daemon")) + for wait in (*PULL_WAITS_S, None): + if subprocess.run(["docker", "pull", "--quiet", BASE], capture_output=True).returncode == 0 or wait is None: + break + time.sleep(wait) tags = [] yield tags for tag in tags: From 55a72e939eafdbcd6ec78c904c36d206d622fb82 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 01:02:02 -0800 Subject: [PATCH 3/3] test(computers): a download dropped mid-read is written afresh by its next attempt Co-Authored-By: Claude Opus 5.5 --- tst/unit/env/envs/test_computer_builder.py | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 329e7e3a1..5ed251508 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -257,6 +257,32 @@ def stream(method, url, **kwargs): assert len(attempts) == builder._DOWNLOAD_RETRIES + 1 +def test_a_download_dropped_mid_read_is_written_afresh_by_its_next_attempt(tmp_path, monkeypatch): + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64") + monkeypatch.setattr(builder.time, "sleep", lambda seconds: None) + attempts = [] + + class Dropped: + def raise_for_status(self): + pass + + def iter_bytes(self, size): + yield b"the first bytes of a body whose connection then drops" + raise httpx.ReadError("connection dropped mid-read") + + @contextlib.contextmanager + def stream(method, url, **kwargs): + attempts.append(url) + yield Dropped() if len(attempts) == 1 else httpx.Response(200, content=b"ok", request=httpx.Request(method, url)) + + monkeypatch.setattr(builder.httpx, "stream", stream) + local.download("https://files.test/dropped", hashlib.sha256(b"ok").hexdigest(), "fetched/dropped") + + assert (tmp_path / "fetched" / "dropped").read_bytes() == b"ok" and len(attempts) == 2 + + def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): listed = json.dumps([{"name": "mss", "version": "10.2.0"}, {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode()