diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 9162a6b84..97d642f61 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -71,6 +71,8 @@ #: How long a builder VM may run, natively and when it emulates the arch, which runs some 10-20 times slower. VM_TIMEOUT_S, EMULATED_VM_TIMEOUT_S = 7200, 21600 _DOWNLOAD_TIMEOUT_S = 600 +#: How often a download that fails in transit is tried again, and how long apart: as the builder VM's helper has curl. +_DOWNLOAD_RETRIES, _DOWNLOAD_RETRY_S = 5, 1.0 _FETCHES_IN_FLIGHT = 8 _CHUNK = 1 << 20 @@ -315,17 +317,23 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: return json.loads((out / "result.json").read_text()) def download(self, url: str, sha256: str, path: str) -> None: - target, hashed = self.work / path, hashlib.sha256() + target = self.work / path target.parent.mkdir(parents=True, exist_ok=True) - try: - with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: - response.raise_for_status() - with target.open("wb") as file: - for chunk in response.iter_bytes(_CHUNK): - hashed.update(chunk) - file.write(chunk) - except httpx.HTTPError as e: - raise ResolveError(f"downloading {url} failed: {e}") from None + for attempt in range(_DOWNLOAD_RETRIES + 1): + hashed = hashlib.sha256() + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: + response.raise_for_status() + with target.open("wb") as file: + for chunk in response.iter_bytes(_CHUNK): + hashed.update(chunk) + file.write(chunk) + break + except httpx.HTTPError as e: + if attempt == _DOWNLOAD_RETRIES: + raise ResolveError(f"downloading {url} failed: {e}") from None + logger.warning(f"downloading {url} failed ({e}); trying again") + time.sleep(_DOWNLOAD_RETRY_S) if hashed.hexdigest() != sha256: raise ResolveError(f"{url} has the sha256 {hashed.hexdigest()}, not the {sha256} its download names") diff --git a/tst/integration/utils/docker/prune_test.py b/tst/integration/utils/docker/prune_test.py index 862aa6e35..5bb746eac 100644 --- a/tst/integration/utils/docker/prune_test.py +++ b/tst/integration/utils/docker/prune_test.py @@ -5,6 +5,7 @@ import shutil import subprocess import tarfile +import time import uuid from pathlib import Path @@ -16,6 +17,8 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] BASE = "mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf" +#: How long to wait before pulling the base again, after each failed pull. +PULL_WAITS_S = (10, 20) DOCKERFILE = f"""FROM {BASE} AS build COPY tools/gen.sh /gen.sh RUN sh /gen.sh > /built.txt @@ -93,8 +96,14 @@ def _image_files(tag: str) -> dict[str, tuple[int, str]]: @pytest.fixture def docker(): + """The tags the test builds, removed after it; its base pulled first, tried again after a failed pull, so the build + takes it from here rather than ask the registry for it again mid-build, which failed a build on main.""" if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0: pytest.skip(missing_capability_reason("docker_daemon")) + for wait in (*PULL_WAITS_S, None): + if subprocess.run(["docker", "pull", "--quiet", BASE], capture_output=True).returncode == 0 or wait is None: + break + time.sleep(wait) tags = [] yield tags for tag in tags: diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index f8f3475c0..5ed251508 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -3,6 +3,7 @@ or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says as it goes. Docker, the registry and the VMs are faked.""" +import contextlib import hashlib import io import json @@ -17,6 +18,7 @@ from pathlib import Path from types import SimpleNamespace +import httpx import pytest from click.testing import CliRunner @@ -230,6 +232,57 @@ def _resolving(tmp_path, monkeypatch, docker): image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") +def test_a_download_its_connection_drops_is_tried_again_and_one_that_never_comes_says_why(tmp_path, monkeypatch): + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64") + attempts, waits = [], [] + monkeypatch.setattr(builder.time, "sleep", waits.append) + + @contextlib.contextmanager + def stream(method, url, **kwargs): + attempts.append(url) + if url.endswith("/down") or len(attempts) < 3: + raise httpx.ConnectError("connection reset by peer") + yield httpx.Response(200, content=b"data", request=httpx.Request(method, url)) + + monkeypatch.setattr(builder.httpx, "stream", stream) + local.download("https://files.test/flaky", hashlib.sha256(b"data").hexdigest(), "fetched/flaky") + fetched, tries, waited = (tmp_path / "fetched" / "flaky").read_bytes(), len(attempts), list(waits) + attempts.clear() + with pytest.raises(builder.ResolveError, match="downloading https://files.test/down failed: connection reset"): + local.download("https://files.test/down", "0" * 64, "fetched/down") + + assert (fetched, tries, waited) == (b"data", 3, [1.0, 1.0]) + assert len(attempts) == builder._DOWNLOAD_RETRIES + 1 + + +def test_a_download_dropped_mid_read_is_written_afresh_by_its_next_attempt(tmp_path, monkeypatch): + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64") + monkeypatch.setattr(builder.time, "sleep", lambda seconds: None) + attempts = [] + + class Dropped: + def raise_for_status(self): + pass + + def iter_bytes(self, size): + yield b"the first bytes of a body whose connection then drops" + raise httpx.ReadError("connection dropped mid-read") + + @contextlib.contextmanager + def stream(method, url, **kwargs): + attempts.append(url) + yield Dropped() if len(attempts) == 1 else httpx.Response(200, content=b"ok", request=httpx.Request(method, url)) + + monkeypatch.setattr(builder.httpx, "stream", stream) + local.download("https://files.test/dropped", hashlib.sha256(b"ok").hexdigest(), "fetched/dropped") + + assert (tmp_path / "fetched" / "dropped").read_bytes() == b"ok" and len(attempts) == 2 + + def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): listed = json.dumps([{"name": "mss", "version": "10.2.0"}, {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode()