Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions src/agent_env/env/envs/computer/utils/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@
#: How long a builder VM may run, natively and when it emulates the arch, which runs some 10-20 times slower.
VM_TIMEOUT_S, EMULATED_VM_TIMEOUT_S = 7200, 21600
_DOWNLOAD_TIMEOUT_S = 600
#: How often a download that fails in transit is tried again, and how long apart: as the builder VM's helper has curl.
_DOWNLOAD_RETRIES, _DOWNLOAD_RETRY_S = 5, 1.0
_FETCHES_IN_FLIGHT = 8
_CHUNK = 1 << 20

Expand Down Expand Up @@ -315,17 +317,23 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict:
return json.loads((out / "result.json").read_text())

def download(self, url: str, sha256: str, path: str) -> None:
target, hashed = self.work / path, hashlib.sha256()
target = self.work / path
target.parent.mkdir(parents=True, exist_ok=True)
try:
with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response:
response.raise_for_status()
with target.open("wb") as file:
for chunk in response.iter_bytes(_CHUNK):
hashed.update(chunk)
file.write(chunk)
except httpx.HTTPError as e:
raise ResolveError(f"downloading {url} failed: {e}") from None
for attempt in range(_DOWNLOAD_RETRIES + 1):
hashed = hashlib.sha256()
try:
with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response:
response.raise_for_status()
with target.open("wb") as file:
for chunk in response.iter_bytes(_CHUNK):
hashed.update(chunk)
file.write(chunk)
break
except httpx.HTTPError as e:
if attempt == _DOWNLOAD_RETRIES:
raise ResolveError(f"downloading {url} failed: {e}") from None
logger.warning(f"downloading {url} failed ({e}); trying again")
time.sleep(_DOWNLOAD_RETRY_S)
if hashed.hexdigest() != sha256:
raise ResolveError(f"{url} has the sha256 {hashed.hexdigest()}, not the {sha256} its download names")

Expand Down
9 changes: 9 additions & 0 deletions tst/integration/utils/docker/prune_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import shutil
import subprocess
import tarfile
import time
import uuid
from pathlib import Path

Expand All @@ -16,6 +17,8 @@
pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow]

BASE = "mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf"
#: How long to wait before pulling the base again, after each failed pull.
PULL_WAITS_S = (10, 20)
DOCKERFILE = f"""FROM {BASE} AS build
COPY tools/gen.sh /gen.sh
RUN sh /gen.sh > /built.txt
Expand Down Expand Up @@ -93,8 +96,14 @@ def _image_files(tag: str) -> dict[str, tuple[int, str]]:

@pytest.fixture
def docker():
"""The tags the test builds, removed after it; its base pulled first, tried again after a failed pull, so the build
takes it from here rather than ask the registry for it again mid-build, which failed a build on main."""
if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0:
pytest.skip(missing_capability_reason("docker_daemon"))
for wait in (*PULL_WAITS_S, None):
if subprocess.run(["docker", "pull", "--quiet", BASE], capture_output=True).returncode == 0 or wait is None:
break
time.sleep(wait)
tags = []
yield tags
for tag in tags:
Expand Down
53 changes: 53 additions & 0 deletions tst/unit/env/envs/test_computer_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says
as it goes. Docker, the registry and the VMs are faked."""

import contextlib
import hashlib
import io
import json
Expand All @@ -17,6 +18,7 @@
from pathlib import Path
from types import SimpleNamespace

import httpx
import pytest
from click.testing import CliRunner

Expand Down Expand Up @@ -230,6 +232,57 @@ def _resolving(tmp_path, monkeypatch, docker):
image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64")


def test_a_download_its_connection_drops_is_tried_again_and_one_that_never_comes_says_why(tmp_path, monkeypatch):
monkeypatch.setattr(builder, "docker_login", lambda auth: None)
local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace(
image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64")
attempts, waits = [], []
monkeypatch.setattr(builder.time, "sleep", waits.append)

@contextlib.contextmanager
def stream(method, url, **kwargs):
attempts.append(url)
if url.endswith("/down") or len(attempts) < 3:
raise httpx.ConnectError("connection reset by peer")
yield httpx.Response(200, content=b"data", request=httpx.Request(method, url))
Comment thread
earakely-scale marked this conversation as resolved.
Resolved

monkeypatch.setattr(builder.httpx, "stream", stream)
local.download("https://files.test/flaky", hashlib.sha256(b"data").hexdigest(), "fetched/flaky")
fetched, tries, waited = (tmp_path / "fetched" / "flaky").read_bytes(), len(attempts), list(waits)
attempts.clear()
with pytest.raises(builder.ResolveError, match="downloading https://files.test/down failed: connection reset"):
local.download("https://files.test/down", "0" * 64, "fetched/down")

assert (fetched, tries, waited) == (b"data", 3, [1.0, 1.0])
assert len(attempts) == builder._DOWNLOAD_RETRIES + 1


def test_a_download_dropped_mid_read_is_written_afresh_by_its_next_attempt(tmp_path, monkeypatch):
monkeypatch.setattr(builder, "docker_login", lambda auth: None)
local = builder.LocalBuilder("LocalSandboxProvider", tmp_path, SimpleNamespace(
image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/amd64")
monkeypatch.setattr(builder.time, "sleep", lambda seconds: None)
attempts = []

class Dropped:
def raise_for_status(self):
pass

def iter_bytes(self, size):
yield b"the first bytes of a body whose connection then drops"
raise httpx.ReadError("connection dropped mid-read")

@contextlib.contextmanager
def stream(method, url, **kwargs):
attempts.append(url)
yield Dropped() if len(attempts) == 1 else httpx.Response(200, content=b"ok", request=httpx.Request(method, url))

monkeypatch.setattr(builder.httpx, "stream", stream)
local.download("https://files.test/dropped", hashlib.sha256(b"ok").hexdigest(), "fetched/dropped")

assert (tmp_path / "fetched" / "dropped").read_bytes() == b"ok" and len(attempts) == 2


def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch):
listed = json.dumps([{"name": "mss", "version": "10.2.0"},
{"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode()
Expand Down
Loading