Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 36 additions & 13 deletions src/agent_env/agents/agents/claude-code/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from __future__ import annotations

import asyncio
import errno
import json
import logging
import os
Expand Down Expand Up @@ -109,6 +110,8 @@
_ERROR_MESSAGE_LIMIT = 1000
# How much of a changing workspace file one read stages.
_STAGE_CHUNK_BYTES = 1024 * 1024
_STORED_GZIP_LEVEL = 0
_SMALLEST_GZIP_LEVEL = 9

# install/v1: the folder, unpacked on the sandbox's host, installs itself (install.sh); install_agent fills each {param}
# in, quoted. The endpoint and CA go in the environment, not on the agent's command line.
Expand Down Expand Up @@ -216,7 +219,7 @@ def _add_workspace_entry(tar: tarfile.TarFile, path: Path, arcname: str, *, stag
return True


def _write_workspace_archive(archive_path: Path, *, stage_sources: bool = False) -> tuple[int, int]:
def _write_workspace_archive(archive_path: Path, *, compresslevel: int, stage_sources: bool = False) -> tuple[int, int]:
def add_tree(tar: tarfile.TarFile, source_root: Path, *, archive_prefix: str = "",
excluded_top_level: frozenset[str] = frozenset()) -> int:
count = 0
Expand All @@ -241,25 +244,43 @@ def add_tree(tar: tarfile.TarFile, source_root: Path, *, archive_prefix: str = "
count += 1
return count

with tarfile.open(archive_path, mode="w:gz", dereference=False) as tar:
with tarfile.open(archive_path, mode="w:gz", compresslevel=compresslevel, dereference=False) as tar:
# The CLI's state goes in as `.claude` from wherever it lives, so a `/app/.claude` isn't added twice.
file_count = add_tree(tar, WORKSPACE_DIR, excluded_top_level=frozenset({".claude"}))
if CLAUDE_STATE_DIR.is_dir():
file_count += add_tree(tar, CLAUDE_STATE_DIR, archive_prefix=".claude")
return file_count, archive_path.stat().st_size


def _build_workspace_archive(archive_path: Path) -> tuple[int, int]:
"""``/app`` and the CLI's state as a gzip tar on disk, out of the agent's heap; its file count and size. A file
that changes while it's read rebuilds the archive with each file staged first, so it can be skipped. Symlinked
directories aren't followed."""
def _build_workspace_archive(archive_path: Path, max_bytes: int | None = None) -> tuple[int, int]:
"""``/app`` and the CLI's state as a gzip tar on disk, out of the agent's heap; its file count and size. The tar
goes in uncompressed, since deflate on one core is the save's bottleneck and gains nothing on media; an
archive over ``max_bytes``, or one the disk has no room for, is rebuilt compressed, so a workspace that fit compressed
still fits. A file that changes while it's read rebuilds the archive with each file staged first, so it can be
skipped. Symlinked directories aren't followed."""
if not WORKSPACE_DIR.is_dir():
raise FileNotFoundError(f"workspace directory does not exist: {WORKSPACE_DIR}")

def write(compresslevel: int) -> tuple[int, int]:
try:
return _write_workspace_archive(archive_path, compresslevel=compresslevel)
except OSError as exc:
if exc.errno == errno.ENOSPC:
raise
logger.warning("Workspace changed while archiving; rebuilding with staged files")
return _write_workspace_archive(archive_path, compresslevel=compresslevel, stage_sources=True)

try:
return _write_workspace_archive(archive_path)
except OSError:
logger.warning("Workspace changed while archiving; rebuilding with staged files")
return _write_workspace_archive(archive_path, stage_sources=True)
file_count, size = write(_STORED_GZIP_LEVEL)
except OSError as exc:
if exc.errno != errno.ENOSPC:
raise
logger.warning("No disk space for the uncompressed workspace archive; compressing")
return write(_SMALLEST_GZIP_LEVEL)
if max_bytes is not None and size > max_bytes:
logger.info(f"Workspace archive is {size} bytes uncompressed, over the {max_bytes}-byte grant; compressing")
file_count, size = write(_SMALLEST_GZIP_LEVEL)
return file_count, size


def _read_complete_session_jsonl(path: Path) -> bytes:
Expand Down Expand Up @@ -584,11 +605,13 @@ async def save_snapshot(self, request: ObjectSnapshotSaveRequest) -> ObjectSnaps
os.close(jsonl_fd)
archive_path, snapshot_jsonl_path = Path(raw_archive_path), Path(raw_jsonl_path)
try:
file_count, bytes_written = await asyncio.to_thread(_build_workspace_archive, archive_path)
workspace = request.objects.workspace
file_count, bytes_written = await asyncio.to_thread(
_build_workspace_archive, archive_path, workspace.max_bytes if workspace is not None else None)
await asyncio.to_thread(snapshot_jsonl_path.write_bytes, jsonl_bytes)
uploaded = {"trajectory": _answer(await upload(request.objects.trajectory, snapshot_jsonl_path))}
if request.objects.workspace is not None:
uploaded["workspace"] = _answer(await upload(request.objects.workspace, archive_path))
if workspace is not None:
uploaded["workspace"] = _answer(await upload(workspace, archive_path))
except FileNotFoundError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
finally:
Expand Down
96 changes: 96 additions & 0 deletions tst/unit/agents/claude_code/test_workspace_snapshot.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
from __future__ import annotations

import asyncio
import errno
import hashlib
import io
import json
import os
import tarfile
from pathlib import Path

import pytest

from agentenv_protocol.transfers import Uploaded

from tst.unit.agents.claude_code.support import extension_methods, read_grant, write_grant
Expand Down Expand Up @@ -344,3 +347,96 @@ def test_a_saved_conversation_holds_no_key_an_api_error_echoed(monkeypatch, tmp_
saved = a2a_server._read_complete_session_jsonl(transcript).decode()

assert key not in saved and "[redacted]" in saved and saved.startswith(f"{ok}\n")


def test_the_workspace_archive_stores_bytes_as_they_are_in_a_gzip_stream_older_loaders_read(monkeypatch, tmp_path):
workspace, _ = _snapshot_roots(monkeypatch, tmp_path)
text = b"the same line of source code\n" * 40_000
text_size = len(text)
(workspace / "notes.txt").write_bytes(text)
archive = tmp_path / "workspace.tar.gz"

_, bytes_written = a2a_server._build_workspace_archive(archive)

assert bytes_written > text_size
with tarfile.open(archive, mode="r:gz") as tar:
assert tar.extractfile("notes.txt").read() == text


@pytest.mark.parametrize("grant_bytes", [1 << 20, 200_000], ids=["fits-uncompressed", "fits-only-compressed"])
def test_a_saved_workspace_fits_its_upload_grant(monkeypatch, tmp_path, agent, grant_bytes):
workspace, state = _snapshot_roots(monkeypatch, tmp_path)
text = b"the same line of source code\n" * 20_000
(workspace / "notes.txt").write_bytes(text)
session_dir = state / "projects" / "-app"
(session_dir / "session-1.jsonl").write_text(json.dumps({"type": "user", "sessionId": "session-1"}) + "\n")
monkeypatch.setattr(claude_bridge, "SESSION_JSONL_DIR", session_dir)
agent.set_session_ref_for_context("ctx-1", "session-1")
uploaded: dict[str, bytes] = {}

async def _upload(target, source):
body = source.read_bytes()
assert len(body) <= target.max_bytes
uploaded[target.write.url] = body
return Uploaded(size_bytes=len(body), sha256=hashlib.sha256(body).hexdigest())

monkeypatch.setattr(a2a_server, "upload", _upload)
request = a2a_server.ObjectSnapshotSaveRequest.model_validate({"context_id": "ctx-1", "objects": {
"trajectory": write_grant("snap/trajectory"), "workspace": write_grant("snap/workspace", max_bytes=grant_bytes)}})

asyncio.run(agent.save_snapshot(request))

with tarfile.open(fileobj=io.BytesIO(uploaded[request.objects.workspace.write.url]), mode="r:gz") as tar:
assert tar.extractfile("notes.txt").read() == text


def test_a_workspace_too_big_for_the_disk_uncompressed_is_saved_compressed(monkeypatch, tmp_path):
workspace, _ = _snapshot_roots(monkeypatch, tmp_path)
text = b"the same line of source code\n" * 20_000
(workspace / "notes.txt").write_bytes(text)
archive = tmp_path / "workspace.tar.gz"
disk_bytes = 200_000
write_archive = a2a_server._write_workspace_archive

def _write_to_a_small_disk(path, **kwargs):
file_count, size = write_archive(path, **kwargs)
if size > disk_bytes:
raise OSError(errno.ENOSPC, "No space left on device")
return file_count, size

monkeypatch.setattr(a2a_server, "_write_workspace_archive", _write_to_a_small_disk)

_, bytes_written = a2a_server._build_workspace_archive(archive)

assert bytes_written <= disk_bytes
with tarfile.open(archive, mode="r:gz") as tar:
assert tar.extractfile("notes.txt").read() == text


def _recompressed(archive: Path, level: int) -> Path:
with tarfile.open(archive, mode="r:gz") as source:
members = [(member, source.extractfile(member).read() if member.isfile() else None)
for member in source.getmembers()]
copy = archive.with_name(f"level-{level}.tar.gz")
with tarfile.open(copy, mode="w:gz", compresslevel=level) as tar:
for member, data in members:
tar.addfile(member, io.BytesIO(data) if data is not None else None)
return copy


@pytest.mark.parametrize("level", [None, 9], ids=["as-saved", "compressed-by-an-older-agent"])
def test_a_workspace_archive_restores_whether_or_not_its_gzip_stream_is_compressed(monkeypatch, tmp_path, level):
_snapshot_roots(monkeypatch, tmp_path / "source")
archive = tmp_path / "workspace.tar.gz"
a2a_server._build_workspace_archive(archive)
if level is not None:
archive = _recompressed(archive, level)
destination_workspace = tmp_path / "destination" / "app"
destination_state = tmp_path / "destination" / "home" / ".claude"
monkeypatch.setattr(a2a_server, "WORKSPACE_DIR", destination_workspace)
monkeypatch.setattr(a2a_server, "CLAUDE_STATE_DIR", destination_state)

a2a_server._restore_workspace_archive(archive)

assert (destination_workspace / "report" / "answer.txt").read_text() == "done"
assert (destination_state / "projects" / "-app" / "memory" / "MEMORY.md").read_text() == "remember this"
Loading