From 2ca03862148e5ba8d616356ee564ab4f96c86017 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 17:21:16 -0800 Subject: [PATCH 1/4] feat(a2a-agent): an atif/1 copy's media files wait on the agent's disk until the worker pulls them An agent that serves atif/1 spools each media file its copies name to its disk, one file per sha256, instead of holding the bytes in memory, with a PNG, JPEG, GIF or WebP image's width and height read from its header when the adapter gives none. The trajectory extension gains a media call, listed on such an agent's card: given a context and an exact-object write grant per media/. path, it uploads each file the context holds, 8 at a time, checking each upload against its name, and answers what it uploaded, the paths it holds no file for and the uploads that failed. An uploaded file leaves the disk, as do a context's files when it is evicted or restored over, and the rest when the server stops. atif.media_file and check_media take a part path only as media/. and only with the bytes that name, and the scrub never takes a media file's name in a grant's URL for its token. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 14 + .../agentenv_protocol/a2a_agent/__init__.py | 4 + .../agentenv_protocol/a2a_agent/_atif_copy.py | 5 +- .../a2a_agent/_media_spool.py | 74 +++++ .../agentenv_protocol/a2a_agent/_secrets.py | 6 +- .../a2a_agent/_trajectory_log.py | 8 +- .../agentenv_protocol/a2a_agent/extensions.py | 46 ++- .../agentenv_protocol/a2a_agent/framework.py | 60 +++- .../src/agentenv_protocol/atif/PROFILE.md | 5 +- .../src/agentenv_protocol/atif/__init__.py | 6 +- .../src/agentenv_protocol/atif/_image_size.py | 73 +++++ .../src/agentenv_protocol/atif/builder.py | 3 + .../src/agentenv_protocol/atif/reader.py | 23 ++ .../src/agentenv_protocol/atif/validator.py | 3 +- .../tests/test_a2a_atif_media.py | 308 ++++++++++++++++++ ...test_prompt_agent_live_trajectory_local.py | 58 +++- 16 files changed, 673 insertions(+), 23 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py create mode 100644 packages/agentenv-protocol/tests/test_a2a_atif_media.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c8f1e7c27..68750365e 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1696,6 +1696,20 @@ first, and a read scrubs as it serves, so a secret learned after a step was written is left out too. Native reads keep every secret byte for byte, so a replay of the served inputs holds what the served copy redacts. +A copy's media files, `media/.` beside the document, wait on the +agent's disk as the adapter names them, one file per sha256, with a PNG, JPEG, +GIF or WebP image's width and height read from its header. The worker pulls +them at each turn's end with the trajectory extension's `media` call, listed on +the card of an agent that serves `atif/1`: `{context_id, objects: {path: write +grant}}` uploads each file the context holds under its exact-object grant, up +to 8 at a time, checking each upload's bytes against its name. It answers what +it uploaded, the paths it holds no file for (`missing`) and the uploads that +failed (`failed`). Neither stops the rest. A file uploaded leaves the agent's +disk, as do a context's files when it is evicted or restored over, and the +rest when the server stops. A reader takes a part path only as +`media/.` and checks the bytes it reads against that name +(`agentenv_protocol.atif.media_file` and `check_media`). + `enable(..., description="...")` is reserved for declarations carrying configuration or metadata. It preserves the agent-specific extension prose published in the Agent Card. The versioned SDK definition provides a generic diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 3c0f2c28c..99910083b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -77,6 +77,8 @@ TaskObjectTrajectoryRequest, TaskTrajectoryRequest, TrajectoryEventsResponse, + TrajectoryMediaRequest, + TrajectoryMediaResponse, TrajectoryObjectsResponse, AtifEventsResponse, AtifObjectsResponse, @@ -256,6 +258,8 @@ "AtifObjectsResponse", "AtifUploadedObjects", "TrajectoryLog", + "TrajectoryMediaRequest", + "TrajectoryMediaResponse", "TrajectoryObjectsResponse", "TrajectoryState", "TrajectoryUploadedObjects", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py index 6e6790437..56ec6bc26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py @@ -68,8 +68,9 @@ class AtifCopy: serves goes through ``scrub``, the agent's secrets redacted.""" def __init__(self, adapter: type[Adapter], context_id: str, *, - scrub: Callable[[Any], Any] = lambda served: served) -> None: - self._run = AdapterRun(adapter, trajectory_id=uuid.uuid4().hex) + scrub: Callable[[Any], Any] = lambda served: served, + media: Callable[[str, bytes], None] | None = None) -> None: + self._run = AdapterRun(adapter, media=media, trajectory_id=uuid.uuid4().hex) self._adapter = adapter.__name__ self._context_id = context_id self._scrub = scrub diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py new file mode 100644 index 000000000..b4a8ee5e3 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py @@ -0,0 +1,74 @@ +"""The media files of an agent's atif/1 copies, spooled to its disk as the adapter names them: one file per sha256, +held by the contexts whose copies reference it, until a push confirms it for a context or that context goes. The worker +pulls them at each turn's end (``TRAJECTORY_V1.media``), so the bytes never wait in memory.""" + +from __future__ import annotations + +import os +import shutil +import tempfile +import threading +from collections.abc import Callable +from pathlib import Path + + +class MediaSpool: + """Each context's media files on disk. Safe to fill from the agent's thread while the loop pushes them; the folder + is made on the first file and removed by ``close``.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._folder: Path | None = None + #: Each context's held paths, ``media/.``. + self._held: dict[str, set[str]] = {} + + def sink(self, context_id: str) -> Callable[[str, bytes], None]: + """Where a context's copy writes each media file once, as its builder names it.""" + + def write(path: str, data: bytes) -> None: + with self._lock: + file = self._file(path) + if not file.exists(): + part = file.with_name(f"{file.name}.{os.getpid()}.part") + part.write_bytes(data) + os.replace(part, file) + self._held.setdefault(context_id, set()).add(path) + + return write + + def held(self, context_id: str, path: str) -> Path | None: + """The file a context holds at ``path``, or None when it holds none there.""" + with self._lock: + return self._file(path) if path in self._held.get(context_id, ()) else None + + def release(self, context_id: str, path: str) -> None: + """``path`` is the context's no longer: a push confirmed it. The file goes once no context holds it.""" + with self._lock: + held = self._held.get(context_id) + if held is not None: + held.discard(path) + if not held: + del self._held[context_id] + self._remove_unheld(path) + + def drop(self, context_id: str) -> None: + """The context is gone, evicted or restored over: its files go, but those another context still holds.""" + with self._lock: + for path in self._held.pop(context_id, set()): + self._remove_unheld(path) + + def close(self) -> None: + with self._lock: + if self._folder is not None: + shutil.rmtree(self._folder, ignore_errors=True) + self._folder = None + self._held.clear() + + def _file(self, path: str) -> Path: + if self._folder is None: + self._folder = Path(tempfile.mkdtemp(prefix="agentenv-atif-media-")) + return self._folder / path.removeprefix("media/") + + def _remove_unheld(self, path: str) -> None: + if self._folder is not None and not any(path in held for held in self._held.values()): + (self._folder / path.removeprefix("media/")).unlink(missing_ok=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py index a8580de35..7d8c3cc5e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py @@ -43,6 +43,9 @@ #: token is: longer than an ordinary path's words, unlike ``MIN_SECRET_LENGTH``, which bounds what is scrubbed at all. MIN_TOKEN_SEGMENT_LENGTH = 16 _TOKEN_SEGMENT = re.compile(rf"[A-Za-z0-9_.=~+-]{{{MIN_TOKEN_SEGMENT_LENGTH},}}") +#: A media file's name, ``.``: a grant's path may name the file it writes, which names its bytes, not a +#: credential, and which every atif/1 copy that holds the file must keep. +_MEDIA_NAME = re.compile(r"[0-9a-f]{64}\.[A-Za-z0-9]+") T = TypeVar("T") @@ -245,7 +248,8 @@ def _url_secrets(url: str) -> list[str]: parts = urlsplit(url) carried = _query_values(url) if not carried: - carried = [segment for segment in parts.path.split("/") if _TOKEN_SEGMENT.fullmatch(segment)] + carried = [segment for segment in parts.path.split("/") + if _TOKEN_SEGMENT.fullmatch(segment) and not _MEDIA_NAME.fullmatch(segment)] return [*carried, *([parts.password] if parts.password else [])] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py index 70a09c901..53beecc59 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py @@ -10,6 +10,7 @@ from ..atif import Adapter from ._atif_copy import AtifCopy, AtifPage, TurnStart +from ._media_spool import MediaSpool from ._turn_index import TurnIndex from .extensions import TrajectoryState from .tasks.v1 import TaskError, TrajectoryLog @@ -98,6 +99,8 @@ def __init__( self._turns = TurnIndex() if atif is not None else None #: What every atif/1 read goes through: the agent's secrets redacted. self._scrub = scrub + #: The copies' media files, on disk until the worker pulls them. + self.media = MediaSpool() self._tasks: dict[str, _TaskRecord] = {} self._contexts: OrderedDict[str, list[str]] = OrderedDict() self._copies: dict[str, AtifCopy] = {} @@ -133,7 +136,8 @@ def start(self, task_id: str, turn: TurnStart | None = None) -> None: copy = self._copies.get(record.context_id) if copy is None: copy = self._copies[record.context_id] = AtifCopy( - self._atif, record.context_id, **({"scrub": self._scrub} if self._scrub is not None else {}) + self._atif, record.context_id, media=self.media.sink(record.context_id), + **({"scrub": self._scrub} if self._scrub is not None else {}), ) copy.start(task_id, self._turns.next(record.context_id), turn) record.log._atif = _CopySink(copy, task_id) @@ -149,6 +153,7 @@ def restored(self, context_id: str, turns: int) -> None: new document, since the conversation is the snapshot's now; an agent that serves atif/1 only.""" self._turns.set(context_id, turns) self._copies.pop(context_id, None) + self.media.drop(context_id) def request_cancel(self, task_id: str) -> None: """Make ``canceled`` the outcome the task is sealed with, whatever ``run`` returns.""" @@ -357,6 +362,7 @@ def _evict(self, *, keep: str) -> None: continue del self._contexts[context_id] self._copies.pop(context_id, None) + self.media.drop(context_id) for task_id in task_ids: del self._tasks[task_id] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index 8de0a8f16..323f49536 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -25,7 +25,7 @@ ) from ..atif import FORMAT as ATIF_FORMAT -from ..atif import Adapter, AdapterSpec +from ..atif import Adapter, AdapterSpec, media_file from ._triggers import MAX_SOLVER_MESSAGE_LENGTH from .tasks.v1 import AgentConfig from ..transfers import ( @@ -205,6 +205,34 @@ class TrajectoryObjectsResponse(ExtensionResponse): format: str | None = None +#: The most media files one media call may name: a turn of a computer-use run holds a few hundred screenshots. +MAX_MEDIA_OBJECTS = 10_000 + + +class TrajectoryMediaRequest(ExtensionRequest): + """The media files of a context's atif/1 copy the caller wants, by their paths in it (``media/.``), + each under its own exact-object write grant.""" + + context_id: str = Field(min_length=1) + objects: dict[str, WriteObject] = Field(min_length=1, max_length=MAX_MEDIA_OBJECTS) + + @field_validator("objects") + @classmethod + def _media_paths(cls, objects: dict[str, WriteObject]) -> dict[str, WriteObject]: + for path in objects: + media_file(path) + return objects + + +class TrajectoryMediaResponse(ExtensionResponse): + """What a media call uploaded, by path; the paths the context holds no file for; and the uploads that failed, with + why. Neither of the last two stops the rest.""" + + objects: dict[str, Uploaded] + missing: list[str] = Field(default_factory=list) + failed: dict[str, str] = Field(default_factory=dict) + + class AtifUploadedObjects(ExtensionResponse): trajectory: Uploaded aux: Uploaded | None = None @@ -1194,6 +1222,8 @@ def _configure_trajectory( return ExtensionConfiguration( wire_params={"formats": formats, "native": native, "default": native} if native else {}, options={"native": native, **({"atif": atif} if atif is not None else {})} if native else {}, + # An agent that serves atif/1 hands over its copies' media files, which the worker pulls at each turn's end. + features={"media"} if atif is not None else frozenset(), variants=( {"get": {"task_events", "context_events", "context", "context_objects"}} if live else {} ), @@ -1341,6 +1371,20 @@ def _configure_trajectory( ), ) }, + optional_features={ + "media": OperationGroup( + operations={ + "media": _op( + "media", + "POST", + "/ext/trajectory/media", + SDK, + request_model=TrajectoryMediaRequest, + response_model=TrajectoryMediaResponse, + ), + } + ) + }, configuration_validator=_configure_trajectory, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index 63f954de7..5debc9c58 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -68,6 +68,8 @@ AtifObjectsResponse, AtifUploadedObjects, TrajectoryEventsResponse, + TrajectoryMediaRequest, + TrajectoryMediaResponse, TrajectoryObjectsResponse, TrajectoryState, TrajectoryUploadedObjects, @@ -75,6 +77,7 @@ TriggerRegisterRequest, enable, ) +from ..atif import media_file from ..computer.errors import ComputerError from ..errors import AgentEnvError from ..types import error_body @@ -101,7 +104,7 @@ from .tasks.v1 import ( TextPart as TaskTextPart, ) -from ..transfers import TransferError, serving_on, upload +from ..transfers import TransferError, Uploaded, serving_on, upload logger = logging.getLogger(__name__) @@ -125,6 +128,8 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: _AGENT_DEFINITION = "_agentenv_a2a_definition" _AgentT = TypeVar("_AgentT", bound="AgentEnvAgent") _MAX_CACHED_CONTEXTS = 1_024 +#: How many of a media call's files upload at once. +MAX_PARALLEL_MEDIA_PUSHES = 8 def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: @@ -682,6 +687,7 @@ def handlers(self) -> dict[tuple[str, str], Callable]: (PEER_AGENTS_V1.uri, "list"): self.peer_agents_list, (SKILL_CONFIG_V1.uri, "list"): self.skill_list, (TRAJECTORY_V1.uri, "get"): self.trajectory_get, + (TRAJECTORY_V1.uri, "media"): self.trajectory_media, (TRIGGERS_V1.uri, "register"): self.triggers_register, (TRIGGERS_V1.uri, "decide"): self.triggers_decide, (TRIGGERS_V1.uri, "state"): self.triggers_state, @@ -964,6 +970,39 @@ async def _atif_document( ) return AtifObjectsResponse(objects=uploaded, **versions).model_dump(mode="json", exclude_none=True) + async def trajectory_media(self, request: TrajectoryMediaRequest) -> dict[str, Any]: + """Upload the media files of a context's atif/1 copy the worker asks for, each under its grant and from the + spool, ``MAX_PARALLEL_MEDIA_PUSHES`` at a time, checking each upload's bytes against its name. A path the + context holds no file for is ``missing``, an upload that fails is ``failed``, and neither stops the rest; a + file uploaded leaves the spool.""" + media = self.task_trajectories.media + uploaded: dict[str, Uploaded] = {} + missing: list[str] = [] + failed: dict[str, str] = {} + gate = asyncio.Semaphore(MAX_PARALLEL_MEDIA_PUSHES) + + async def push(path: str, target: Any) -> None: + file = media.held(request.context_id, path) + if file is None: + missing.append(path) + return + async with gate: + try: + done = await upload(target, file) + except TransferError as exc: + failed[path] = f"{exc.code}: {exc}" + return + if done.sha256 is not None and done.sha256 != media_file(path): + failed[path] = f"the upload's sha256 is {done.sha256}" + return + uploaded[path] = done + media.release(request.context_id, path) + + await asyncio.gather(*(push(path, target) for path, target in request.objects.items())) + return TrajectoryMediaResponse(objects=uploaded, missing=sorted(missing), failed=failed).model_dump( + mode="json", exclude_none=True + ) + def check_trajectory_format(self, format: str | None) -> None: """Refuse a trajectory read naming a format the card doesn't offer, before any handler runs, the agent's own included.""" @@ -1128,7 +1167,8 @@ def _taking_the_control_token( inner: Callable[[Starlette], AbstractAsyncContextManager] | None, ) -> Callable[[Starlette], AbstractAsyncContextManager]: """The application's lifespan, ``inner``'s if it has one: serving starts by taking the agent's control token out of - the environment, and the extension routes take it from then on. An application that isn't served stays open.""" + the environment, and the extension routes take it from then on, and ends by removing the media spool. An + application that isn't served stays open.""" @asynccontextmanager async def lifespan(app: Starlette) -> AsyncIterator[Any]: @@ -1136,11 +1176,15 @@ async def lifespan(app: Starlette) -> AsyncIterator[Any]: log_guard(application.control_token) if application.services.secrets is not None: application.services.secrets.add(application.control_token, source="the control token") - if inner is None: - yield None - else: - async with inner(app) as state: - yield state + try: + if inner is None: + yield None + else: + async with inner(app) as state: + yield state + finally: + # The media the worker didn't pull go with the server. + application.services.task_trajectories.media.close() return lifespan @@ -1354,7 +1398,7 @@ async def invoke(request: Request) -> JSONResponse: extension_request = request_model.model_validate(payload) if self.services.secrets is not None: self.services.secrets.add_request(extension_request) - if extension_uri == TRAJECTORY_V1.uri: + if extension_uri == TRAJECTORY_V1.uri and operation.definition.name == "get": self.services.check_trajectory_format(extension_request.format) if extension_request.format == ATIF_FORMAT: # The SDK serves atif/1 from its own copy, context reads an agent binds included. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 709942e9e..811658112 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -108,7 +108,10 @@ One vocabulary, in three places: `[video: media/.mp4, 4.4 MB, 31 s]`. That covers video, documents, images other than JPEG, PNG, GIF and WebP (such as HEIC or SVG), audio outside ATIF's eight types, and other files. - Each file's details go in the `media` map of the profile of the object that holds the part (Step for a message, - ObservationResult for a result), keyed by its path. + ObservationResult for a result), keyed by its path. A PNG, JPEG, GIF or WebP image's `width` and `height` are read + from its header when the producer gives none. +- A reader takes a part path only as `media/.` in the document's own folder, never another path or a URL, + and checks each file's bytes against its name (`media_file`, `check_media`). ## Fields diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index b66f06e58..85bfb47d5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -46,16 +46,16 @@ read_profile, ) from .kit import KIT_RULES, KitResult, check_adapter -from .reader import Task, check, is_prefix, load, media_paths, strip_profile, tasks +from .reader import MEDIA_PATH, Task, check, check_media, is_prefix, load, media_file, media_paths, strip_profile, tasks from .runner import AdapterRun, Fed, RecordedTask, replay from .served import AuxEntry, ContentItem, MediaItem, ServedData, ServedError, ServedMedia, ServedTurn from .validator import RULES, Finding, Rule, errors_in, validate, validate_live __all__ = [ "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", - "AtifError", "AtifTurn", "AudioSource", "aux", "AuxEntry", "check", "check_adapter", "ContentItem", "ContentPart", + "AtifError", "AtifTurn", "AudioSource", "aux", "AuxEntry", "check", "check_adapter", "check_media", "ContentItem", "ContentPart", "Document", "dump_profile", "errors_in", "Fed", "FinalMetrics", "FinalMetricsProfile", "Finding", "finish", - "FORMAT", "ImageSource", "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "media_paths", "MediaInfo", + "FORMAT", "ImageSource", "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "MEDIA_PATH", "media_file", "media_paths", "MediaInfo", "MediaItem", "Metrics", "MetricsProfile", "Native", "native", "Observation", "ObservationResult", "ObservationResultProfile", "PROFILE_KEY", "profile_of", "PROFILE_VERSION", "PROFILES", "RAW_ARGUMENTS", "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", "RULES", "SCHEMA_VERSION", "Selector", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py new file mode 100644 index 000000000..6b5e1b226 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py @@ -0,0 +1,73 @@ +"""An image's width and height, read from its header for the four image types ATIF has parts for, so the profile can +carry them without Pillow. A header this can't read gives None: the image is stored all the same.""" + +from __future__ import annotations + +import struct + +_PNG = b"\x89PNG\r\n\x1a\n" +#: JPEG's start-of-frame markers, the segments that hold the frame's size: every SOFn but DHT, JPG and DAC. +_JPEG_FRAMES = frozenset({0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}) +#: JPEG markers that stand alone, with no length after them. +_JPEG_BARE = frozenset({0x01, *range(0xD0, 0xD9)}) + + +def image_size(data: bytes, media_type: str) -> tuple[int, int] | None: + """``(width, height)`` of a PNG, JPEG, GIF or WebP image from its header, or None.""" + read = {"image/png": _png, "image/jpeg": _jpeg, "image/gif": _gif, "image/webp": _webp}.get(media_type) + try: + size = read(data) if read is not None else None + except (IndexError, struct.error): + return None + return size if size is not None and size[0] > 0 and size[1] > 0 else None + + +def _png(data: bytes) -> tuple[int, int] | None: + if data[:8] != _PNG or data[12:16] != b"IHDR": + return None + return struct.unpack(">II", data[16:24]) + + +def _gif(data: bytes) -> tuple[int, int] | None: + if data[:6] not in (b"GIF87a", b"GIF89a"): + return None + return struct.unpack(" tuple[int, int] | None: + if data[:2] != b"\xff\xd8": + return None + at = 2 + while at + 4 <= len(data): + if data[at] != 0xFF: + return None + marker = data[at + 1] + if marker == 0xFF: + at += 1 + continue + if marker in _JPEG_BARE: + at += 2 + continue + (length,) = struct.unpack(">H", data[at + 2:at + 4]) + if marker in _JPEG_FRAMES: + height, width = struct.unpack(">HH", data[at + 5:at + 9]) + return width, height + if marker == 0xDA: + return None + at += 2 + length + return None + + +def _webp(data: bytes) -> tuple[int, int] | None: + if data[:4] != b"RIFF" or data[8:12] != b"WEBP": + return None + chunk = data[12:16] + if chunk == b"VP8 " and data[23:26] == b"\x9d\x01\x2a": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + if chunk == b"VP8X": + return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1 + return None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py index cd7748390..d01d6a8dc 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py @@ -16,6 +16,7 @@ from pydantic import ValidationError +from ._image_size import image_size from .adapter import AdapterSpec from .models.content import AudioMediaType, ContentPart, ImageSource from .profile import ( @@ -418,6 +419,8 @@ def store(self, data: bytes, media_type: str, info: Mapping[str, Any]) -> Media: for key in ("width", "height", "duration_ms", "pages"): _int(info.get(key), key) _str(info.get("name"), "a media file's name") + if info.get("width") is None and info.get("height") is None and (size := image_size(bytes(data), media_type)): + info = {**info, "width": size[0], "height": size[1]} digest = hashlib.sha256(data).hexdigest() extension = _EXTENSIONS.get(media_type) or mimetypes.guess_extension(media_type, strict=False) or ".bin" path = f"media/{digest}{extension}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py index a8dc625d9..fe58598fd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py @@ -4,7 +4,9 @@ from __future__ import annotations import copy +import hashlib import json +import re from dataclasses import dataclass, field from pathlib import Path from typing import Any, Iterator, Mapping @@ -93,6 +95,27 @@ def _parts(value: Any) -> list[Mapping[str, Any]]: return [p for p in value if isinstance(p, Mapping)] if isinstance(value, list) else [] +#: The only path a media file of a document can have: ``media/.``, in the document's own folder. +MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") + + +def media_file(path: str) -> str: + """``path``'s sha256, once ``path`` is ``media/.``: a part path resolves only to a file in the + document's own media folder, never to another path or a URL. AtifError otherwise.""" + named = MEDIA_PATH.match(path) if isinstance(path, str) else None + if named is None: + raise AtifError(f"{path!r} isn't media/. in the document's folder") + return named["sha256"] + + +def check_media(path: str, data: bytes) -> None: + """Refuse media whose bytes aren't what its name says: a reader checks each file it reads, since a store can hold + another file under that name. AtifError naming the difference.""" + digest = hashlib.sha256(data).hexdigest() + if media_file(path) != digest: + raise AtifError(f"{path}'s bytes have sha256 {digest}") + + def media_paths(document: Mapping[str, Any]) -> list[str]: """Every media file the document names, once each, in the order they appear: image and audio parts' paths, and the files the profile records for media ATIF has no part type for.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 2a18737c5..7675a0200 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -26,12 +26,11 @@ StepError, profile_of, ) -from .reader import is_prefix, tasks +from .reader import MEDIA_PATH, is_prefix, tasks ERROR = "error" WARNING = "warning" -MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3}|\.\d{6})?Z$") _COUNTS = (("prompt_tokens", "total_prompt_tokens"), ("completion_tokens", "total_completion_tokens"), ("cached_tokens", "total_cached_tokens")) diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_media.py b/packages/agentenv-protocol/tests/test_a2a_atif_media.py new file mode 100644 index 000000000..be0c00d60 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_atif_media.py @@ -0,0 +1,308 @@ +"""An agent that serves atif/1 spools its copies' media files to its disk, one file per sha256, with each image's size +read from its header, and uploads the ones the worker asks for under exact-object grants (``TRAJECTORY_V1.media``), +which is how the worker pulls a turn's media at its end. Readers take a part path only as media/., and +only with the bytes that name.""" +from __future__ import annotations + +import base64 +import hashlib +import struct +import zlib +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +from starlette.testclient import TestClient +from test_a2a_atif_serving import _atif, _ended, _send + +from agentenv_protocol.a2a_agent import ( + TRAJECTORY_V1, + AgentEnvAgent, + AgentIdentity, + TaskRequest, + TaskResult, + TextPart, + TrajectoryState, + Uploaded, + a2a_agent, + enable, + framework, +) +from agentenv_protocol.a2a_agent._atif_copy import TurnStart +from agentenv_protocol.a2a_agent._trajectory_log import TaskTrajectories +from agentenv_protocol.atif import Adapter, AtifError, atif_adapter, check_media, errors_in, media_file, native, validate +from agentenv_protocol.atif._image_size import image_size +from agentenv_protocol.transfers import TransferError + +NATIVE = "shots/1" + + +def _png(width: int, height: int) -> bytes: + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) + + +SHOT = _png(4, 3) + + +@atif_adapter(native_format=NATIVE, version="1", input_includes_cache=False) +class Shots(Adapter): + @native(type="shot") + def shot(self, record: dict[str, Any], turn: Any) -> None: + data = base64.b64decode(record["data"]) + media = turn.media(data, record["media_type"], **record.get("info", {})) + turn.message(step=record["step"], content=[media]) + turn.step(key=record["step"], llm_call_count=1) + + +def _shooting(*, atif: bool = True) -> AgentEnvAgent: + """An agent whose turn appends each base64 file its prompt's JSON lists as a native ``shot``.""" + trajectory = enable(TRAJECTORY_V1, live=True, native=NATIVE, **({"atif": Shots} if atif else {})) + + @a2a_agent(identity=AgentIdentity(name="shots", description="test", version="1"), extensions=(trajectory,)) + class Shooting(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + request.trajectory.set_format(NATIVE) + for index, part in enumerate(request.parts[1:]): + request.trajectory.append({"type": "shot", "step": f"s{index}", "data": part.bytes, + "media_type": part.mime_type}) + return TaskResult.text("done") + + return Shooting() + + +def _file(data: bytes, media_type: str, name: str = "file") -> dict[str, Any]: + return {"kind": "file", "file": {"bytes": base64.b64encode(data).decode(), "mimeType": media_type, "name": name}} + + +def _grant(path: str, media_type: str = "image/png") -> dict[str, Any]: + expires = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(timespec="seconds").replace("+00:00", "Z") + return {"media_type": media_type, "max_bytes": 10_000_000, + "write": {"kind": "http-put", "url": f"https://objects.example.test/{path}", "expires_at": expires}} + + +def _path(data: bytes, extension: str) -> str: + return f"media/{hashlib.sha256(data).hexdigest()}{extension}" + + +@pytest.fixture +def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, bytes]: + """Where the media calls' grants upload: each object by its path.""" + stored: dict[str, bytes] = {} + + async def upload(target, source) -> Uploaded: + data = Path(source).read_bytes() + stored[target.write.url.removeprefix("https://objects.example.test/")] = data + return Uploaded(size_bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) + + monkeypatch.setattr(framework, "upload", upload) + return stored + + +def _media(client: TestClient, paths: dict[str, str], context_id: str = "ctx-1") -> Any: + return client.post("/ext/trajectory/media", json={"context_id": context_id, "objects": { + path: _grant(path, media_type) for path, media_type in paths.items()}}) + + +def test_a_copys_images_are_spooled_by_sha256_with_their_size_and_pushed_once_asked(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png", "inbound.png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + path = _path(SHOT, ".png") + held = spool.held("ctx-1", path) + assert held is not None and held.read_bytes() == SHOT + pushed = _media(client, {path: "image/png"}) + assert pushed.status_code == 200, pushed.text + assert spool.held("ctx-1", path) is None and not held.exists() + assert errors_in(validate(document)) == [] + [user] = [step for step in document["steps"] if step["source"] == "user"] + image = next(part for part in user["message"] if isinstance(part, dict) and part.get("type") == "image") + assert image["source"]["path"] == path + assert user["extra"]["agentenv"]["media"][path] == {"sha256": hashlib.sha256(SHOT).hexdigest(), + "bytes": len(SHOT), "width": 4, "height": 3, "name": "inbound.png"} + assert pushed.json() == {"objects": {path: {"size_bytes": len(SHOT), "sha256": hashlib.sha256(SHOT).hexdigest()}}, + "missing": [], "failed": {}} + assert store == {path: SHOT} + + +def test_a_media_calls_grants_leave_the_copys_media_paths_as_they_are(store: dict[str, bytes]) -> None: + """A grant's URL may name the file it writes, ``media/.``: the scrub takes a URL's credential from it, + never the file's name, which every read of the copy keeps.""" + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + assert _media(client, {_path(SHOT, ".png"): "image/png"}).status_code == 200 + document = _atif(client, task_id=task_id)["trajectory"] + [user] = [step for step in document["steps"] if step["source"] == "user"] + assert any(part.get("source", {}).get("path") == _path(SHOT, ".png") for part in user["message"] + if isinstance(part, dict)) + + +def test_the_same_bytes_added_twice_are_spooled_and_pushed_once(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(SHOT, "image/png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + folder = client.app.state.agentenv_a2a.services.task_trajectories.media._folder + assert sorted(file.name for file in folder.iterdir()) == [_path(SHOT, ".png").removeprefix("media/")] + pushed = _media(client, {_path(SHOT, ".png"): "image/png"}) + paths = [part["source"]["path"] for step in document["steps"] for part in step.get("message") or [] + if isinstance(part, dict) and part.get("type") == "image"] + assert paths == [_path(SHOT, ".png")] * 4 + assert list(pushed.json()["objects"]) == [_path(SHOT, ".png")] and len(store) == 1 + + +def test_a_path_the_context_doesnt_hold_and_a_failed_upload_dont_stop_the_rest( + store: dict[str, bytes], monkeypatch: pytest.MonkeyPatch) -> None: + other = _png(2, 2) + stored = framework.upload + + async def upload(target, source) -> Uploaded: + if target.write.url.endswith(_path(other, ".png")): + raise TransferError("grant_expired", "The transfer grant expired.") + return await stored(target, source) + + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(other, "image/png")]) + _ended(client, task_id) + monkeypatch.setattr(framework, "upload", upload) + never = _path(b"never added", ".png") + pushed = _media(client, {_path(SHOT, ".png"): "image/png", _path(other, ".png"): "image/png", + never: "image/png"}).json() + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + assert spool.held("ctx-1", _path(other, ".png")) is not None + other_context = _media(client, {_path(SHOT, ".png"): "image/png"}, context_id="ctx-2").json() + assert list(pushed["objects"]) == [_path(SHOT, ".png")] and pushed["missing"] == [never] + assert pushed["failed"] == {_path(other, ".png"): "grant_expired: The transfer grant expired."} + assert other_context == {"objects": {}, "missing": [_path(SHOT, ".png")], "failed": {}} + + +def test_a_media_call_takes_only_media_paths() -> None: + with TestClient(_shooting().create_app()) as client: + for path in ("../../etc/passwd", "media/abc.png", "https://elsewhere.example.test/x.png", + f"media/{'a' * 64}/x.png"): + refused = _media(client, {path: "image/png"}) + assert refused.status_code == 400, (path, refused.text) + + +def test_only_an_agent_that_serves_atif_offers_media() -> None: + def methods(agent: AgentEnvAgent) -> set[str]: + card = TestClient(agent.create_app()).get("/.well-known/agent-card.json").json() + [trajectory] = [ext for ext in card["capabilities"]["extensions"] if ext["uri"] == TRAJECTORY_V1.uri] + return set(trajectory["params"]["methods"]) + + assert methods(_shooting()) == {"get", "media"} + assert methods(_shooting(atif=False)) == {"get"} + + +def test_a_video_and_a_heic_image_are_spooled_and_pushed_like_any_file(store: dict[str, bytes]) -> None: + video, heic = b"\x00\x00\x00\x18ftypmp42" + b"v" * 64, b"\x00\x00\x00\x18ftypheic" + b"h" * 64 + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(video, "video/mp4"), _file(heic, "image/heic")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + pushed = _media(client, {_path(video, ".mp4"): "video/mp4", _path(heic, ".heic"): "image/heic"}).json() + assert errors_in(validate(document)) == [] + [user] = [step for step in document["steps"] if step["source"] == "user"] + # ATIF has no part for either, so the typed content names each file in a plain note. + assert f"[video: {_path(video, '.mp4')}, 76 B]" in user["message"] + assert f"[image: {_path(heic, '.heic')}, 76 B]" in user["message"] + assert sorted(pushed["objects"]) == sorted([_path(video, ".mp4"), _path(heic, ".heic")]) + assert store[_path(video, ".mp4")] == video and store[_path(heic, ".heic")] == heic + + +def test_a_context_evicted_or_restored_over_drops_its_media_but_what_another_holds() -> None: + trajectories = TaskTrajectories(max_contexts=1, native=NATIVE, atif=Shots) + + def play(task_id: str, context_id: str, data: bytes) -> None: + log = trajectories.register(task_id, context_id) + trajectories.start(task_id, TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", + agent_version="1", agent_model=None)) + log.append({"type": "shot", "step": "s", "data": base64.b64encode(data).decode(), "media_type": "image/png"}) + trajectories.seal(task_id, TrajectoryState.COMPLETED, final=(TextPart(text="done"),)) + + other = _png(1, 1) + play("a1", "context-a", SHOT) + file = trajectories.media.held("context-a", _path(SHOT, ".png")) + play("b1", "context-b", other) + assert trajectories.media.held("context-a", _path(SHOT, ".png")) is None and not file.exists() + play("b2", "context-b", SHOT) + trajectories.restored("context-b", 0) + assert trajectories.media.held("context-b", _path(other, ".png")) is None + trajectories.media.close() + + +def test_the_spool_goes_with_the_server(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + folder = client.app.state.agentenv_a2a.services.task_trajectories.media._folder + assert folder.is_dir() + assert not folder.exists() + + +def test_the_native_trajectory_is_unchanged_by_media(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + served = client.post("/ext/trajectory", json={"task_id": task_id}).json()["trajectory"] + assert served == [{"type": "shot", "step": "s0", "data": base64.b64encode(SHOT).decode(), "media_type": "image/png"}] + + +def _jpeg(width: int, height: int) -> bytes: + app0 = b"\xff\xe0" + struct.pack(">H", 16) + b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00" + sof = b"\xff\xc0" + struct.pack(">HBHHB", 11, 8, height, width, 1) + b"\x01\x11\x00" + return b"\xff\xd8" + app0 + sof + b"\xff\xda\x00\x08\x01\x01\x00\x00\x3f\x00" + b"\x00" * 8 + b"\xff\xd9" + + +def _webp(kind: bytes, body: bytes) -> bytes: + chunk = kind + struct.pack(" None: + assert image_size(data, media_type) == size + + +def test_dimensions_an_adapter_passes_win_over_the_header() -> None: + trajectories = TaskTrajectories(native=NATIVE, atif=Shots) + log = trajectories.register("t1", "ctx") + trajectories.start("t1", TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", + agent_version="1", agent_model=None)) + log.append({"type": "shot", "step": "s", "data": base64.b64encode(SHOT).decode(), "media_type": "image/png", + "info": {"width": 40, "height": 30}}) + trajectories.seal("t1", TrajectoryState.COMPLETED, final=(TextPart(text="done"),)) + document = trajectories.atif_task("t1").document + media = next(step["extra"]["agentenv"]["media"] for step in document["steps"] if step["source"] == "agent") + assert (media[_path(SHOT, ".png")]["width"], media[_path(SHOT, ".png")]["height"]) == (40, 30) + trajectories.media.close() + + +def test_a_reader_takes_a_part_path_only_as_media_sha256_and_only_with_the_bytes_that_name() -> None: + path = _path(SHOT, ".png") + assert media_file(path) == hashlib.sha256(SHOT).hexdigest() + check_media(path, SHOT) + for bad in ("../media/x.png", "media/abc.png", "https://example.test/media/x.png", f"/{path}", f"{path}/x"): + with pytest.raises(AtifError): + media_file(bad) + with pytest.raises(AtifError, match="bytes have sha256"): + check_media(path, SHOT + b"tampered") diff --git a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py index 1b6f44491..ed3125720 100644 --- a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py +++ b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py @@ -5,24 +5,30 @@ the last none. The echo agent declares its native format and an ATIF adapter, so its card offers both; its native reads name their format, its atif/1 reads are valid ATIF, and a read naming another format is refused with a code agent-env reads. A snapshot of it carries its atif/1 copy's turn count to the fresh agent it is restored into, and its atif/1 -reads leave out the model key a command it runs prints, which its native read keeps. Needs Docker and a throwaway local -registry.""" +reads leave out the model key a command it runs prints, which its native read keeps. An image sent to it is spooled +as its copy's media file, and pushed under an exact-object grant from the local store's grant server. Needs Docker and a +throwaway local registry.""" +import base64 +import hashlib import json import re +import struct import shutil import socket import subprocess import time import uuid +import zlib import httpx import pytest -from agentenv_protocol.atif import FORMAT as ATIF, PROFILE_VERSION, SCHEMA_VERSION, errors_in, validate +from agentenv_protocol.atif import FORMAT as ATIF, PROFILE_VERSION, SCHEMA_VERSION, errors_in, media_paths, validate from agent_env.a2a_agent import A2AAgent -from agent_env.a2a_agent.protocol import raise_for_extension_status +from agent_env.a2a_agent.object_transfer import write_object +from agent_env.a2a_agent.protocol import poll_a2a_task, raise_for_extension_status, send_a2a_message from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.artifact.store import reset_artifact_store from agent_env.config import configure, get_config, reset_config, set_image_store @@ -256,3 +262,47 @@ async def test_a_model_key_a_command_prints_is_redacted_in_atif_and_kept_in_the_ served = json.dumps(atif) assert key not in served and f"[redacted:{len(key)}]" in served assert errors_in(validate(atif["trajectory"])) == [] + + +def _png(width: int, height: int) -> bytes: + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) + + +@pytest.mark.asyncio +async def test_a_deployed_agent_pushes_its_copys_media_under_exact_object_grants(local_registry): + suffix = uuid.uuid4().hex[:8] + context_id = f"media-{suffix}" + agent = put_test_agent(f"media-agent-{suffix}") + context = await Task.put(id=f"media-{suffix}", steps=[_deploy(agent, "solver")]).run() + assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") + [deployed] = context.deployed_agents + image = _png(6, 4) + parts = [{"kind": "text", "text": "look"}, + {"kind": "file", "file": {"bytes": base64.b64encode(image).decode(), "mimeType": "image/png", + "name": "shot.png"}}] + task_id, _ = await send_a2a_message(deployed.a2a_url, parts, f"m-{suffix}", context_id, 60) + await poll_a2a_task(deployed.a2a_url, task_id, 120, 1) + trajectory = A2AAgent.find_extension(deployed.a2a_card, A2AAgent.EXT_TRAJECTORY) + _, media_endpoint = A2AAgent.operation(trajectory, "media") + store = get_config().get_object_store() + async with httpx.AsyncClient(timeout=60, headers=deployed.control_headers) as client: + atif = (await client.post(deployed.a2a_url + trajectory["params"]["endpoint"], + json={"task_id": task_id, "format": ATIF})).json() + [path] = media_paths(atif["trajectory"]) + url = store.object_url(f"atif-media-{suffix}/{path}") + grant = write_object(store, url, media_type="image/png", max_bytes=len(image)) + pushed = await client.post(deployed.a2a_url + media_endpoint, json={ + "context_id": context_id, "objects": {path: grant.model_dump(mode="json", exclude_none=True)}}) + raise_for_extension_status(pushed, operation="media push") + digest = hashlib.sha256(image).hexdigest() + assert path == f"media/{digest}.png" + assert pushed.json() == {"objects": {path: {"size_bytes": len(image), "sha256": digest}}, "missing": [], "failed": {}} + assert store.get(url) == image + [user] = [step for step in atif["trajectory"]["steps"] if step["source"] == "user"] + assert user["extra"]["agentenv"]["media"][path] == {"sha256": digest, "bytes": len(image), "width": 6, "height": 4, + "name": "shot.png"} From c7b8dc83132364f2ab4f55c78a55bcd25142323d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 17:42:38 -0800 Subject: [PATCH 2/4] fix(a2a_agent): media pushes survive overrides and restores, and check what they send - The media call is served by the SDK alone (`OperationDefinition.sdk_only`): an agent that serves its own trajectory reads keeps it, and binding it is refused. - Each spool hold has its own number, so an upload that outlives its context's copy releases only the hold it began with. - A spooled file is hashed before its upload; one that no longer matches its name is reported failed and never reaches the store. - A WebP header too short for its size fields gives no size. Co-Authored-By: Claude Opus 5.5 --- .../a2a_agent/_media_spool.py | 34 +++++---- .../agentenv_protocol/a2a_agent/extensions.py | 7 ++ .../agentenv_protocol/a2a_agent/framework.py | 28 ++++++-- .../agentenv_protocol/a2a_agent/registry.py | 4 +- .../src/agentenv_protocol/atif/_image_size.py | 2 + .../tests/test_a2a_atif_media.py | 72 +++++++++++++++++-- 6 files changed, 124 insertions(+), 23 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py index b4a8ee5e3..3c7e1fc1d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py @@ -4,6 +4,7 @@ from __future__ import annotations +import itertools import os import shutil import tempfile @@ -14,13 +15,15 @@ class MediaSpool: """Each context's media files on disk. Safe to fill from the agent's thread while the loop pushes them; the folder - is made on the first file and removed by ``close``.""" + is made on the first file and removed by ``close``. Each hold has its own number, so an upload that outlives its + context's copy (a restore, an eviction) releases only the hold it began with, never a later copy's.""" def __init__(self) -> None: self._lock = threading.Lock() self._folder: Path | None = None - #: Each context's held paths, ``media/.``. - self._held: dict[str, set[str]] = {} + #: Each context's held paths, ``media/.``, each with its hold's number. + self._held: dict[str, dict[str, int]] = {} + self._holds = itertools.count(1) def sink(self, context_id: str) -> Callable[[str, bytes], None]: """Where a context's copy writes each media file once, as its builder names it.""" @@ -32,29 +35,32 @@ def write(path: str, data: bytes) -> None: part = file.with_name(f"{file.name}.{os.getpid()}.part") part.write_bytes(data) os.replace(part, file) - self._held.setdefault(context_id, set()).add(path) + self._held.setdefault(context_id, {}).setdefault(path, next(self._holds)) return write - def held(self, context_id: str, path: str) -> Path | None: - """The file a context holds at ``path``, or None when it holds none there.""" + def held(self, context_id: str, path: str) -> tuple[Path, int] | None: + """The file a context holds at ``path`` and its hold's number, or None when it holds none there.""" with self._lock: - return self._file(path) if path in self._held.get(context_id, ()) else None + hold = self._held.get(context_id, {}).get(path) + return None if hold is None else (self._file(path), hold) - def release(self, context_id: str, path: str) -> None: - """``path`` is the context's no longer: a push confirmed it. The file goes once no context holds it.""" + def release(self, context_id: str, path: str, hold: int) -> None: + """Hold ``hold`` on ``path`` is the context's no longer: a push confirmed it. A later hold stays, and the file + goes once no context holds it.""" with self._lock: held = self._held.get(context_id) - if held is not None: - held.discard(path) - if not held: - del self._held[context_id] + if held is None or held.get(path) != hold: + return + del held[path] + if not held: + del self._held[context_id] self._remove_unheld(path) def drop(self, context_id: str) -> None: """The context is gone, evicted or restored over: its files go, but those another context still holds.""" with self._lock: - for path in self._held.pop(context_id, set()): + for path in self._held.pop(context_id, {}): self._remove_unheld(path) def close(self) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index 323f49536..7058649f1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -844,6 +844,9 @@ class OperationDefinition: implementation: ImplementationOwner request: RequestDefinition | type[BaseModel] | None = None response: FieldSchema | type[BaseModel] | None = None + #: Served by the SDK alone, from state only it holds: no agent overrides it, and an agent that overrides the + #: extension's other SDK operations needn't override it with them. + sdk_only: bool = False response_model: type[BaseModel] | None = field(default=None, init=False) def __post_init__(self) -> None: @@ -1132,6 +1135,7 @@ def _op( response_required: Iterable[str] = (), response_optional: Iterable[str] = (), variants: Iterable[RequestVariant] = (), + sdk_only: bool = False, ) -> OperationDefinition: response_required = _tuple(response_required) response_optional = _tuple(response_optional) @@ -1151,6 +1155,7 @@ def _op( implementation=owner, request=request, response=response, + sdk_only=sdk_only, ) @@ -1381,6 +1386,8 @@ def _configure_trajectory( SDK, request_model=TrajectoryMediaRequest, response_model=TrajectoryMediaResponse, + # The spool it reads is the SDK's, so an agent that serves its own trajectory reads still has it. + sdk_only=True, ), } ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index 5debc9c58..f1285e6d7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -4,6 +4,7 @@ import asyncio import dataclasses +import hashlib import inspect import json import logging @@ -151,6 +152,14 @@ def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: return result +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for block in iter(lambda: file.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + def _config_write_only_wire_names(config: type[AgentConfig]) -> set[str]: """Return wire names explicitly marked as write-only in the config schema.""" wire_names = _config_wire_names(config) @@ -982,21 +991,32 @@ async def trajectory_media(self, request: TrajectoryMediaRequest) -> dict[str, A gate = asyncio.Semaphore(MAX_PARALLEL_MEDIA_PUSHES) async def push(path: str, target: Any) -> None: - file = media.held(request.context_id, path) - if file is None: + held = media.held(request.context_id, path) + if held is None: missing.append(path) return + file, hold = held async with gate: + # Checked before the bytes leave, so a spool file that no longer matches its name never reaches the + # store, and after, for bytes that changed in transit. + try: + digest = await asyncio.to_thread(_sha256_file, file) + except OSError as exc: + failed[path] = f"the spooled file can't be read: {exc.strerror}" + return + if digest != media_file(path): + failed[path] = f"the spooled file's sha256 is {digest}" + return try: done = await upload(target, file) except TransferError as exc: failed[path] = f"{exc.code}: {exc}" return - if done.sha256 is not None and done.sha256 != media_file(path): + if done.sha256 is not None and done.sha256 != digest: failed[path] = f"the upload's sha256 is {done.sha256}" return uploaded[path] = done - media.release(request.context_id, path) + media.release(request.context_id, path, hold) await asyncio.gather(*(push(path, target) for path, target in request.objects.items())) return TrajectoryMediaResponse(objects=uploaded, missing=sorted(missing), failed=failed).model_dump( diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py index c0a4ea7be..1c560ff8a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py @@ -304,6 +304,8 @@ def build_registry( variant_handlers[key] = handler continue + if operation.sdk_only: + raise ValueError(f"{uri}.{operation.name} is served by the SDK alone; an agent can't override it") key = (uri, binding.operation) if key in operation_handlers: raise ValueError(f"multiple handlers for {uri}.{operation.name}") @@ -424,7 +426,7 @@ def build_registry( active_sdk_operations = { name for name, operation in resolved_operations.items() - if operation.definition.implementation is ImplementationOwner.SDK + if operation.definition.implementation is ImplementationOwner.SDK and not operation.definition.sdk_only } overridden_sdk_operations = { name diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py index 6b5e1b226..e5c1eddd9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py @@ -62,6 +62,8 @@ def _webp(data: bytes) -> tuple[int, int] | None: if data[:4] != b"RIFF" or data[8:12] != b"WEBP": return None chunk = data[12:16] + if len(data) < 30: + return None if chunk == b"VP8 " and data[23:26] == b"\x9d\x01\x2a": width, height = struct.unpack(" Uploaded: assert other_context == {"objects": {}, "missing": [_path(SHOT, ".png")], "failed": {}} +def test_an_agent_that_serves_its_own_trajectory_reads_still_has_the_sdks_media_and_cant_take_it_over( + store: dict[str, bytes]) -> None: + @a2a_agent(identity=AgentIdentity(name="own-reads", description="test", version="1"), + extensions=(enable(TRAJECTORY_V1, native=NATIVE, atif=Shots),)) + class OwnReads(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + request.trajectory.set_format(NATIVE) + return TaskResult.text("done") + + @extension(TRAJECTORY_V1.get) + async def trajectory(self, request: TaskTrajectoryRequest | TaskObjectTrajectoryRequest) -> dict[str, Any]: + return {"trajectory": []} + + with TestClient(OwnReads().create_app()) as client: + _ended(client, _send(client, "look", "m-1", extra_parts=[_file(SHOT, "image/png")])) + assert list(_media(client, {_path(SHOT, ".png"): "image/png"}).json()["objects"]) == [_path(SHOT, ".png")] + + @a2a_agent(identity=AgentIdentity(name="takes-media", description="test", version="1"), + extensions=(enable(TRAJECTORY_V1, live=True, native=NATIVE, atif=Shots),)) + class TakesMedia(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + return TaskResult.text("done") + + @extension(TRAJECTORY_V1.media.media) + async def media(self, request: Any) -> dict[str, Any]: + return {"objects": {}} + + with pytest.raises(ValueError, match="served by the SDK alone"): + TakesMedia().create_app() + + +def test_an_upload_that_outlives_its_copy_releases_only_the_hold_it_began_with() -> None: + spool = MediaSpool() + path = _path(SHOT, ".png") + spool.sink("ctx")(path, SHOT) + _, old = spool.held("ctx", path) + spool.drop("ctx") + spool.sink("ctx")(path, SHOT) + spool.release("ctx", path, old) + file, new = spool.held("ctx", path) + assert new != old and file.read_bytes() == SHOT + spool.release("ctx", path, new) + assert spool.held("ctx", path) is None and not file.exists() + spool.close() + + +def test_a_spooled_file_that_no_longer_matches_its_name_never_reaches_the_store(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + file, _ = spool.held("ctx-1", _path(SHOT, ".png")) + file.write_bytes(b"changed on disk") + pushed = _media(client, {_path(SHOT, ".png"): "image/png"}).json() + assert pushed["objects"] == {} and store == {} + assert pushed["failed"][_path(SHOT, ".png")].startswith("the spooled file's sha256 is ") + + def test_a_media_call_takes_only_media_paths() -> None: with TestClient(_shooting().create_app()) as client: for path in ("../../etc/passwd", "media/abc.png", "https://elsewhere.example.test/x.png", @@ -230,7 +291,7 @@ def play(task_id: str, context_id: str, data: bytes) -> None: other = _png(1, 1) play("a1", "context-a", SHOT) - file = trajectories.media.held("context-a", _path(SHOT, ".png")) + file, _ = trajectories.media.held("context-a", _path(SHOT, ".png")) play("b1", "context-b", other) assert trajectories.media.held("context-a", _path(SHOT, ".png")) is None and not file.exists() play("b2", "context-b", SHOT) @@ -275,9 +336,12 @@ def _webp(kind: bytes, body: bytes) -> bytes: (_webp(b"VP8X", b"\x00" * 4 + (1919).to_bytes(3, "little") + (1079).to_bytes(3, "little")), "image/webp", (1920, 1080)), (b"\x89PNG not really", "image/png", None), + (_webp(b"VP8X", b""), "image/webp", None), + (_webp(b"VP8X", b"\x00" * 4 + b"\x01\x02"), "image/webp", None), (b"\xff\xd8\xff\xda", "image/jpeg", None), (_png(8, 8), "image/heic", None), -], ids=["png", "jpeg", "gif", "webp-lossy", "webp-lossless", "webp-extended", "bad-png", "jpeg-without-frame", +], ids=["png", "jpeg", "gif", "webp-lossy", "webp-lossless", "webp-extended", "bad-png", "truncated-webp", + "half-a-webp-size", "jpeg-without-frame", "not-an-atif-image-type"]) def test_an_images_size_is_read_from_its_header(data: bytes, media_type: str, size: tuple[int, int] | None) -> None: assert image_size(data, media_type) == size From 2343d57ff058a13497e6bcc52b749d4256fd5177 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 17:55:18 -0800 Subject: [PATCH 3/4] refactor(a2a_agent): name the media hash block size Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/a2a_agent/framework.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index f1285e6d7..982a26833 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -131,6 +131,8 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: _MAX_CACHED_CONTEXTS = 1_024 #: How many of a media call's files upload at once. MAX_PARALLEL_MEDIA_PUSHES = 8 +#: How much of a spooled media file is read at a time to hash it before its upload. +_MEDIA_HASH_BLOCK_BYTES = 1 << 20 def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: @@ -155,7 +157,7 @@ def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: - for block in iter(lambda: file.read(1 << 20), b""): + for block in iter(lambda: file.read(_MEDIA_HASH_BLOCK_BYTES), b""): digest.update(block) return digest.hexdigest() From 2500a2dab090cfc0731bdd68dcf9790240beff84 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 21:13:40 -0800 Subject: [PATCH 4/4] fix(a2a_agent): a media file the spool can't write is left out, and the copy goes on A write that failed, on a full disk say, raised in the copy's guard and stopped the context's atif/1 copy for good, naming the adapter. The spool now logs it and keeps no file, so a push reports that path missing, as the validator's media rule does, and later files still spool. Co-Authored-By: Claude Opus 5.5 --- .../a2a_agent/_media_spool.py | 24 ++++++++++++++----- .../tests/test_a2a_atif_media.py | 21 ++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py index 3c7e1fc1d..cb7745998 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py @@ -5,6 +5,7 @@ from __future__ import annotations import itertools +import logging import os import shutil import tempfile @@ -12,11 +13,14 @@ from collections.abc import Callable from pathlib import Path +logger = logging.getLogger(__name__) + class MediaSpool: """Each context's media files on disk. Safe to fill from the agent's thread while the loop pushes them; the folder is made on the first file and removed by ``close``. Each hold has its own number, so an upload that outlives its - context's copy (a restore, an eviction) releases only the hold it began with, never a later copy's.""" + context's copy (a restore, an eviction) releases only the hold it began with, never a later copy's. A file that + can't be written, on a full disk say, is left out: a push reports it missing, and the copy goes on.""" def __init__(self) -> None: self._lock = threading.Lock() @@ -30,11 +34,19 @@ def sink(self, context_id: str) -> Callable[[str, bytes], None]: def write(path: str, data: bytes) -> None: with self._lock: - file = self._file(path) - if not file.exists(): - part = file.with_name(f"{file.name}.{os.getpid()}.part") - part.write_bytes(data) - os.replace(part, file) + try: + file = self._file(path) + if not file.exists(): + part = file.with_name(f"{file.name}.{os.getpid()}.part") + try: + part.write_bytes(data) + os.replace(part, file) + finally: + part.unlink(missing_ok=True) + except OSError as exc: + logger.warning("the atif/1 copy of context %s keeps no file for %s, which a push reports " + "missing: %s", context_id, path, exc) + return self._held.setdefault(context_id, {}).setdefault(path, next(self._holds)) return write diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_media.py b/packages/agentenv-protocol/tests/test_a2a_atif_media.py index 05ed56623..5bc380ce6 100644 --- a/packages/agentenv-protocol/tests/test_a2a_atif_media.py +++ b/packages/agentenv-protocol/tests/test_a2a_atif_media.py @@ -245,6 +245,27 @@ def test_a_spooled_file_that_no_longer_matches_its_name_never_reaches_the_store( assert pushed["failed"][_path(SHOT, ".png")].startswith("the spooled file's sha256 is ") +def test_a_file_the_spool_cant_write_is_reported_missing_and_the_copy_goes_on( + store: dict[str, bytes], tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + other = _png(2, 2) + with TestClient(_shooting().create_app()) as client: + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + spool._folder = tmp_path / "removed" + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + spool._folder = None + task_id = _send(client, "again", "m-2", extra_parts=[_file(other, "image/png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + pushed = _media(client, {_path(SHOT, ".png"): "image/png", _path(other, ".png"): "image/png"}).json() + assert pushed == {"objects": {_path(other, ".png"): {"size_bytes": len(other), + "sha256": hashlib.sha256(other).hexdigest()}}, + "missing": [_path(SHOT, ".png")], "failed": {}} + paths = {part["source"]["path"] for step in document["steps"] for part in step.get("message") or [] + if isinstance(part, dict) and part.get("type") == "image"} + assert paths == {_path(SHOT, ".png"), _path(other, ".png")} + assert f"keeps no file for {_path(SHOT, '.png')}" in caplog.text + + def test_a_media_call_takes_only_media_paths() -> None: with TestClient(_shooting().create_app()) as client: for path in ("../../etc/passwd", "media/abc.png", "https://elsewhere.example.test/x.png",