diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c8f1e7c27..68750365e 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1696,6 +1696,20 @@ first, and a read scrubs as it serves, so a secret learned after a step was written is left out too. Native reads keep every secret byte for byte, so a replay of the served inputs holds what the served copy redacts. +A copy's media files, `media/.` beside the document, wait on the +agent's disk as the adapter names them, one file per sha256, with a PNG, JPEG, +GIF or WebP image's width and height read from its header. The worker pulls +them at each turn's end with the trajectory extension's `media` call, listed on +the card of an agent that serves `atif/1`: `{context_id, objects: {path: write +grant}}` uploads each file the context holds under its exact-object grant, up +to 8 at a time, checking each upload's bytes against its name. It answers what +it uploaded, the paths it holds no file for (`missing`) and the uploads that +failed (`failed`). Neither stops the rest. A file uploaded leaves the agent's +disk, as do a context's files when it is evicted or restored over, and the +rest when the server stops. A reader takes a part path only as +`media/.` and checks the bytes it reads against that name +(`agentenv_protocol.atif.media_file` and `check_media`). + `enable(..., description="...")` is reserved for declarations carrying configuration or metadata. It preserves the agent-specific extension prose published in the Agent Card. The versioned SDK definition provides a generic diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 3c0f2c28c..99910083b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -77,6 +77,8 @@ TaskObjectTrajectoryRequest, TaskTrajectoryRequest, TrajectoryEventsResponse, + TrajectoryMediaRequest, + TrajectoryMediaResponse, TrajectoryObjectsResponse, AtifEventsResponse, AtifObjectsResponse, @@ -256,6 +258,8 @@ "AtifObjectsResponse", "AtifUploadedObjects", "TrajectoryLog", + "TrajectoryMediaRequest", + "TrajectoryMediaResponse", "TrajectoryObjectsResponse", "TrajectoryState", "TrajectoryUploadedObjects", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py index 6e6790437..56ec6bc26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_atif_copy.py @@ -68,8 +68,9 @@ class AtifCopy: serves goes through ``scrub``, the agent's secrets redacted.""" def __init__(self, adapter: type[Adapter], context_id: str, *, - scrub: Callable[[Any], Any] = lambda served: served) -> None: - self._run = AdapterRun(adapter, trajectory_id=uuid.uuid4().hex) + scrub: Callable[[Any], Any] = lambda served: served, + media: Callable[[str, bytes], None] | None = None) -> None: + self._run = AdapterRun(adapter, media=media, trajectory_id=uuid.uuid4().hex) self._adapter = adapter.__name__ self._context_id = context_id self._scrub = scrub diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py new file mode 100644 index 000000000..cb7745998 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_media_spool.py @@ -0,0 +1,92 @@ +"""The media files of an agent's atif/1 copies, spooled to its disk as the adapter names them: one file per sha256, +held by the contexts whose copies reference it, until a push confirms it for a context or that context goes. The worker +pulls them at each turn's end (``TRAJECTORY_V1.media``), so the bytes never wait in memory.""" + +from __future__ import annotations + +import itertools +import logging +import os +import shutil +import tempfile +import threading +from collections.abc import Callable +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class MediaSpool: + """Each context's media files on disk. Safe to fill from the agent's thread while the loop pushes them; the folder + is made on the first file and removed by ``close``. Each hold has its own number, so an upload that outlives its + context's copy (a restore, an eviction) releases only the hold it began with, never a later copy's. A file that + can't be written, on a full disk say, is left out: a push reports it missing, and the copy goes on.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._folder: Path | None = None + #: Each context's held paths, ``media/.``, each with its hold's number. + self._held: dict[str, dict[str, int]] = {} + self._holds = itertools.count(1) + + def sink(self, context_id: str) -> Callable[[str, bytes], None]: + """Where a context's copy writes each media file once, as its builder names it.""" + + def write(path: str, data: bytes) -> None: + with self._lock: + try: + file = self._file(path) + if not file.exists(): + part = file.with_name(f"{file.name}.{os.getpid()}.part") + try: + part.write_bytes(data) + os.replace(part, file) + finally: + part.unlink(missing_ok=True) + except OSError as exc: + logger.warning("the atif/1 copy of context %s keeps no file for %s, which a push reports " + "missing: %s", context_id, path, exc) + return + self._held.setdefault(context_id, {}).setdefault(path, next(self._holds)) + + return write + + def held(self, context_id: str, path: str) -> tuple[Path, int] | None: + """The file a context holds at ``path`` and its hold's number, or None when it holds none there.""" + with self._lock: + hold = self._held.get(context_id, {}).get(path) + return None if hold is None else (self._file(path), hold) + + def release(self, context_id: str, path: str, hold: int) -> None: + """Hold ``hold`` on ``path`` is the context's no longer: a push confirmed it. A later hold stays, and the file + goes once no context holds it.""" + with self._lock: + held = self._held.get(context_id) + if held is None or held.get(path) != hold: + return + del held[path] + if not held: + del self._held[context_id] + self._remove_unheld(path) + + def drop(self, context_id: str) -> None: + """The context is gone, evicted or restored over: its files go, but those another context still holds.""" + with self._lock: + for path in self._held.pop(context_id, {}): + self._remove_unheld(path) + + def close(self) -> None: + with self._lock: + if self._folder is not None: + shutil.rmtree(self._folder, ignore_errors=True) + self._folder = None + self._held.clear() + + def _file(self, path: str) -> Path: + if self._folder is None: + self._folder = Path(tempfile.mkdtemp(prefix="agentenv-atif-media-")) + return self._folder / path.removeprefix("media/") + + def _remove_unheld(self, path: str) -> None: + if self._folder is not None and not any(path in held for held in self._held.values()): + (self._folder / path.removeprefix("media/")).unlink(missing_ok=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py index a8580de35..7d8c3cc5e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_secrets.py @@ -43,6 +43,9 @@ #: token is: longer than an ordinary path's words, unlike ``MIN_SECRET_LENGTH``, which bounds what is scrubbed at all. MIN_TOKEN_SEGMENT_LENGTH = 16 _TOKEN_SEGMENT = re.compile(rf"[A-Za-z0-9_.=~+-]{{{MIN_TOKEN_SEGMENT_LENGTH},}}") +#: A media file's name, ``.``: a grant's path may name the file it writes, which names its bytes, not a +#: credential, and which every atif/1 copy that holds the file must keep. +_MEDIA_NAME = re.compile(r"[0-9a-f]{64}\.[A-Za-z0-9]+") T = TypeVar("T") @@ -245,7 +248,8 @@ def _url_secrets(url: str) -> list[str]: parts = urlsplit(url) carried = _query_values(url) if not carried: - carried = [segment for segment in parts.path.split("/") if _TOKEN_SEGMENT.fullmatch(segment)] + carried = [segment for segment in parts.path.split("/") + if _TOKEN_SEGMENT.fullmatch(segment) and not _MEDIA_NAME.fullmatch(segment)] return [*carried, *([parts.password] if parts.password else [])] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py index 70a09c901..53beecc59 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py @@ -10,6 +10,7 @@ from ..atif import Adapter from ._atif_copy import AtifCopy, AtifPage, TurnStart +from ._media_spool import MediaSpool from ._turn_index import TurnIndex from .extensions import TrajectoryState from .tasks.v1 import TaskError, TrajectoryLog @@ -98,6 +99,8 @@ def __init__( self._turns = TurnIndex() if atif is not None else None #: What every atif/1 read goes through: the agent's secrets redacted. self._scrub = scrub + #: The copies' media files, on disk until the worker pulls them. + self.media = MediaSpool() self._tasks: dict[str, _TaskRecord] = {} self._contexts: OrderedDict[str, list[str]] = OrderedDict() self._copies: dict[str, AtifCopy] = {} @@ -133,7 +136,8 @@ def start(self, task_id: str, turn: TurnStart | None = None) -> None: copy = self._copies.get(record.context_id) if copy is None: copy = self._copies[record.context_id] = AtifCopy( - self._atif, record.context_id, **({"scrub": self._scrub} if self._scrub is not None else {}) + self._atif, record.context_id, media=self.media.sink(record.context_id), + **({"scrub": self._scrub} if self._scrub is not None else {}), ) copy.start(task_id, self._turns.next(record.context_id), turn) record.log._atif = _CopySink(copy, task_id) @@ -149,6 +153,7 @@ def restored(self, context_id: str, turns: int) -> None: new document, since the conversation is the snapshot's now; an agent that serves atif/1 only.""" self._turns.set(context_id, turns) self._copies.pop(context_id, None) + self.media.drop(context_id) def request_cancel(self, task_id: str) -> None: """Make ``canceled`` the outcome the task is sealed with, whatever ``run`` returns.""" @@ -357,6 +362,7 @@ def _evict(self, *, keep: str) -> None: continue del self._contexts[context_id] self._copies.pop(context_id, None) + self.media.drop(context_id) for task_id in task_ids: del self._tasks[task_id] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index 8de0a8f16..7058649f1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -25,7 +25,7 @@ ) from ..atif import FORMAT as ATIF_FORMAT -from ..atif import Adapter, AdapterSpec +from ..atif import Adapter, AdapterSpec, media_file from ._triggers import MAX_SOLVER_MESSAGE_LENGTH from .tasks.v1 import AgentConfig from ..transfers import ( @@ -205,6 +205,34 @@ class TrajectoryObjectsResponse(ExtensionResponse): format: str | None = None +#: The most media files one media call may name: a turn of a computer-use run holds a few hundred screenshots. +MAX_MEDIA_OBJECTS = 10_000 + + +class TrajectoryMediaRequest(ExtensionRequest): + """The media files of a context's atif/1 copy the caller wants, by their paths in it (``media/.``), + each under its own exact-object write grant.""" + + context_id: str = Field(min_length=1) + objects: dict[str, WriteObject] = Field(min_length=1, max_length=MAX_MEDIA_OBJECTS) + + @field_validator("objects") + @classmethod + def _media_paths(cls, objects: dict[str, WriteObject]) -> dict[str, WriteObject]: + for path in objects: + media_file(path) + return objects + + +class TrajectoryMediaResponse(ExtensionResponse): + """What a media call uploaded, by path; the paths the context holds no file for; and the uploads that failed, with + why. Neither of the last two stops the rest.""" + + objects: dict[str, Uploaded] + missing: list[str] = Field(default_factory=list) + failed: dict[str, str] = Field(default_factory=dict) + + class AtifUploadedObjects(ExtensionResponse): trajectory: Uploaded aux: Uploaded | None = None @@ -816,6 +844,9 @@ class OperationDefinition: implementation: ImplementationOwner request: RequestDefinition | type[BaseModel] | None = None response: FieldSchema | type[BaseModel] | None = None + #: Served by the SDK alone, from state only it holds: no agent overrides it, and an agent that overrides the + #: extension's other SDK operations needn't override it with them. + sdk_only: bool = False response_model: type[BaseModel] | None = field(default=None, init=False) def __post_init__(self) -> None: @@ -1104,6 +1135,7 @@ def _op( response_required: Iterable[str] = (), response_optional: Iterable[str] = (), variants: Iterable[RequestVariant] = (), + sdk_only: bool = False, ) -> OperationDefinition: response_required = _tuple(response_required) response_optional = _tuple(response_optional) @@ -1123,6 +1155,7 @@ def _op( implementation=owner, request=request, response=response, + sdk_only=sdk_only, ) @@ -1194,6 +1227,8 @@ def _configure_trajectory( return ExtensionConfiguration( wire_params={"formats": formats, "native": native, "default": native} if native else {}, options={"native": native, **({"atif": atif} if atif is not None else {})} if native else {}, + # An agent that serves atif/1 hands over its copies' media files, which the worker pulls at each turn's end. + features={"media"} if atif is not None else frozenset(), variants=( {"get": {"task_events", "context_events", "context", "context_objects"}} if live else {} ), @@ -1341,6 +1376,22 @@ def _configure_trajectory( ), ) }, + optional_features={ + "media": OperationGroup( + operations={ + "media": _op( + "media", + "POST", + "/ext/trajectory/media", + SDK, + request_model=TrajectoryMediaRequest, + response_model=TrajectoryMediaResponse, + # The spool it reads is the SDK's, so an agent that serves its own trajectory reads still has it. + sdk_only=True, + ), + } + ) + }, configuration_validator=_configure_trajectory, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index 63f954de7..982a26833 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -4,6 +4,7 @@ import asyncio import dataclasses +import hashlib import inspect import json import logging @@ -68,6 +69,8 @@ AtifObjectsResponse, AtifUploadedObjects, TrajectoryEventsResponse, + TrajectoryMediaRequest, + TrajectoryMediaResponse, TrajectoryObjectsResponse, TrajectoryState, TrajectoryUploadedObjects, @@ -75,6 +78,7 @@ TriggerRegisterRequest, enable, ) +from ..atif import media_file from ..computer.errors import ComputerError from ..errors import AgentEnvError from ..types import error_body @@ -101,7 +105,7 @@ from .tasks.v1 import ( TextPart as TaskTextPart, ) -from ..transfers import TransferError, serving_on, upload +from ..transfers import TransferError, Uploaded, serving_on, upload logger = logging.getLogger(__name__) @@ -125,6 +129,10 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: _AGENT_DEFINITION = "_agentenv_a2a_definition" _AgentT = TypeVar("_AgentT", bound="AgentEnvAgent") _MAX_CACHED_CONTEXTS = 1_024 +#: How many of a media call's files upload at once. +MAX_PARALLEL_MEDIA_PUSHES = 8 +#: How much of a spooled media file is read at a time to hash it before its upload. +_MEDIA_HASH_BLOCK_BYTES = 1 << 20 def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: @@ -146,6 +154,14 @@ def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: return result +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for block in iter(lambda: file.read(_MEDIA_HASH_BLOCK_BYTES), b""): + digest.update(block) + return digest.hexdigest() + + def _config_write_only_wire_names(config: type[AgentConfig]) -> set[str]: """Return wire names explicitly marked as write-only in the config schema.""" wire_names = _config_wire_names(config) @@ -682,6 +698,7 @@ def handlers(self) -> dict[tuple[str, str], Callable]: (PEER_AGENTS_V1.uri, "list"): self.peer_agents_list, (SKILL_CONFIG_V1.uri, "list"): self.skill_list, (TRAJECTORY_V1.uri, "get"): self.trajectory_get, + (TRAJECTORY_V1.uri, "media"): self.trajectory_media, (TRIGGERS_V1.uri, "register"): self.triggers_register, (TRIGGERS_V1.uri, "decide"): self.triggers_decide, (TRIGGERS_V1.uri, "state"): self.triggers_state, @@ -964,6 +981,50 @@ async def _atif_document( ) return AtifObjectsResponse(objects=uploaded, **versions).model_dump(mode="json", exclude_none=True) + async def trajectory_media(self, request: TrajectoryMediaRequest) -> dict[str, Any]: + """Upload the media files of a context's atif/1 copy the worker asks for, each under its grant and from the + spool, ``MAX_PARALLEL_MEDIA_PUSHES`` at a time, checking each upload's bytes against its name. A path the + context holds no file for is ``missing``, an upload that fails is ``failed``, and neither stops the rest; a + file uploaded leaves the spool.""" + media = self.task_trajectories.media + uploaded: dict[str, Uploaded] = {} + missing: list[str] = [] + failed: dict[str, str] = {} + gate = asyncio.Semaphore(MAX_PARALLEL_MEDIA_PUSHES) + + async def push(path: str, target: Any) -> None: + held = media.held(request.context_id, path) + if held is None: + missing.append(path) + return + file, hold = held + async with gate: + # Checked before the bytes leave, so a spool file that no longer matches its name never reaches the + # store, and after, for bytes that changed in transit. + try: + digest = await asyncio.to_thread(_sha256_file, file) + except OSError as exc: + failed[path] = f"the spooled file can't be read: {exc.strerror}" + return + if digest != media_file(path): + failed[path] = f"the spooled file's sha256 is {digest}" + return + try: + done = await upload(target, file) + except TransferError as exc: + failed[path] = f"{exc.code}: {exc}" + return + if done.sha256 is not None and done.sha256 != digest: + failed[path] = f"the upload's sha256 is {done.sha256}" + return + uploaded[path] = done + media.release(request.context_id, path, hold) + + await asyncio.gather(*(push(path, target) for path, target in request.objects.items())) + return TrajectoryMediaResponse(objects=uploaded, missing=sorted(missing), failed=failed).model_dump( + mode="json", exclude_none=True + ) + def check_trajectory_format(self, format: str | None) -> None: """Refuse a trajectory read naming a format the card doesn't offer, before any handler runs, the agent's own included.""" @@ -1128,7 +1189,8 @@ def _taking_the_control_token( inner: Callable[[Starlette], AbstractAsyncContextManager] | None, ) -> Callable[[Starlette], AbstractAsyncContextManager]: """The application's lifespan, ``inner``'s if it has one: serving starts by taking the agent's control token out of - the environment, and the extension routes take it from then on. An application that isn't served stays open.""" + the environment, and the extension routes take it from then on, and ends by removing the media spool. An + application that isn't served stays open.""" @asynccontextmanager async def lifespan(app: Starlette) -> AsyncIterator[Any]: @@ -1136,11 +1198,15 @@ async def lifespan(app: Starlette) -> AsyncIterator[Any]: log_guard(application.control_token) if application.services.secrets is not None: application.services.secrets.add(application.control_token, source="the control token") - if inner is None: - yield None - else: - async with inner(app) as state: - yield state + try: + if inner is None: + yield None + else: + async with inner(app) as state: + yield state + finally: + # The media the worker didn't pull go with the server. + application.services.task_trajectories.media.close() return lifespan @@ -1354,7 +1420,7 @@ async def invoke(request: Request) -> JSONResponse: extension_request = request_model.model_validate(payload) if self.services.secrets is not None: self.services.secrets.add_request(extension_request) - if extension_uri == TRAJECTORY_V1.uri: + if extension_uri == TRAJECTORY_V1.uri and operation.definition.name == "get": self.services.check_trajectory_format(extension_request.format) if extension_request.format == ATIF_FORMAT: # The SDK serves atif/1 from its own copy, context reads an agent binds included. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py index c0a4ea7be..1c560ff8a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/registry.py @@ -304,6 +304,8 @@ def build_registry( variant_handlers[key] = handler continue + if operation.sdk_only: + raise ValueError(f"{uri}.{operation.name} is served by the SDK alone; an agent can't override it") key = (uri, binding.operation) if key in operation_handlers: raise ValueError(f"multiple handlers for {uri}.{operation.name}") @@ -424,7 +426,7 @@ def build_registry( active_sdk_operations = { name for name, operation in resolved_operations.items() - if operation.definition.implementation is ImplementationOwner.SDK + if operation.definition.implementation is ImplementationOwner.SDK and not operation.definition.sdk_only } overridden_sdk_operations = { name diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 709942e9e..811658112 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -108,7 +108,10 @@ One vocabulary, in three places: `[video: media/.mp4, 4.4 MB, 31 s]`. That covers video, documents, images other than JPEG, PNG, GIF and WebP (such as HEIC or SVG), audio outside ATIF's eight types, and other files. - Each file's details go in the `media` map of the profile of the object that holds the part (Step for a message, - ObservationResult for a result), keyed by its path. + ObservationResult for a result), keyed by its path. A PNG, JPEG, GIF or WebP image's `width` and `height` are read + from its header when the producer gives none. +- A reader takes a part path only as `media/.` in the document's own folder, never another path or a URL, + and checks each file's bytes against its name (`media_file`, `check_media`). ## Fields diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index b66f06e58..85bfb47d5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -46,16 +46,16 @@ read_profile, ) from .kit import KIT_RULES, KitResult, check_adapter -from .reader import Task, check, is_prefix, load, media_paths, strip_profile, tasks +from .reader import MEDIA_PATH, Task, check, check_media, is_prefix, load, media_file, media_paths, strip_profile, tasks from .runner import AdapterRun, Fed, RecordedTask, replay from .served import AuxEntry, ContentItem, MediaItem, ServedData, ServedError, ServedMedia, ServedTurn from .validator import RULES, Finding, Rule, errors_in, validate, validate_live __all__ = [ "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", - "AtifError", "AtifTurn", "AudioSource", "aux", "AuxEntry", "check", "check_adapter", "ContentItem", "ContentPart", + "AtifError", "AtifTurn", "AudioSource", "aux", "AuxEntry", "check", "check_adapter", "check_media", "ContentItem", "ContentPart", "Document", "dump_profile", "errors_in", "Fed", "FinalMetrics", "FinalMetricsProfile", "Finding", "finish", - "FORMAT", "ImageSource", "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "media_paths", "MediaInfo", + "FORMAT", "ImageSource", "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "MEDIA_PATH", "media_file", "media_paths", "MediaInfo", "MediaItem", "Metrics", "MetricsProfile", "Native", "native", "Observation", "ObservationResult", "ObservationResultProfile", "PROFILE_KEY", "profile_of", "PROFILE_VERSION", "PROFILES", "RAW_ARGUMENTS", "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", "RULES", "SCHEMA_VERSION", "Selector", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py new file mode 100644 index 000000000..e5c1eddd9 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/_image_size.py @@ -0,0 +1,75 @@ +"""An image's width and height, read from its header for the four image types ATIF has parts for, so the profile can +carry them without Pillow. A header this can't read gives None: the image is stored all the same.""" + +from __future__ import annotations + +import struct + +_PNG = b"\x89PNG\r\n\x1a\n" +#: JPEG's start-of-frame markers, the segments that hold the frame's size: every SOFn but DHT, JPG and DAC. +_JPEG_FRAMES = frozenset({0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}) +#: JPEG markers that stand alone, with no length after them. +_JPEG_BARE = frozenset({0x01, *range(0xD0, 0xD9)}) + + +def image_size(data: bytes, media_type: str) -> tuple[int, int] | None: + """``(width, height)`` of a PNG, JPEG, GIF or WebP image from its header, or None.""" + read = {"image/png": _png, "image/jpeg": _jpeg, "image/gif": _gif, "image/webp": _webp}.get(media_type) + try: + size = read(data) if read is not None else None + except (IndexError, struct.error): + return None + return size if size is not None and size[0] > 0 and size[1] > 0 else None + + +def _png(data: bytes) -> tuple[int, int] | None: + if data[:8] != _PNG or data[12:16] != b"IHDR": + return None + return struct.unpack(">II", data[16:24]) + + +def _gif(data: bytes) -> tuple[int, int] | None: + if data[:6] not in (b"GIF87a", b"GIF89a"): + return None + return struct.unpack(" tuple[int, int] | None: + if data[:2] != b"\xff\xd8": + return None + at = 2 + while at + 4 <= len(data): + if data[at] != 0xFF: + return None + marker = data[at + 1] + if marker == 0xFF: + at += 1 + continue + if marker in _JPEG_BARE: + at += 2 + continue + (length,) = struct.unpack(">H", data[at + 2:at + 4]) + if marker in _JPEG_FRAMES: + height, width = struct.unpack(">HH", data[at + 5:at + 9]) + return width, height + if marker == 0xDA: + return None + at += 2 + length + return None + + +def _webp(data: bytes) -> tuple[int, int] | None: + if data[:4] != b"RIFF" or data[8:12] != b"WEBP": + return None + chunk = data[12:16] + if len(data) < 30: + return None + if chunk == b"VP8 " and data[23:26] == b"\x9d\x01\x2a": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + if chunk == b"VP8X": + return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1 + return None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py index cd7748390..d01d6a8dc 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/builder.py @@ -16,6 +16,7 @@ from pydantic import ValidationError +from ._image_size import image_size from .adapter import AdapterSpec from .models.content import AudioMediaType, ContentPart, ImageSource from .profile import ( @@ -418,6 +419,8 @@ def store(self, data: bytes, media_type: str, info: Mapping[str, Any]) -> Media: for key in ("width", "height", "duration_ms", "pages"): _int(info.get(key), key) _str(info.get("name"), "a media file's name") + if info.get("width") is None and info.get("height") is None and (size := image_size(bytes(data), media_type)): + info = {**info, "width": size[0], "height": size[1]} digest = hashlib.sha256(data).hexdigest() extension = _EXTENSIONS.get(media_type) or mimetypes.guess_extension(media_type, strict=False) or ".bin" path = f"media/{digest}{extension}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py index a8dc625d9..fe58598fd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py @@ -4,7 +4,9 @@ from __future__ import annotations import copy +import hashlib import json +import re from dataclasses import dataclass, field from pathlib import Path from typing import Any, Iterator, Mapping @@ -93,6 +95,27 @@ def _parts(value: Any) -> list[Mapping[str, Any]]: return [p for p in value if isinstance(p, Mapping)] if isinstance(value, list) else [] +#: The only path a media file of a document can have: ``media/.``, in the document's own folder. +MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") + + +def media_file(path: str) -> str: + """``path``'s sha256, once ``path`` is ``media/.``: a part path resolves only to a file in the + document's own media folder, never to another path or a URL. AtifError otherwise.""" + named = MEDIA_PATH.match(path) if isinstance(path, str) else None + if named is None: + raise AtifError(f"{path!r} isn't media/. in the document's folder") + return named["sha256"] + + +def check_media(path: str, data: bytes) -> None: + """Refuse media whose bytes aren't what its name says: a reader checks each file it reads, since a store can hold + another file under that name. AtifError naming the difference.""" + digest = hashlib.sha256(data).hexdigest() + if media_file(path) != digest: + raise AtifError(f"{path}'s bytes have sha256 {digest}") + + def media_paths(document: Mapping[str, Any]) -> list[str]: """Every media file the document names, once each, in the order they appear: image and audio parts' paths, and the files the profile records for media ATIF has no part type for.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 2a18737c5..7675a0200 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -26,12 +26,11 @@ StepError, profile_of, ) -from .reader import is_prefix, tasks +from .reader import MEDIA_PATH, is_prefix, tasks ERROR = "error" WARNING = "warning" -MEDIA_PATH = re.compile(r"^media/(?P[0-9a-f]{64})\.[A-Za-z0-9]+$") TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3}|\.\d{6})?Z$") _COUNTS = (("prompt_tokens", "total_prompt_tokens"), ("completion_tokens", "total_completion_tokens"), ("cached_tokens", "total_cached_tokens")) diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_media.py b/packages/agentenv-protocol/tests/test_a2a_atif_media.py new file mode 100644 index 000000000..5bc380ce6 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_atif_media.py @@ -0,0 +1,393 @@ +"""An agent that serves atif/1 spools its copies' media files to its disk, one file per sha256, with each image's size +read from its header, and uploads the ones the worker asks for under exact-object grants (``TRAJECTORY_V1.media``), +which is how the worker pulls a turn's media at its end. Readers take a part path only as media/., and +only with the bytes that name.""" +from __future__ import annotations + +import base64 +import hashlib +import struct +import zlib +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +from starlette.testclient import TestClient +from test_a2a_atif_serving import _atif, _ended, _send + +from agentenv_protocol.a2a_agent import ( + TRAJECTORY_V1, + TaskObjectTrajectoryRequest, + TaskTrajectoryRequest, + AgentEnvAgent, + AgentIdentity, + TaskRequest, + TaskResult, + TextPart, + TrajectoryState, + Uploaded, + a2a_agent, + enable, + extension, + framework, +) +from agentenv_protocol.a2a_agent._media_spool import MediaSpool +from agentenv_protocol.a2a_agent._atif_copy import TurnStart +from agentenv_protocol.a2a_agent._trajectory_log import TaskTrajectories +from agentenv_protocol.atif import Adapter, AtifError, atif_adapter, check_media, errors_in, media_file, native, validate +from agentenv_protocol.atif._image_size import image_size +from agentenv_protocol.transfers import TransferError + +NATIVE = "shots/1" + + +def _png(width: int, height: int) -> bytes: + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) + + +SHOT = _png(4, 3) + + +@atif_adapter(native_format=NATIVE, version="1", input_includes_cache=False) +class Shots(Adapter): + @native(type="shot") + def shot(self, record: dict[str, Any], turn: Any) -> None: + data = base64.b64decode(record["data"]) + media = turn.media(data, record["media_type"], **record.get("info", {})) + turn.message(step=record["step"], content=[media]) + turn.step(key=record["step"], llm_call_count=1) + + +def _shooting(*, atif: bool = True) -> AgentEnvAgent: + """An agent whose turn appends each base64 file its prompt's JSON lists as a native ``shot``.""" + trajectory = enable(TRAJECTORY_V1, live=True, native=NATIVE, **({"atif": Shots} if atif else {})) + + @a2a_agent(identity=AgentIdentity(name="shots", description="test", version="1"), extensions=(trajectory,)) + class Shooting(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + request.trajectory.set_format(NATIVE) + for index, part in enumerate(request.parts[1:]): + request.trajectory.append({"type": "shot", "step": f"s{index}", "data": part.bytes, + "media_type": part.mime_type}) + return TaskResult.text("done") + + return Shooting() + + +def _file(data: bytes, media_type: str, name: str = "file") -> dict[str, Any]: + return {"kind": "file", "file": {"bytes": base64.b64encode(data).decode(), "mimeType": media_type, "name": name}} + + +def _grant(path: str, media_type: str = "image/png") -> dict[str, Any]: + expires = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(timespec="seconds").replace("+00:00", "Z") + return {"media_type": media_type, "max_bytes": 10_000_000, + "write": {"kind": "http-put", "url": f"https://objects.example.test/{path}", "expires_at": expires}} + + +def _path(data: bytes, extension: str) -> str: + return f"media/{hashlib.sha256(data).hexdigest()}{extension}" + + +@pytest.fixture +def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, bytes]: + """Where the media calls' grants upload: each object by its path.""" + stored: dict[str, bytes] = {} + + async def upload(target, source) -> Uploaded: + data = Path(source).read_bytes() + stored[target.write.url.removeprefix("https://objects.example.test/")] = data + return Uploaded(size_bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) + + monkeypatch.setattr(framework, "upload", upload) + return stored + + +def _media(client: TestClient, paths: dict[str, str], context_id: str = "ctx-1") -> Any: + return client.post("/ext/trajectory/media", json={"context_id": context_id, "objects": { + path: _grant(path, media_type) for path, media_type in paths.items()}}) + + +def test_a_copys_images_are_spooled_by_sha256_with_their_size_and_pushed_once_asked(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png", "inbound.png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + path = _path(SHOT, ".png") + held, _ = spool.held("ctx-1", path) + assert held.read_bytes() == SHOT + pushed = _media(client, {path: "image/png"}) + assert pushed.status_code == 200, pushed.text + assert spool.held("ctx-1", path) is None and not held.exists() + assert errors_in(validate(document)) == [] + [user] = [step for step in document["steps"] if step["source"] == "user"] + image = next(part for part in user["message"] if isinstance(part, dict) and part.get("type") == "image") + assert image["source"]["path"] == path + assert user["extra"]["agentenv"]["media"][path] == {"sha256": hashlib.sha256(SHOT).hexdigest(), + "bytes": len(SHOT), "width": 4, "height": 3, "name": "inbound.png"} + assert pushed.json() == {"objects": {path: {"size_bytes": len(SHOT), "sha256": hashlib.sha256(SHOT).hexdigest()}}, + "missing": [], "failed": {}} + assert store == {path: SHOT} + + +def test_a_media_calls_grants_leave_the_copys_media_paths_as_they_are(store: dict[str, bytes]) -> None: + """A grant's URL may name the file it writes, ``media/.``: the scrub takes a URL's credential from it, + never the file's name, which every read of the copy keeps.""" + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + assert _media(client, {_path(SHOT, ".png"): "image/png"}).status_code == 200 + document = _atif(client, task_id=task_id)["trajectory"] + [user] = [step for step in document["steps"] if step["source"] == "user"] + assert any(part.get("source", {}).get("path") == _path(SHOT, ".png") for part in user["message"] + if isinstance(part, dict)) + + +def test_the_same_bytes_added_twice_are_spooled_and_pushed_once(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(SHOT, "image/png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + folder = client.app.state.agentenv_a2a.services.task_trajectories.media._folder + assert sorted(file.name for file in folder.iterdir()) == [_path(SHOT, ".png").removeprefix("media/")] + pushed = _media(client, {_path(SHOT, ".png"): "image/png"}) + paths = [part["source"]["path"] for step in document["steps"] for part in step.get("message") or [] + if isinstance(part, dict) and part.get("type") == "image"] + assert paths == [_path(SHOT, ".png")] * 4 + assert list(pushed.json()["objects"]) == [_path(SHOT, ".png")] and len(store) == 1 + + +def test_a_path_the_context_doesnt_hold_and_a_failed_upload_dont_stop_the_rest( + store: dict[str, bytes], monkeypatch: pytest.MonkeyPatch) -> None: + other = _png(2, 2) + stored = framework.upload + + async def upload(target, source) -> Uploaded: + if target.write.url.endswith(_path(other, ".png")): + raise TransferError("grant_expired", "The transfer grant expired.") + return await stored(target, source) + + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(other, "image/png")]) + _ended(client, task_id) + monkeypatch.setattr(framework, "upload", upload) + never = _path(b"never added", ".png") + pushed = _media(client, {_path(SHOT, ".png"): "image/png", _path(other, ".png"): "image/png", + never: "image/png"}).json() + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + assert spool.held("ctx-1", _path(other, ".png")) is not None + other_context = _media(client, {_path(SHOT, ".png"): "image/png"}, context_id="ctx-2").json() + assert list(pushed["objects"]) == [_path(SHOT, ".png")] and pushed["missing"] == [never] + assert pushed["failed"] == {_path(other, ".png"): "grant_expired: The transfer grant expired."} + assert other_context == {"objects": {}, "missing": [_path(SHOT, ".png")], "failed": {}} + + +def test_an_agent_that_serves_its_own_trajectory_reads_still_has_the_sdks_media_and_cant_take_it_over( + store: dict[str, bytes]) -> None: + @a2a_agent(identity=AgentIdentity(name="own-reads", description="test", version="1"), + extensions=(enable(TRAJECTORY_V1, native=NATIVE, atif=Shots),)) + class OwnReads(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + request.trajectory.set_format(NATIVE) + return TaskResult.text("done") + + @extension(TRAJECTORY_V1.get) + async def trajectory(self, request: TaskTrajectoryRequest | TaskObjectTrajectoryRequest) -> dict[str, Any]: + return {"trajectory": []} + + with TestClient(OwnReads().create_app()) as client: + _ended(client, _send(client, "look", "m-1", extra_parts=[_file(SHOT, "image/png")])) + assert list(_media(client, {_path(SHOT, ".png"): "image/png"}).json()["objects"]) == [_path(SHOT, ".png")] + + @a2a_agent(identity=AgentIdentity(name="takes-media", description="test", version="1"), + extensions=(enable(TRAJECTORY_V1, live=True, native=NATIVE, atif=Shots),)) + class TakesMedia(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + return TaskResult.text("done") + + @extension(TRAJECTORY_V1.media.media) + async def media(self, request: Any) -> dict[str, Any]: + return {"objects": {}} + + with pytest.raises(ValueError, match="served by the SDK alone"): + TakesMedia().create_app() + + +def test_an_upload_that_outlives_its_copy_releases_only_the_hold_it_began_with() -> None: + spool = MediaSpool() + path = _path(SHOT, ".png") + spool.sink("ctx")(path, SHOT) + _, old = spool.held("ctx", path) + spool.drop("ctx") + spool.sink("ctx")(path, SHOT) + spool.release("ctx", path, old) + file, new = spool.held("ctx", path) + assert new != old and file.read_bytes() == SHOT + spool.release("ctx", path, new) + assert spool.held("ctx", path) is None and not file.exists() + spool.close() + + +def test_a_spooled_file_that_no_longer_matches_its_name_never_reaches_the_store(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + file, _ = spool.held("ctx-1", _path(SHOT, ".png")) + file.write_bytes(b"changed on disk") + pushed = _media(client, {_path(SHOT, ".png"): "image/png"}).json() + assert pushed["objects"] == {} and store == {} + assert pushed["failed"][_path(SHOT, ".png")].startswith("the spooled file's sha256 is ") + + +def test_a_file_the_spool_cant_write_is_reported_missing_and_the_copy_goes_on( + store: dict[str, bytes], tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + other = _png(2, 2) + with TestClient(_shooting().create_app()) as client: + spool = client.app.state.agentenv_a2a.services.task_trajectories.media + spool._folder = tmp_path / "removed" + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + spool._folder = None + task_id = _send(client, "again", "m-2", extra_parts=[_file(other, "image/png")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + pushed = _media(client, {_path(SHOT, ".png"): "image/png", _path(other, ".png"): "image/png"}).json() + assert pushed == {"objects": {_path(other, ".png"): {"size_bytes": len(other), + "sha256": hashlib.sha256(other).hexdigest()}}, + "missing": [_path(SHOT, ".png")], "failed": {}} + paths = {part["source"]["path"] for step in document["steps"] for part in step.get("message") or [] + if isinstance(part, dict) and part.get("type") == "image"} + assert paths == {_path(SHOT, ".png"), _path(other, ".png")} + assert f"keeps no file for {_path(SHOT, '.png')}" in caplog.text + + +def test_a_media_call_takes_only_media_paths() -> None: + with TestClient(_shooting().create_app()) as client: + for path in ("../../etc/passwd", "media/abc.png", "https://elsewhere.example.test/x.png", + f"media/{'a' * 64}/x.png"): + refused = _media(client, {path: "image/png"}) + assert refused.status_code == 400, (path, refused.text) + + +def test_only_an_agent_that_serves_atif_offers_media() -> None: + def methods(agent: AgentEnvAgent) -> set[str]: + card = TestClient(agent.create_app()).get("/.well-known/agent-card.json").json() + [trajectory] = [ext for ext in card["capabilities"]["extensions"] if ext["uri"] == TRAJECTORY_V1.uri] + return set(trajectory["params"]["methods"]) + + assert methods(_shooting()) == {"get", "media"} + assert methods(_shooting(atif=False)) == {"get"} + + +def test_a_video_and_a_heic_image_are_spooled_and_pushed_like_any_file(store: dict[str, bytes]) -> None: + video, heic = b"\x00\x00\x00\x18ftypmp42" + b"v" * 64, b"\x00\x00\x00\x18ftypheic" + b"h" * 64 + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(video, "video/mp4"), _file(heic, "image/heic")]) + _ended(client, task_id) + document = _atif(client, task_id=task_id)["trajectory"] + pushed = _media(client, {_path(video, ".mp4"): "video/mp4", _path(heic, ".heic"): "image/heic"}).json() + assert errors_in(validate(document)) == [] + [user] = [step for step in document["steps"] if step["source"] == "user"] + # ATIF has no part for either, so the typed content names each file in a plain note. + assert f"[video: {_path(video, '.mp4')}, 76 B]" in user["message"] + assert f"[image: {_path(heic, '.heic')}, 76 B]" in user["message"] + assert sorted(pushed["objects"]) == sorted([_path(video, ".mp4"), _path(heic, ".heic")]) + assert store[_path(video, ".mp4")] == video and store[_path(heic, ".heic")] == heic + + +def test_a_context_evicted_or_restored_over_drops_its_media_but_what_another_holds() -> None: + trajectories = TaskTrajectories(max_contexts=1, native=NATIVE, atif=Shots) + + def play(task_id: str, context_id: str, data: bytes) -> None: + log = trajectories.register(task_id, context_id) + trajectories.start(task_id, TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", + agent_version="1", agent_model=None)) + log.append({"type": "shot", "step": "s", "data": base64.b64encode(data).decode(), "media_type": "image/png"}) + trajectories.seal(task_id, TrajectoryState.COMPLETED, final=(TextPart(text="done"),)) + + other = _png(1, 1) + play("a1", "context-a", SHOT) + file, _ = trajectories.media.held("context-a", _path(SHOT, ".png")) + play("b1", "context-b", other) + assert trajectories.media.held("context-a", _path(SHOT, ".png")) is None and not file.exists() + play("b2", "context-b", SHOT) + trajectories.restored("context-b", 0) + assert trajectories.media.held("context-b", _path(other, ".png")) is None + trajectories.media.close() + + +def test_the_spool_goes_with_the_server(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) + folder = client.app.state.agentenv_a2a.services.task_trajectories.media._folder + assert folder.is_dir() + assert not folder.exists() + + +def test_the_native_trajectory_is_unchanged_by_media(store: dict[str, bytes]) -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + served = client.post("/ext/trajectory", json={"task_id": task_id}).json()["trajectory"] + assert served == [{"type": "shot", "step": "s0", "data": base64.b64encode(SHOT).decode(), "media_type": "image/png"}] + + +def _jpeg(width: int, height: int) -> bytes: + app0 = b"\xff\xe0" + struct.pack(">H", 16) + b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00" + sof = b"\xff\xc0" + struct.pack(">HBHHB", 11, 8, height, width, 1) + b"\x01\x11\x00" + return b"\xff\xd8" + app0 + sof + b"\xff\xda\x00\x08\x01\x01\x00\x00\x3f\x00" + b"\x00" * 8 + b"\xff\xd9" + + +def _webp(kind: bytes, body: bytes) -> bytes: + chunk = kind + struct.pack(" None: + assert image_size(data, media_type) == size + + +def test_dimensions_an_adapter_passes_win_over_the_header() -> None: + trajectories = TaskTrajectories(native=NATIVE, atif=Shots) + log = trajectories.register("t1", "ctx") + trajectories.start("t1", TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", + agent_version="1", agent_model=None)) + log.append({"type": "shot", "step": "s", "data": base64.b64encode(SHOT).decode(), "media_type": "image/png", + "info": {"width": 40, "height": 30}}) + trajectories.seal("t1", TrajectoryState.COMPLETED, final=(TextPart(text="done"),)) + document = trajectories.atif_task("t1").document + media = next(step["extra"]["agentenv"]["media"] for step in document["steps"] if step["source"] == "agent") + assert (media[_path(SHOT, ".png")]["width"], media[_path(SHOT, ".png")]["height"]) == (40, 30) + trajectories.media.close() + + +def test_a_reader_takes_a_part_path_only_as_media_sha256_and_only_with_the_bytes_that_name() -> None: + path = _path(SHOT, ".png") + assert media_file(path) == hashlib.sha256(SHOT).hexdigest() + check_media(path, SHOT) + for bad in ("../media/x.png", "media/abc.png", "https://example.test/media/x.png", f"/{path}", f"{path}/x"): + with pytest.raises(AtifError): + media_file(bad) + with pytest.raises(AtifError, match="bytes have sha256"): + check_media(path, SHOT + b"tampered") diff --git a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py index 1b6f44491..ed3125720 100644 --- a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py +++ b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py @@ -5,24 +5,30 @@ the last none. The echo agent declares its native format and an ATIF adapter, so its card offers both; its native reads name their format, its atif/1 reads are valid ATIF, and a read naming another format is refused with a code agent-env reads. A snapshot of it carries its atif/1 copy's turn count to the fresh agent it is restored into, and its atif/1 -reads leave out the model key a command it runs prints, which its native read keeps. Needs Docker and a throwaway local -registry.""" +reads leave out the model key a command it runs prints, which its native read keeps. An image sent to it is spooled +as its copy's media file, and pushed under an exact-object grant from the local store's grant server. Needs Docker and a +throwaway local registry.""" +import base64 +import hashlib import json import re +import struct import shutil import socket import subprocess import time import uuid +import zlib import httpx import pytest -from agentenv_protocol.atif import FORMAT as ATIF, PROFILE_VERSION, SCHEMA_VERSION, errors_in, validate +from agentenv_protocol.atif import FORMAT as ATIF, PROFILE_VERSION, SCHEMA_VERSION, errors_in, media_paths, validate from agent_env.a2a_agent import A2AAgent -from agent_env.a2a_agent.protocol import raise_for_extension_status +from agent_env.a2a_agent.object_transfer import write_object +from agent_env.a2a_agent.protocol import poll_a2a_task, raise_for_extension_status, send_a2a_message from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.artifact.store import reset_artifact_store from agent_env.config import configure, get_config, reset_config, set_image_store @@ -256,3 +262,47 @@ async def test_a_model_key_a_command_prints_is_redacted_in_atif_and_kept_in_the_ served = json.dumps(atif) assert key not in served and f"[redacted:{len(key)}]" in served assert errors_in(validate(atif["trajectory"])) == [] + + +def _png(width: int, height: int) -> bytes: + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) + + +@pytest.mark.asyncio +async def test_a_deployed_agent_pushes_its_copys_media_under_exact_object_grants(local_registry): + suffix = uuid.uuid4().hex[:8] + context_id = f"media-{suffix}" + agent = put_test_agent(f"media-agent-{suffix}") + context = await Task.put(id=f"media-{suffix}", steps=[_deploy(agent, "solver")]).run() + assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") + [deployed] = context.deployed_agents + image = _png(6, 4) + parts = [{"kind": "text", "text": "look"}, + {"kind": "file", "file": {"bytes": base64.b64encode(image).decode(), "mimeType": "image/png", + "name": "shot.png"}}] + task_id, _ = await send_a2a_message(deployed.a2a_url, parts, f"m-{suffix}", context_id, 60) + await poll_a2a_task(deployed.a2a_url, task_id, 120, 1) + trajectory = A2AAgent.find_extension(deployed.a2a_card, A2AAgent.EXT_TRAJECTORY) + _, media_endpoint = A2AAgent.operation(trajectory, "media") + store = get_config().get_object_store() + async with httpx.AsyncClient(timeout=60, headers=deployed.control_headers) as client: + atif = (await client.post(deployed.a2a_url + trajectory["params"]["endpoint"], + json={"task_id": task_id, "format": ATIF})).json() + [path] = media_paths(atif["trajectory"]) + url = store.object_url(f"atif-media-{suffix}/{path}") + grant = write_object(store, url, media_type="image/png", max_bytes=len(image)) + pushed = await client.post(deployed.a2a_url + media_endpoint, json={ + "context_id": context_id, "objects": {path: grant.model_dump(mode="json", exclude_none=True)}}) + raise_for_extension_status(pushed, operation="media push") + digest = hashlib.sha256(image).hexdigest() + assert path == f"media/{digest}.png" + assert pushed.json() == {"objects": {path: {"size_bytes": len(image), "sha256": digest}}, "missing": [], "failed": {}} + assert store.get(url) == image + [user] = [step for step in atif["trajectory"]["steps"] if step["source"] == "user"] + assert user["extra"]["agentenv"]["media"][path] == {"sha256": digest, "bytes": len(image), "width": 6, "height": 4, + "name": "shot.png"}