Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1696,6 +1696,20 @@ first, and a read scrubs as it serves, so a secret learned after a step was
written is left out too. Native reads keep every secret byte for byte, so a
replay of the served inputs holds what the served copy redacts.

A copy's media files, `media/<sha256>.<ext>` beside the document, wait on the
agent's disk as the adapter names them, one file per sha256, with a PNG, JPEG,
GIF or WebP image's width and height read from its header. The worker pulls
them at each turn's end with the trajectory extension's `media` call, listed on
the card of an agent that serves `atif/1`: `{context_id, objects: {path: write
grant}}` uploads each file the context holds under its exact-object grant, up
to 8 at a time, checking each upload's bytes against its name. It answers what
it uploaded, the paths it holds no file for (`missing`) and the uploads that
failed (`failed`). Neither stops the rest. A file uploaded leaves the agent's
disk, as do a context's files when it is evicted or restored over, and the
rest when the server stops. A reader takes a part path only as
`media/<sha256>.<ext>` and checks the bytes it reads against that name
(`agentenv_protocol.atif.media_file` and `check_media`).

`enable(..., description="...")` is reserved for declarations carrying
configuration or metadata. It preserves the agent-specific extension prose
published in the Agent Card. The versioned SDK definition provides a generic
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@
TaskObjectTrajectoryRequest,
TaskTrajectoryRequest,
TrajectoryEventsResponse,
TrajectoryMediaRequest,
TrajectoryMediaResponse,
TrajectoryObjectsResponse,
AtifEventsResponse,
AtifObjectsResponse,
Expand Down Expand Up @@ -256,6 +258,8 @@
"AtifObjectsResponse",
"AtifUploadedObjects",
"TrajectoryLog",
"TrajectoryMediaRequest",
"TrajectoryMediaResponse",
"TrajectoryObjectsResponse",
"TrajectoryState",
"TrajectoryUploadedObjects",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ class AtifCopy:
serves goes through ``scrub``, the agent's secrets redacted."""

def __init__(self, adapter: type[Adapter], context_id: str, *,
scrub: Callable[[Any], Any] = lambda served: served) -> None:
self._run = AdapterRun(adapter, trajectory_id=uuid.uuid4().hex)
scrub: Callable[[Any], Any] = lambda served: served,
media: Callable[[str, bytes], None] | None = None) -> None:
self._run = AdapterRun(adapter, media=media, trajectory_id=uuid.uuid4().hex)
self._adapter = adapter.__name__
self._context_id = context_id
self._scrub = scrub
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""The media files of an agent's atif/1 copies, spooled to its disk as the adapter names them: one file per sha256,
held by the contexts whose copies reference it, until a push confirms it for a context or that context goes. The worker
pulls them at each turn's end (``TRAJECTORY_V1.media``), so the bytes never wait in memory."""

from __future__ import annotations

import itertools
import os
import shutil
import tempfile
import threading
from collections.abc import Callable
from pathlib import Path


class MediaSpool:
"""Each context's media files on disk. Safe to fill from the agent's thread while the loop pushes them; the folder
is made on the first file and removed by ``close``. Each hold has its own number, so an upload that outlives its
context's copy (a restore, an eviction) releases only the hold it began with, never a later copy's."""

def __init__(self) -> None:
self._lock = threading.Lock()
self._folder: Path | None = None
#: Each context's held paths, ``media/<sha256>.<ext>``, each with its hold's number.
self._held: dict[str, dict[str, int]] = {}
self._holds = itertools.count(1)

def sink(self, context_id: str) -> Callable[[str, bytes], None]:
"""Where a context's copy writes each media file once, as its builder names it."""

def write(path: str, data: bytes) -> None:
with self._lock:
file = self._file(path)
if not file.exists():
part = file.with_name(f"{file.name}.{os.getpid()}.part")
part.write_bytes(data)
os.replace(part, file)
self._held.setdefault(context_id, {}).setdefault(path, next(self._holds))

return write

def held(self, context_id: str, path: str) -> tuple[Path, int] | None:
"""The file a context holds at ``path`` and its hold's number, or None when it holds none there."""
with self._lock:
hold = self._held.get(context_id, {}).get(path)
return None if hold is None else (self._file(path), hold)

def release(self, context_id: str, path: str, hold: int) -> None:
"""Hold ``hold`` on ``path`` is the context's no longer: a push confirmed it. A later hold stays, and the file
goes once no context holds it."""
with self._lock:
held = self._held.get(context_id)
if held is None or held.get(path) != hold:
return
del held[path]
if not held:
del self._held[context_id]
self._remove_unheld(path)

def drop(self, context_id: str) -> None:
"""The context is gone, evicted or restored over: its files go, but those another context still holds."""
with self._lock:
for path in self._held.pop(context_id, {}):
self._remove_unheld(path)

def close(self) -> None:
with self._lock:
if self._folder is not None:
shutil.rmtree(self._folder, ignore_errors=True)
self._folder = None
self._held.clear()

def _file(self, path: str) -> Path:
if self._folder is None:
self._folder = Path(tempfile.mkdtemp(prefix="agentenv-atif-media-"))
return self._folder / path.removeprefix("media/")

def _remove_unheld(self, path: str) -> None:
if self._folder is not None and not any(path in held for held in self._held.values()):
(self._folder / path.removeprefix("media/")).unlink(missing_ok=True)
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@
#: token is: longer than an ordinary path's words, unlike ``MIN_SECRET_LENGTH``, which bounds what is scrubbed at all.
MIN_TOKEN_SEGMENT_LENGTH = 16
_TOKEN_SEGMENT = re.compile(rf"[A-Za-z0-9_.=~+-]{{{MIN_TOKEN_SEGMENT_LENGTH},}}")
#: A media file's name, ``<sha256>.<ext>``: a grant's path may name the file it writes, which names its bytes, not a
#: credential, and which every atif/1 copy that holds the file must keep.
_MEDIA_NAME = re.compile(r"[0-9a-f]{64}\.[A-Za-z0-9]+")

T = TypeVar("T")

Expand Down Expand Up @@ -245,7 +248,8 @@ def _url_secrets(url: str) -> list[str]:
parts = urlsplit(url)
carried = _query_values(url)
if not carried:
carried = [segment for segment in parts.path.split("/") if _TOKEN_SEGMENT.fullmatch(segment)]
carried = [segment for segment in parts.path.split("/")
if _TOKEN_SEGMENT.fullmatch(segment) and not _MEDIA_NAME.fullmatch(segment)]
return [*carried, *([parts.password] if parts.password else [])]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from ..atif import Adapter
from ._atif_copy import AtifCopy, AtifPage, TurnStart
from ._media_spool import MediaSpool
from ._turn_index import TurnIndex
from .extensions import TrajectoryState
from .tasks.v1 import TaskError, TrajectoryLog
Expand Down Expand Up @@ -98,6 +99,8 @@ def __init__(
self._turns = TurnIndex() if atif is not None else None
#: What every atif/1 read goes through: the agent's secrets redacted.
self._scrub = scrub
#: The copies' media files, on disk until the worker pulls them.
self.media = MediaSpool()
self._tasks: dict[str, _TaskRecord] = {}
self._contexts: OrderedDict[str, list[str]] = OrderedDict()
self._copies: dict[str, AtifCopy] = {}
Expand Down Expand Up @@ -133,7 +136,8 @@ def start(self, task_id: str, turn: TurnStart | None = None) -> None:
copy = self._copies.get(record.context_id)
if copy is None:
copy = self._copies[record.context_id] = AtifCopy(
self._atif, record.context_id, **({"scrub": self._scrub} if self._scrub is not None else {})
self._atif, record.context_id, media=self.media.sink(record.context_id),
**({"scrub": self._scrub} if self._scrub is not None else {}),
)
copy.start(task_id, self._turns.next(record.context_id), turn)
record.log._atif = _CopySink(copy, task_id)
Expand All @@ -149,6 +153,7 @@ def restored(self, context_id: str, turns: int) -> None:
new document, since the conversation is the snapshot's now; an agent that serves atif/1 only."""
self._turns.set(context_id, turns)
self._copies.pop(context_id, None)
self.media.drop(context_id)

def request_cancel(self, task_id: str) -> None:
"""Make ``canceled`` the outcome the task is sealed with, whatever ``run`` returns."""
Expand Down Expand Up @@ -357,6 +362,7 @@ def _evict(self, *, keep: str) -> None:
continue
del self._contexts[context_id]
self._copies.pop(context_id, None)
self.media.drop(context_id)
for task_id in task_ids:
del self._tasks[task_id]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
)

from ..atif import FORMAT as ATIF_FORMAT
from ..atif import Adapter, AdapterSpec
from ..atif import Adapter, AdapterSpec, media_file
from ._triggers import MAX_SOLVER_MESSAGE_LENGTH
from .tasks.v1 import AgentConfig
from ..transfers import (
Expand Down Expand Up @@ -205,6 +205,34 @@ class TrajectoryObjectsResponse(ExtensionResponse):
format: str | None = None


#: The most media files one media call may name: a turn of a computer-use run holds a few hundred screenshots.
MAX_MEDIA_OBJECTS = 10_000


class TrajectoryMediaRequest(ExtensionRequest):
"""The media files of a context's atif/1 copy the caller wants, by their paths in it (``media/<sha256>.<ext>``),
each under its own exact-object write grant."""

context_id: str = Field(min_length=1)
objects: dict[str, WriteObject] = Field(min_length=1, max_length=MAX_MEDIA_OBJECTS)

@field_validator("objects")
@classmethod
def _media_paths(cls, objects: dict[str, WriteObject]) -> dict[str, WriteObject]:
for path in objects:
media_file(path)
return objects


class TrajectoryMediaResponse(ExtensionResponse):
"""What a media call uploaded, by path; the paths the context holds no file for; and the uploads that failed, with
why. Neither of the last two stops the rest."""

objects: dict[str, Uploaded]
missing: list[str] = Field(default_factory=list)
failed: dict[str, str] = Field(default_factory=dict)


class AtifUploadedObjects(ExtensionResponse):
trajectory: Uploaded
aux: Uploaded | None = None
Expand Down Expand Up @@ -816,6 +844,9 @@ class OperationDefinition:
implementation: ImplementationOwner
request: RequestDefinition | type[BaseModel] | None = None
response: FieldSchema | type[BaseModel] | None = None
#: Served by the SDK alone, from state only it holds: no agent overrides it, and an agent that overrides the
#: extension's other SDK operations needn't override it with them.
sdk_only: bool = False
response_model: type[BaseModel] | None = field(default=None, init=False)

def __post_init__(self) -> None:
Expand Down Expand Up @@ -1104,6 +1135,7 @@ def _op(
response_required: Iterable[str] = (),
response_optional: Iterable[str] = (),
variants: Iterable[RequestVariant] = (),
sdk_only: bool = False,
) -> OperationDefinition:
response_required = _tuple(response_required)
response_optional = _tuple(response_optional)
Expand All @@ -1123,6 +1155,7 @@ def _op(
implementation=owner,
request=request,
response=response,
sdk_only=sdk_only,
)


Expand Down Expand Up @@ -1194,6 +1227,8 @@ def _configure_trajectory(
return ExtensionConfiguration(
wire_params={"formats": formats, "native": native, "default": native} if native else {},
options={"native": native, **({"atif": atif} if atif is not None else {})} if native else {},
# An agent that serves atif/1 hands over its copies' media files, which the worker pulls at each turn's end.
features={"media"} if atif is not None else frozenset(),
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
variants=(
{"get": {"task_events", "context_events", "context", "context_objects"}} if live else {}
),
Expand Down Expand Up @@ -1341,6 +1376,22 @@ def _configure_trajectory(
),
)
},
optional_features={
"media": OperationGroup(
operations={
"media": _op(
"media",
"POST",
"/ext/trajectory/media",
SDK,
request_model=TrajectoryMediaRequest,
response_model=TrajectoryMediaResponse,
# The spool it reads is the SDK's, so an agent that serves its own trajectory reads still has it.
sdk_only=True,
),
}
)
},
configuration_validator=_configure_trajectory,
)

Expand Down
Loading
Loading