Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -759,7 +759,8 @@ class MyDriver(ComputerDriverEnvironment):
`{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment
and returns the manifest; `end_all {reason?}` ends every recording not yet ended, each under the
grant it started with, and returns `{manifests}`, as the framework does before it resets or tears a
computer down. A repeated `request_id`, or a repeated `end`, answers the same. A start past
computer down; `list` (a GET) returns `{recordings}`, each still recording as `{recording_id,
started_at}`. A repeated `request_id`, or a repeated `end`, answers the same. A start past
`max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream
failing for 60 s end a recording by itself.
- **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the
Expand Down Expand Up @@ -797,7 +798,7 @@ driver booted with a control token is probed with it: `--token`, by default
| Level | Checks | Needs on the computer |
|---|---|---|
| `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing |
| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` |
| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held, counting the recordings a driver that `list`s them has open already; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` |

At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), and its layout
(`conformance/desk.py`) into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@
#: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was.
BUSY_DESK_ATTEMPTS = 2
RECORD_FPS = 5
#: How many starts per slot of ``max_concurrent`` the limit check tries before it gives up on a computer whose other
#: recordings keep starting and ending under it.
LIMIT_STARTS = 3
#: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer.
READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();"
"print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest()}))")
Expand Down Expand Up @@ -1037,10 +1040,35 @@ async def recorded() -> str | None:
async def joined() -> str | None:
return None if state.get("joined") else "a repeated request_id started a second recording"

async def recording_now() -> int:
return len((await self.driver.invoke(COMPUTER_RECORDING, "list"))["recordings"])

async def limited_counting(most: int) -> str | None:
"""Below ``most`` open recordings a start succeeds, and at ``most`` it's refused, counting every recording the
driver lists. Others may start or end between a listing and a start, so a surprise is judged by a fresh one."""
tries = LIMIT_STARTS * (most + 1)
for _ in range(tries):
before = await recording_now()
try:
started = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}")
except ExtensionCallError as e:
if e.code != "limit_exceeded":
return f"refused {e.code!r}, not 'limit_exceeded': {e.detail}"
if before >= most or (now := await recording_now()) >= most:
return None
return f"refused limit_exceeded with {now} recordings open, of at most {most}"
if before >= most and (now := await recording_now()) > most:
return f"recording {now} of at most {most} started: {started['recording_id']}"
return f"the computer's recordings kept changing over {tries} starts"

async def limited() -> str | None:
"""A start past ``max_concurrent`` is refused. A driver that lists its recordings is checked with them all
counted; one that doesn't, after the probe's own ``max_concurrent``, as before it could."""
most = params.get("max_concurrent")
if not isinstance(most, int):
return "the card declares no max_concurrent"
if "list" in (params.get("methods") or {}):
return await limited_counting(most)
for _ in range(most):
await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}")
try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st
@extension(COMPUTER_RECORDING, op="start",
description="Record the display: start returns a recording whose segments upload through its grant as "
"they close; end seals it and returns its manifest, uploaded beside them; end_all ends "
"every one not yet ended; repeats answer the same.")
"every one not yet ended; list names those still recording; repeats answer the same.")
async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None,
fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True,
upload: Optional[dict] = None) -> dict:
Expand All @@ -145,6 +145,10 @@ async def _computer_recording_end(self, recording_id: str, upload: Optional[dict
async def _computer_recording_end_all(self, reason: str = "end") -> dict:
return {"manifests": await self._computer.recordings.end_all(reason)}

@extension(COMPUTER_RECORDING, op="list", method="GET")
async def _computer_recording_list(self) -> dict:
return {"recordings": await self._computer.recordings.listed()}


async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict],
timeout: float) -> dict:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,12 @@ async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> d
self.ended.popitem(last=False)
return manifest

async def listed(self) -> list[dict]:
"""Each recording still recording, by id and start time: the ones counted against ``max_concurrent``."""
async with self._lock:
return [{"recording_id": rid, "started_at": _iso(r.requested_at)}
for rid, r in self.running.items() if not r.done]

async def end_all(self, reason: str) -> list[dict]:
"""End every recording not yet ended, each uploading under the grant it was started with; their manifests."""
return [await self.end(recording_id, None, reason) for recording_id in list(self.running)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from probe_fake import FixtureDriver, served # noqa: E402

from agentenv_protocol.computer import RecordedFrame, Recording, capture, record # noqa: E402
from agentenv_protocol.computer import COMPUTER_RECORDING, RecordedFrame, Recording, capture, record # noqa: E402
from agentenv_protocol.computer import recording as recording_module # noqa: E402
from agentenv_protocol.computer.conformance import checks, run_probe # noqa: E402
from agentenv_protocol.computer.conformance.client import Driver # noqa: E402
Expand Down Expand Up @@ -45,6 +45,12 @@ async def _recording_checks(driver) -> dict[str, tuple[str, str]]:
return {r.name: (r.status, r.detail) for r in probe.results}


def _advertising(monkeypatch, most: int) -> None:
advertised = recording_module.recording_params
monkeypatch.setattr("agentenv_protocol.computer.card.recording_params",
lambda features, streams: {**advertised(features, streams), "max_concurrent": most})


@pytest.mark.asyncio
async def test_a_driver_that_records_passes_the_recording_checks_at_full_and_leaves_nothing(home):
with served(Recorder()) as url:
Expand All @@ -68,16 +74,86 @@ async def test_a_manifest_whose_hash_or_size_isnt_the_files_fails(home, monkeypa

@pytest.mark.asyncio
async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still_ended(home, monkeypatch):
advertised = recording_module.recording_params
monkeypatch.setattr("agentenv_protocol.computer.card.recording_params",
lambda features, streams: {**advertised(features, streams), "max_concurrent": 1})
_advertising(monkeypatch, 1)
driver = Recorder()
found = await _recording_checks(driver)
assert found[NAMES[2]][0] == "failed" and "recording 2 of at most 1 started" in found[NAMES[2]][1]
assert found[NAMES[0]][0] == found[NAMES[1]][0] == "passed"
assert not driver._computer.recordings.running and not (home / ".agentenv-recordings").exists()


async def _checks_while_recording(driver) -> tuple[dict[str, tuple[str, str]], str, list[str]]:
"""The recording checks on a computer recording already, as a task's that records while it probes: what they
found, the task's recording, and what was still recording once they were done."""
with served(driver) as url:
connected = await Driver.connect(url, 10)
async with connected.session() as session:
theirs = (await connected.invoke(COMPUTER_RECORDING, "start", {"request_id": "task"}))["recording_id"]
probe = checks.Probe(connected, session, 10)
await probe._recording()
listed = await connected.invoke(COMPUTER_RECORDING, "list")
await connected.invoke(COMPUTER_RECORDING, "end", {"recording_id": theirs})
return {r.name: (r.status, r.detail) for r in probe.results}, theirs, [r["recording_id"] for r in listed["recordings"]]


@pytest.mark.asyncio
async def test_a_computer_already_recording_passes_the_recording_checks_and_keeps_its_recording(home):
"""The probe counts the recording it didn't start against max_concurrent, and ends only its own."""
found, theirs, still = await _checks_while_recording(Recorder())
assert {name: status for name, (status, _) in found.items()} == dict.fromkeys(NAMES, "passed")
assert still == [theirs]


@pytest.mark.asyncio
@pytest.mark.parametrize("recording_already", [False, True], ids=["idle", "recording"])
async def test_a_driver_that_refuses_short_of_the_limit_its_card_declares_fails(home, monkeypatch, recording_already):
class Short(FixtureDriver):
recording = Recording(max_concurrent=2)

_advertising(monkeypatch, 3)
found = (await _checks_while_recording(Short()))[0] if recording_already else await _recording_checks(Short())
assert found[NAMES[2]][0] == "failed" and "limit_exceeded" in found[NAMES[2]][1]


@pytest.mark.asyncio
@pytest.mark.parametrize("meanwhile", ["ends", "starts"])
async def test_a_recording_that_ends_or_starts_between_the_probes_count_and_its_start_doesnt_fail_it(home, monkeypatch,
meanwhile):
"""The task's recording ending just before the probe's second limit start, or another starting just before its
first: a listing after the start says the limit held."""
real, limit_starts = recording_module.Recordings.start, []

async def start(self, request_id, *args):
if "-limit-" in request_id:
limit_starts.append(request_id)
if meanwhile == "ends" and len(limit_starts) == 2:
await self.end(self.started["task"]["recording_id"], None, "end")
if meanwhile == "starts" and len(limit_starts) == 1:
await real(self, "late", *args)
return await real(self, request_id, *args)

monkeypatch.setattr(recording_module.Recordings, "start", start)
driver = Recorder()
found, theirs, still = await _checks_while_recording(driver)
late = driver._computer.recordings.started.get("late")
if late:
await driver._computer.recordings.end(late["recording_id"], None, "end")
assert found[NAMES[2]] == ("passed", ""), found[NAMES[2]]
assert still == ([] if meanwhile == "ends" else [theirs, late["recording_id"]])


@pytest.mark.asyncio
async def test_from_a_driver_that_doesnt_list_its_recordings_the_probe_starts_the_whole_limit_as_before(home, monkeypatch):
"""So a computer that's recording already fails the limit check, as it did before drivers listed them."""
params = Driver.params
monkeypatch.setattr(Driver, "params", lambda self, uri: {**params(self, uri), "methods": {
op: m for op, m in (params(self, uri).get("methods") or {}).items() if op != "list"}})
idle = await _recording_checks(Recorder())
recording, _, _ = await _checks_while_recording(Recorder())
assert idle[NAMES[2]][0] == "passed"
assert recording[NAMES[2]][0] == "failed" and "limit_exceeded" in recording[NAMES[2]][1]


@pytest.mark.asyncio
async def test_a_driver_that_turns_recording_off_gets_no_recording_checks(home):
class Off(FixtureDriver):
Expand Down
25 changes: 24 additions & 1 deletion packages/agentenv-protocol/tests/test_computer_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@
)
from agentenv_protocol.computer import recording as recording_module # noqa: E402
from agentenv_protocol.transfers import TransferError # noqa: E402
from agentenv_protocol.types import CONTROL_TOKEN_ENV # noqa: E402

START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end"
END_ALL = "/agentenv/ext/computer-recording/end_all"
LIST = "/agentenv/ext/computer-recording/list"


@pytest.fixture(autouse=True)
Expand Down Expand Up @@ -135,7 +137,7 @@ class Off(FakeDriver):
async with http_client(Off()) as http:
off = (await http.get("/.well-known/agent-env.json")).json()
extension = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING)
assert set(extension["params"]["methods"]) == {"start", "end", "end_all"}
assert set(extension["params"]["methods"]) == {"start", "end", "end_all", "list"}
assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == {
"formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15},
"cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400}
Expand Down Expand Up @@ -337,6 +339,27 @@ async def test_a_start_past_max_concurrent_is_refused_and_a_reason_is_kept(home)
assert ended["ended_by"] == "reset" and after.status_code == 200


@pytest.mark.asyncio
async def test_list_names_each_recording_still_recording_as_its_start_answered(home):
async with http_client(FakeDriver()) as http:
first = (await http.post(START, json={"request_id": "a"})).json()
second = (await http.post(START, json={"request_id": "b"})).json()
await http.post(END, json={"recording_id": first["recording_id"]})
listed = (await http.get(LIST)).json()["recordings"]
await http.post(END, json={"recording_id": second["recording_id"]})
after = (await http.get(LIST)).json()["recordings"]
assert listed == [{"recording_id": second["recording_id"], "started_at": second["started_at"]}] and after == []


@pytest.mark.asyncio
async def test_list_takes_the_control_token_as_every_framework_only_route_does(home, monkeypatch):
monkeypatch.setenv(CONTROL_TOKEN_ENV, "s3cret")
async with http_client(FakeDriver()) as http:
refused = await http.get(LIST)
allowed = await http.get(LIST, headers={"Authorization": "Bearer s3cret"})
assert refused.status_code == 401 and allowed.status_code == 200 and allowed.json() == {"recordings": []}


@pytest.mark.asyncio
async def test_max_duration_and_a_full_disk_end_a_recording_as_limit(home, monkeypatch):
driver = FakeDriver()
Expand Down
Loading