diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 75aaead43..3c73e8b07 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -759,7 +759,8 @@ class MyDriver(ComputerDriverEnvironment): `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment and returns the manifest; `end_all {reason?}` ends every recording not yet ended, each under the grant it started with, and returns `{manifests}`, as the framework does before it resets or tears a - computer down. A repeated `request_id`, or a repeated `end`, answers the same. A start past + computer down; `list` (a GET) returns `{recordings}`, each still recording as `{recording_id, + started_at}`. A repeated `request_id`, or a repeated `end`, answers the same. A start past `max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream failing for 60 s end a recording by itself. - **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the @@ -797,7 +798,7 @@ driver booted with a control token is probed with it: `--token`, by default | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held, counting the recordings a driver that `list`s them has open already; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), and its layout (`conformance/desk.py`) into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 3ac54bf68..a2f13c1da 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -68,6 +68,9 @@ #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. BUSY_DESK_ATTEMPTS = 2 RECORD_FPS = 5 +#: How many starts per slot of ``max_concurrent`` the limit check tries before it gives up on a computer whose other +#: recordings keep starting and ending under it. +LIMIT_STARTS = 3 #: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer. READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();" "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest()}))") @@ -1037,10 +1040,35 @@ async def recorded() -> str | None: async def joined() -> str | None: return None if state.get("joined") else "a repeated request_id started a second recording" + async def recording_now() -> int: + return len((await self.driver.invoke(COMPUTER_RECORDING, "list"))["recordings"]) + + async def limited_counting(most: int) -> str | None: + """Below ``most`` open recordings a start succeeds, and at ``most`` it's refused, counting every recording the + driver lists. Others may start or end between a listing and a start, so a surprise is judged by a fresh one.""" + tries = LIMIT_STARTS * (most + 1) + for _ in range(tries): + before = await recording_now() + try: + started = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + except ExtensionCallError as e: + if e.code != "limit_exceeded": + return f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" + if before >= most or (now := await recording_now()) >= most: + return None + return f"refused limit_exceeded with {now} recordings open, of at most {most}" + if before >= most and (now := await recording_now()) > most: + return f"recording {now} of at most {most} started: {started['recording_id']}" + return f"the computer's recordings kept changing over {tries} starts" + async def limited() -> str | None: + """A start past ``max_concurrent`` is refused. A driver that lists its recordings is checked with them all + counted; one that doesn't, after the probe's own ``max_concurrent``, as before it could.""" most = params.get("max_concurrent") if not isinstance(most, int): return "the card declares no max_concurrent" + if "list" in (params.get("methods") or {}): + return await limited_counting(most) for _ in range(most): await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") try: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 78aaf932d..0dde59926 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -130,7 +130,7 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st @extension(COMPUTER_RECORDING, op="start", description="Record the display: start returns a recording whose segments upload through its grant as " "they close; end seals it and returns its manifest, uploaded beside them; end_all ends " - "every one not yet ended; repeats answer the same.") + "every one not yet ended; list names those still recording; repeats answer the same.") async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, upload: Optional[dict] = None) -> dict: @@ -145,6 +145,10 @@ async def _computer_recording_end(self, recording_id: str, upload: Optional[dict async def _computer_recording_end_all(self, reason: str = "end") -> dict: return {"manifests": await self._computer.recordings.end_all(reason)} + @extension(COMPUTER_RECORDING, op="list", method="GET") + async def _computer_recording_list(self) -> dict: + return {"recordings": await self._computer.recordings.listed()} + async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 2b3840067..a6c7c846a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -197,6 +197,12 @@ async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> d self.ended.popitem(last=False) return manifest + async def listed(self) -> list[dict]: + """Each recording still recording, by id and start time: the ones counted against ``max_concurrent``.""" + async with self._lock: + return [{"recording_id": rid, "started_at": _iso(r.requested_at)} + for rid, r in self.running.items() if not r.done] + async def end_all(self, reason: str) -> list[dict]: """End every recording not yet ended, each uploading under the grant it was started with; their manifests.""" return [await self.end(recording_id, None, reason) for recording_id in list(self.running)] diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index fbcf36228..5164ebdd6 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -15,7 +15,7 @@ from probe_fake import FixtureDriver, served # noqa: E402 -from agentenv_protocol.computer import RecordedFrame, Recording, capture, record # noqa: E402 +from agentenv_protocol.computer import COMPUTER_RECORDING, RecordedFrame, Recording, capture, record # noqa: E402 from agentenv_protocol.computer import recording as recording_module # noqa: E402 from agentenv_protocol.computer.conformance import checks, run_probe # noqa: E402 from agentenv_protocol.computer.conformance.client import Driver # noqa: E402 @@ -45,6 +45,12 @@ async def _recording_checks(driver) -> dict[str, tuple[str, str]]: return {r.name: (r.status, r.detail) for r in probe.results} +def _advertising(monkeypatch, most: int) -> None: + advertised = recording_module.recording_params + monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", + lambda features, streams: {**advertised(features, streams), "max_concurrent": most}) + + @pytest.mark.asyncio async def test_a_driver_that_records_passes_the_recording_checks_at_full_and_leaves_nothing(home): with served(Recorder()) as url: @@ -68,9 +74,7 @@ async def test_a_manifest_whose_hash_or_size_isnt_the_files_fails(home, monkeypa @pytest.mark.asyncio async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still_ended(home, monkeypatch): - advertised = recording_module.recording_params - monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", - lambda features, streams: {**advertised(features, streams), "max_concurrent": 1}) + _advertising(monkeypatch, 1) driver = Recorder() found = await _recording_checks(driver) assert found[NAMES[2]][0] == "failed" and "recording 2 of at most 1 started" in found[NAMES[2]][1] @@ -78,6 +82,78 @@ async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still assert not driver._computer.recordings.running and not (home / ".agentenv-recordings").exists() +async def _checks_while_recording(driver) -> tuple[dict[str, tuple[str, str]], str, list[str]]: + """The recording checks on a computer recording already, as a task's that records while it probes: what they + found, the task's recording, and what was still recording once they were done.""" + with served(driver) as url: + connected = await Driver.connect(url, 10) + async with connected.session() as session: + theirs = (await connected.invoke(COMPUTER_RECORDING, "start", {"request_id": "task"}))["recording_id"] + probe = checks.Probe(connected, session, 10) + await probe._recording() + listed = await connected.invoke(COMPUTER_RECORDING, "list") + await connected.invoke(COMPUTER_RECORDING, "end", {"recording_id": theirs}) + return {r.name: (r.status, r.detail) for r in probe.results}, theirs, [r["recording_id"] for r in listed["recordings"]] + + +@pytest.mark.asyncio +async def test_a_computer_already_recording_passes_the_recording_checks_and_keeps_its_recording(home): + """The probe counts the recording it didn't start against max_concurrent, and ends only its own.""" + found, theirs, still = await _checks_while_recording(Recorder()) + assert {name: status for name, (status, _) in found.items()} == dict.fromkeys(NAMES, "passed") + assert still == [theirs] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("recording_already", [False, True], ids=["idle", "recording"]) +async def test_a_driver_that_refuses_short_of_the_limit_its_card_declares_fails(home, monkeypatch, recording_already): + class Short(FixtureDriver): + recording = Recording(max_concurrent=2) + + _advertising(monkeypatch, 3) + found = (await _checks_while_recording(Short()))[0] if recording_already else await _recording_checks(Short()) + assert found[NAMES[2]][0] == "failed" and "limit_exceeded" in found[NAMES[2]][1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("meanwhile", ["ends", "starts"]) +async def test_a_recording_that_ends_or_starts_between_the_probes_count_and_its_start_doesnt_fail_it(home, monkeypatch, + meanwhile): + """The task's recording ending just before the probe's second limit start, or another starting just before its + first: a listing after the start says the limit held.""" + real, limit_starts = recording_module.Recordings.start, [] + + async def start(self, request_id, *args): + if "-limit-" in request_id: + limit_starts.append(request_id) + if meanwhile == "ends" and len(limit_starts) == 2: + await self.end(self.started["task"]["recording_id"], None, "end") + if meanwhile == "starts" and len(limit_starts) == 1: + await real(self, "late", *args) + return await real(self, request_id, *args) + + monkeypatch.setattr(recording_module.Recordings, "start", start) + driver = Recorder() + found, theirs, still = await _checks_while_recording(driver) + late = driver._computer.recordings.started.get("late") + if late: + await driver._computer.recordings.end(late["recording_id"], None, "end") + assert found[NAMES[2]] == ("passed", ""), found[NAMES[2]] + assert still == ([] if meanwhile == "ends" else [theirs, late["recording_id"]]) + + +@pytest.mark.asyncio +async def test_from_a_driver_that_doesnt_list_its_recordings_the_probe_starts_the_whole_limit_as_before(home, monkeypatch): + """So a computer that's recording already fails the limit check, as it did before drivers listed them.""" + params = Driver.params + monkeypatch.setattr(Driver, "params", lambda self, uri: {**params(self, uri), "methods": { + op: m for op, m in (params(self, uri).get("methods") or {}).items() if op != "list"}}) + idle = await _recording_checks(Recorder()) + recording, _, _ = await _checks_while_recording(Recorder()) + assert idle[NAMES[2]][0] == "passed" + assert recording[NAMES[2]][0] == "failed" and "limit_exceeded" in recording[NAMES[2]][1] + + @pytest.mark.asyncio async def test_a_driver_that_turns_recording_off_gets_no_recording_checks(home): class Off(FixtureDriver): diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index bfca15e67..59a3badb8 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -31,9 +31,11 @@ ) from agentenv_protocol.computer import recording as recording_module # noqa: E402 from agentenv_protocol.transfers import TransferError # noqa: E402 +from agentenv_protocol.types import CONTROL_TOKEN_ENV # noqa: E402 START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" END_ALL = "/agentenv/ext/computer-recording/end_all" +LIST = "/agentenv/ext/computer-recording/list" @pytest.fixture(autouse=True) @@ -135,7 +137,7 @@ class Off(FakeDriver): async with http_client(Off()) as http: off = (await http.get("/.well-known/agent-env.json")).json() extension = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) - assert set(extension["params"]["methods"]) == {"start", "end", "end_all"} + assert set(extension["params"]["methods"]) == {"start", "end", "end_all", "list"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400} @@ -337,6 +339,27 @@ async def test_a_start_past_max_concurrent_is_refused_and_a_reason_is_kept(home) assert ended["ended_by"] == "reset" and after.status_code == 200 +@pytest.mark.asyncio +async def test_list_names_each_recording_still_recording_as_its_start_answered(home): + async with http_client(FakeDriver()) as http: + first = (await http.post(START, json={"request_id": "a"})).json() + second = (await http.post(START, json={"request_id": "b"})).json() + await http.post(END, json={"recording_id": first["recording_id"]}) + listed = (await http.get(LIST)).json()["recordings"] + await http.post(END, json={"recording_id": second["recording_id"]}) + after = (await http.get(LIST)).json()["recordings"] + assert listed == [{"recording_id": second["recording_id"], "started_at": second["started_at"]}] and after == [] + + +@pytest.mark.asyncio +async def test_list_takes_the_control_token_as_every_framework_only_route_does(home, monkeypatch): + monkeypatch.setenv(CONTROL_TOKEN_ENV, "s3cret") + async with http_client(FakeDriver()) as http: + refused = await http.get(LIST) + allowed = await http.get(LIST, headers={"Authorization": "Bearer s3cret"}) + assert refused.status_code == 401 and allowed.status_code == 200 and allowed.json() == {"recordings": []} + + @pytest.mark.asyncio async def test_max_duration_and_a_full_disk_end_a_recording_as_limit(home, monkeypatch): driver = FakeDriver()