From fad6ff8fadd0df54ffb10094a13382e219b03b32 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 10:35:11 -0800 Subject: [PATCH 1/4] fix(computer): the probe's recording-limit check holds on a computer already recording The full probe's limit check started max_concurrent recordings, catching nothing, and expected only the next to be refused. On a computer recording already, as a task's that records while it probes, the loop's own second start was refused (limit_exceeded) and the check failed, though the limit held. The probe can't count recordings it didn't start, so a limit_exceeded refusal at any of its starts now holds the limit. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 16 ++++++++++------ .../test_computer_conformance_recording.py | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 3ac54bf68..18c6797de 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -1038,15 +1038,19 @@ async def joined() -> str | None: return None if state.get("joined") else "a repeated request_id started a second recording" async def limited() -> str | None: + """Past ``max_concurrent``, a start is refused. A recording the probe didn't start, as a task's that records + while it probes, counts against the limit too, and the probe can't count those, so a refusal sooner holds + the limit as well.""" most = params.get("max_concurrent") if not isinstance(most, int): return "the card declares no max_concurrent" - for _ in range(most): - await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") - try: - extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") - except ExtensionCallError as e: - return None if e.code == "limit_exceeded" else f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" + for _ in range(most + 1): + try: + extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + except ExtensionCallError as e: + if e.code == "limit_exceeded": + return None + return f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" return f"recording {most + 1} of at most {most} started: {extra['recording_id']}" try: diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index fbcf36228..d124a750a 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -15,7 +15,7 @@ from probe_fake import FixtureDriver, served # noqa: E402 -from agentenv_protocol.computer import RecordedFrame, Recording, capture, record # noqa: E402 +from agentenv_protocol.computer import COMPUTER_RECORDING, RecordedFrame, Recording, capture, record # noqa: E402 from agentenv_protocol.computer import recording as recording_module # noqa: E402 from agentenv_protocol.computer.conformance import checks, run_probe # noqa: E402 from agentenv_protocol.computer.conformance.client import Driver # noqa: E402 @@ -78,6 +78,23 @@ async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still assert not driver._computer.recordings.running and not (home / ".agentenv-recordings").exists() +@pytest.mark.asyncio +async def test_a_computer_already_recording_passes_the_recording_checks_and_keeps_its_recording(home): + """As a task's recording while it probes: it counts against max_concurrent, so the probe's own second start is + refused, and the probe ends only the recordings it started.""" + driver = Recorder() + with served(driver) as url: + connected = await Driver.connect(url, 10) + async with connected.session() as session: + theirs = (await connected.invoke(COMPUTER_RECORDING, "start", {"request_id": "task"}))["recording_id"] + probe = checks.Probe(connected, session, 10) + await probe._recording() + running = [rid for rid, r in driver._computer.recordings.running.items() if not r.done] + await connected.invoke(COMPUTER_RECORDING, "end", {"recording_id": theirs}) + assert {r.name: r.status for r in probe.results} == dict.fromkeys(NAMES, "passed") + assert running == [theirs] + + @pytest.mark.asyncio async def test_a_driver_that_turns_recording_off_gets_no_recording_checks(home): class Off(FixtureDriver): From 7880e516574c02c4ed60a5cb9eaccadfac03867d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 11:03:07 -0800 Subject: [PATCH 2/4] fix(computer): the limit check counts the recordings a driver lists, so an early refusal still fails A refusal at any start passed the limit check, so a driver that refused short of its declared max_concurrent passed even on an idle computer, which the check caught before. computer-recording/v1 now has a list op, served by the SDK's recordings, naming those still recording. The probe starts only what's left of the limit beside the recordings a driver lists, and from a driver that doesn't list them starts the whole limit, as before. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 5 +- .../computer/conformance/checks.py | 23 +++++---- .../agentenv_protocol/computer/extensions.py | 6 ++- .../agentenv_protocol/computer/recording.py | 6 +++ .../test_computer_conformance_recording.py | 50 ++++++++++++++++--- .../tests/test_computer_recording.py | 25 +++++++++- 6 files changed, 93 insertions(+), 22 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 75aaead43..3c73e8b07 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -759,7 +759,8 @@ class MyDriver(ComputerDriverEnvironment): `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment and returns the manifest; `end_all {reason?}` ends every recording not yet ended, each under the grant it started with, and returns `{manifests}`, as the framework does before it resets or tears a - computer down. A repeated `request_id`, or a repeated `end`, answers the same. A start past + computer down; `list` (a GET) returns `{recordings}`, each still recording as `{recording_id, + started_at}`. A repeated `request_id`, or a repeated `end`, answers the same. A start past `max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream failing for 60 s end a recording by itself. - **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the @@ -797,7 +798,7 @@ driver booted with a control token is probed with it: `--token`, by default | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held, counting the recordings a driver that `list`s them has open already; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), and its layout (`conformance/desk.py`) into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 18c6797de..ab3e5ba85 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -1038,19 +1038,22 @@ async def joined() -> str | None: return None if state.get("joined") else "a repeated request_id started a second recording" async def limited() -> str | None: - """Past ``max_concurrent``, a start is refused. A recording the probe didn't start, as a task's that records - while it probes, counts against the limit too, and the probe can't count those, so a refusal sooner holds - the limit as well.""" + """Past ``max_concurrent``, a start is refused. Recordings the probe didn't start, as a task's that records + while it probes, count against the limit too, so the probe starts only what's left of it: as many as the + driver lists still recording beside its own, or, from a driver that doesn't list them, ``max_concurrent``.""" most = params.get("max_concurrent") if not isinstance(most, int): return "the card declares no max_concurrent" - for _ in range(most + 1): - try: - extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") - except ExtensionCallError as e: - if e.code == "limit_exceeded": - return None - return f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" + others = 0 + if "list" in (params.get("methods") or {}): + listed = await self.driver.invoke(COMPUTER_RECORDING, "list") + others = sum(r["recording_id"] not in made for r in listed["recordings"]) + for _ in range(most - others): + await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + try: + extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + except ExtensionCallError as e: + return None if e.code == "limit_exceeded" else f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" return f"recording {most + 1} of at most {most} started: {extra['recording_id']}" try: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 78aaf932d..0dde59926 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -130,7 +130,7 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st @extension(COMPUTER_RECORDING, op="start", description="Record the display: start returns a recording whose segments upload through its grant as " "they close; end seals it and returns its manifest, uploaded beside them; end_all ends " - "every one not yet ended; repeats answer the same.") + "every one not yet ended; list names those still recording; repeats answer the same.") async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, upload: Optional[dict] = None) -> dict: @@ -145,6 +145,10 @@ async def _computer_recording_end(self, recording_id: str, upload: Optional[dict async def _computer_recording_end_all(self, reason: str = "end") -> dict: return {"manifests": await self._computer.recordings.end_all(reason)} + @extension(COMPUTER_RECORDING, op="list", method="GET") + async def _computer_recording_list(self) -> dict: + return {"recordings": await self._computer.recordings.listed()} + async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 2b3840067..73156ed06 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -197,6 +197,12 @@ async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> d self.ended.popitem(last=False) return manifest + async def listed(self) -> list[dict]: + """Each recording still recording, as its start answered: the ones counted against ``max_concurrent``.""" + async with self._lock: + return [{"recording_id": rid, "started_at": _iso(r.requested_at)} + for rid, r in self.running.items() if not r.done] + async def end_all(self, reason: str) -> list[dict]: """End every recording not yet ended, each uploading under the grant it was started with; their manifests.""" return [await self.end(recording_id, None, reason) for recording_id in list(self.running)] diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index d124a750a..5b2571748 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -78,21 +78,55 @@ async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still assert not driver._computer.recordings.running and not (home / ".agentenv-recordings").exists() -@pytest.mark.asyncio -async def test_a_computer_already_recording_passes_the_recording_checks_and_keeps_its_recording(home): - """As a task's recording while it probes: it counts against max_concurrent, so the probe's own second start is - refused, and the probe ends only the recordings it started.""" - driver = Recorder() +async def _checks_while_recording(driver) -> tuple[dict[str, tuple[str, str]], str, list[str]]: + """The recording checks on a computer recording already, as a task's that records while it probes: what they + found, the task's recording, and what was still recording once they were done.""" with served(driver) as url: connected = await Driver.connect(url, 10) async with connected.session() as session: theirs = (await connected.invoke(COMPUTER_RECORDING, "start", {"request_id": "task"}))["recording_id"] probe = checks.Probe(connected, session, 10) await probe._recording() - running = [rid for rid, r in driver._computer.recordings.running.items() if not r.done] + listed = await connected.invoke(COMPUTER_RECORDING, "list") await connected.invoke(COMPUTER_RECORDING, "end", {"recording_id": theirs}) - assert {r.name: r.status for r in probe.results} == dict.fromkeys(NAMES, "passed") - assert running == [theirs] + return {r.name: (r.status, r.detail) for r in probe.results}, theirs, [r["recording_id"] for r in listed["recordings"]] + + +@pytest.mark.asyncio +async def test_a_computer_already_recording_passes_the_recording_checks_and_keeps_its_recording(home): + """The probe counts the recording it didn't start against max_concurrent, and ends only its own.""" + found, theirs, still = await _checks_while_recording(Recorder()) + assert {name: status for name, (status, _) in found.items()} == dict.fromkeys(NAMES, "passed") + assert still == [theirs] + + +def _advertising(monkeypatch, most: int) -> None: + advertised = recording_module.recording_params + monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", + lambda features, streams: {**advertised(features, streams), "max_concurrent": most}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("recording_already", [False, True], ids=["idle", "recording"]) +async def test_a_driver_that_refuses_short_of_the_limit_its_card_declares_fails(home, monkeypatch, recording_already): + class Short(FixtureDriver): + recording = Recording(max_concurrent=2) + + _advertising(monkeypatch, 3) + found = (await _checks_while_recording(Short()))[0] if recording_already else await _recording_checks(Short()) + assert found[NAMES[2]][0] == "failed" and "limit_exceeded" in found[NAMES[2]][1] + + +@pytest.mark.asyncio +async def test_from_a_driver_that_doesnt_list_its_recordings_the_probe_starts_the_whole_limit_as_before(home, monkeypatch): + """So a computer that's recording already fails the limit check, as it did before drivers listed them.""" + params = Driver.params + monkeypatch.setattr(Driver, "params", lambda self, uri: {**params(self, uri), "methods": { + op: m for op, m in (params(self, uri).get("methods") or {}).items() if op != "list"}}) + idle = await _recording_checks(Recorder()) + recording, _, _ = await _checks_while_recording(Recorder()) + assert idle[NAMES[2]][0] == "passed" + assert recording[NAMES[2]][0] == "failed" and "limit_exceeded" in recording[NAMES[2]][1] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index bfca15e67..59a3badb8 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -31,9 +31,11 @@ ) from agentenv_protocol.computer import recording as recording_module # noqa: E402 from agentenv_protocol.transfers import TransferError # noqa: E402 +from agentenv_protocol.types import CONTROL_TOKEN_ENV # noqa: E402 START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" END_ALL = "/agentenv/ext/computer-recording/end_all" +LIST = "/agentenv/ext/computer-recording/list" @pytest.fixture(autouse=True) @@ -135,7 +137,7 @@ class Off(FakeDriver): async with http_client(Off()) as http: off = (await http.get("/.well-known/agent-env.json")).json() extension = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) - assert set(extension["params"]["methods"]) == {"start", "end", "end_all"} + assert set(extension["params"]["methods"]) == {"start", "end", "end_all", "list"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400} @@ -337,6 +339,27 @@ async def test_a_start_past_max_concurrent_is_refused_and_a_reason_is_kept(home) assert ended["ended_by"] == "reset" and after.status_code == 200 +@pytest.mark.asyncio +async def test_list_names_each_recording_still_recording_as_its_start_answered(home): + async with http_client(FakeDriver()) as http: + first = (await http.post(START, json={"request_id": "a"})).json() + second = (await http.post(START, json={"request_id": "b"})).json() + await http.post(END, json={"recording_id": first["recording_id"]}) + listed = (await http.get(LIST)).json()["recordings"] + await http.post(END, json={"recording_id": second["recording_id"]}) + after = (await http.get(LIST)).json()["recordings"] + assert listed == [{"recording_id": second["recording_id"], "started_at": second["started_at"]}] and after == [] + + +@pytest.mark.asyncio +async def test_list_takes_the_control_token_as_every_framework_only_route_does(home, monkeypatch): + monkeypatch.setenv(CONTROL_TOKEN_ENV, "s3cret") + async with http_client(FakeDriver()) as http: + refused = await http.get(LIST) + allowed = await http.get(LIST, headers={"Authorization": "Bearer s3cret"}) + assert refused.status_code == 401 and allowed.status_code == 200 and allowed.json() == {"recordings": []} + + @pytest.mark.asyncio async def test_max_duration_and_a_full_disk_end_a_recording_as_limit(home, monkeypatch): driver = FakeDriver() From dff6313f263161c47f3634db855870286cbb9bf1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 11:13:10 -0800 Subject: [PATCH 3/4] fix(computer): the limit check judges each start by the recordings listed around it The limit check counted the other recordings once, so one that ended before the probe's last start made that start succeed and read as a violation, and one that started meanwhile made an earlier start refused. Each start is now judged by a listing before it, and an unexpected outcome by a listing after it: a refusal fails only while fewer than max_concurrent are recording, and a start past the limit only once more than max_concurrent are. The path for a driver that doesn't list its recordings is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 35 +++++++++++++++---- .../test_computer_conformance_recording.py | 27 ++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index ab3e5ba85..af9e9e407 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -68,6 +68,9 @@ #: sees its motion as one event, and keys sent while the click that focuses a box waits go wherever focus was. BUSY_DESK_ATTEMPTS = 2 RECORD_FPS = 5 +#: How many starts per slot of ``max_concurrent`` the limit check tries before it gives up on a computer whose other +#: recordings keep starting and ending under it. +LIMIT_STARTS = 3 #: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer. READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();" "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest()}))") @@ -1037,18 +1040,36 @@ async def recorded() -> str | None: async def joined() -> str | None: return None if state.get("joined") else "a repeated request_id started a second recording" + async def recording_now() -> int: + return len((await self.driver.invoke(COMPUTER_RECORDING, "list"))["recordings"]) + + async def limited_counting(most: int) -> str | None: + """Recordings the probe didn't start, as a task's that records while it probes, count against the limit + too and may start or end meanwhile, so each start is judged by the recordings listed around it: one refused + below the limit, or one started past it, fails only if a listing after it still says so.""" + for _ in range(LIMIT_STARTS * (most + 1)): + before = await recording_now() + try: + started = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + except ExtensionCallError as e: + if e.code != "limit_exceeded": + return f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" + if before >= most or (now := await recording_now()) >= most: + return None + return f"refused limit_exceeded with {now} recordings open, of at most {most}" + if before >= most and (now := await recording_now()) > most: + return f"recording {now} of at most {most} started: {started['recording_id']}" + return f"the computer's recordings kept changing over {LIMIT_STARTS * (most + 1)} starts" + async def limited() -> str | None: - """Past ``max_concurrent``, a start is refused. Recordings the probe didn't start, as a task's that records - while it probes, count against the limit too, so the probe starts only what's left of it: as many as the - driver lists still recording beside its own, or, from a driver that doesn't list them, ``max_concurrent``.""" + """Past ``max_concurrent``, a start is refused: counting the recordings a driver lists, or, from one that + doesn't list them, after the probe's own ``max_concurrent``.""" most = params.get("max_concurrent") if not isinstance(most, int): return "the card declares no max_concurrent" - others = 0 if "list" in (params.get("methods") or {}): - listed = await self.driver.invoke(COMPUTER_RECORDING, "list") - others = sum(r["recording_id"] not in made for r in listed["recordings"]) - for _ in range(most - others): + return await limited_counting(most) + for _ in range(most): await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") try: extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index 5b2571748..782898da3 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -117,6 +117,33 @@ class Short(FixtureDriver): assert found[NAMES[2]][0] == "failed" and "limit_exceeded" in found[NAMES[2]][1] +@pytest.mark.asyncio +@pytest.mark.parametrize("meanwhile", ["ends", "starts"]) +async def test_a_recording_that_ends_or_starts_between_the_probes_count_and_its_start_doesnt_fail_it(home, monkeypatch, + meanwhile): + """The task's recording ending just before the probe's second limit start, or another starting just before its + first: a listing after the start says the limit held.""" + real, limit_starts = recording_module.Recordings.start, [] + + async def start(self, request_id, *args): + if "-limit-" in request_id: + limit_starts.append(request_id) + if meanwhile == "ends" and len(limit_starts) == 2: + await self.end(self.started["task"]["recording_id"], None, "end") + if meanwhile == "starts" and len(limit_starts) == 1: + await real(self, "late", *args) + return await real(self, request_id, *args) + + monkeypatch.setattr(recording_module.Recordings, "start", start) + driver = Recorder() + found, theirs, still = await _checks_while_recording(driver) + late = driver._computer.recordings.started.get("late") + if late: + await driver._computer.recordings.end(late["recording_id"], None, "end") + assert found[NAMES[2]] == ("passed", ""), found[NAMES[2]] + assert still == ([] if meanwhile == "ends" else [theirs, late["recording_id"]]) + + @pytest.mark.asyncio async def test_from_a_driver_that_doesnt_list_its_recordings_the_probe_starts_the_whole_limit_as_before(home, monkeypatch): """So a computer that's recording already fails the limit check, as it did before drivers listed them.""" From a7cfe2389c7899185d57d56f4fde5e72e2ea7ab8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 11:25:17 -0800 Subject: [PATCH 4/4] refactor(computer): plainer limit-check docstrings, one count of its tries, and one card helper in its tests Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 14 +++++++------- .../src/agentenv_protocol/computer/recording.py | 2 +- .../tests/test_computer_conformance_recording.py | 16 +++++++--------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index af9e9e407..a2f13c1da 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -1044,10 +1044,10 @@ async def recording_now() -> int: return len((await self.driver.invoke(COMPUTER_RECORDING, "list"))["recordings"]) async def limited_counting(most: int) -> str | None: - """Recordings the probe didn't start, as a task's that records while it probes, count against the limit - too and may start or end meanwhile, so each start is judged by the recordings listed around it: one refused - below the limit, or one started past it, fails only if a listing after it still says so.""" - for _ in range(LIMIT_STARTS * (most + 1)): + """Below ``most`` open recordings a start succeeds, and at ``most`` it's refused, counting every recording the + driver lists. Others may start or end between a listing and a start, so a surprise is judged by a fresh one.""" + tries = LIMIT_STARTS * (most + 1) + for _ in range(tries): before = await recording_now() try: started = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") @@ -1059,11 +1059,11 @@ async def limited_counting(most: int) -> str | None: return f"refused limit_exceeded with {now} recordings open, of at most {most}" if before >= most and (now := await recording_now()) > most: return f"recording {now} of at most {most} started: {started['recording_id']}" - return f"the computer's recordings kept changing over {LIMIT_STARTS * (most + 1)} starts" + return f"the computer's recordings kept changing over {tries} starts" async def limited() -> str | None: - """Past ``max_concurrent``, a start is refused: counting the recordings a driver lists, or, from one that - doesn't list them, after the probe's own ``max_concurrent``.""" + """A start past ``max_concurrent`` is refused. A driver that lists its recordings is checked with them all + counted; one that doesn't, after the probe's own ``max_concurrent``, as before it could.""" most = params.get("max_concurrent") if not isinstance(most, int): return "the card declares no max_concurrent" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 73156ed06..a6c7c846a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -198,7 +198,7 @@ async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> d return manifest async def listed(self) -> list[dict]: - """Each recording still recording, as its start answered: the ones counted against ``max_concurrent``.""" + """Each recording still recording, by id and start time: the ones counted against ``max_concurrent``.""" async with self._lock: return [{"recording_id": rid, "started_at": _iso(r.requested_at)} for rid, r in self.running.items() if not r.done] diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index 782898da3..5164ebdd6 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -45,6 +45,12 @@ async def _recording_checks(driver) -> dict[str, tuple[str, str]]: return {r.name: (r.status, r.detail) for r in probe.results} +def _advertising(monkeypatch, most: int) -> None: + advertised = recording_module.recording_params + monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", + lambda features, streams: {**advertised(features, streams), "max_concurrent": most}) + + @pytest.mark.asyncio async def test_a_driver_that_records_passes_the_recording_checks_at_full_and_leaves_nothing(home): with served(Recorder()) as url: @@ -68,9 +74,7 @@ async def test_a_manifest_whose_hash_or_size_isnt_the_files_fails(home, monkeypa @pytest.mark.asyncio async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still_ended(home, monkeypatch): - advertised = recording_module.recording_params - monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", - lambda features, streams: {**advertised(features, streams), "max_concurrent": 1}) + _advertising(monkeypatch, 1) driver = Recorder() found = await _recording_checks(driver) assert found[NAMES[2]][0] == "failed" and "recording 2 of at most 1 started" in found[NAMES[2]][1] @@ -100,12 +104,6 @@ async def test_a_computer_already_recording_passes_the_recording_checks_and_keep assert still == [theirs] -def _advertising(monkeypatch, most: int) -> None: - advertised = recording_module.recording_params - monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", - lambda features, streams: {**advertised(features, streams), "max_concurrent": most}) - - @pytest.mark.asyncio @pytest.mark.parametrize("recording_already", [False, True], ids=["idle", "recording"]) async def test_a_driver_that_refuses_short_of_the_limit_its_card_declares_fails(home, monkeypatch, recording_already):