From a83bdac6e32a65471665dfc818c11d38f0ae8fc5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 14:01:54 -0800 Subject: [PATCH 1/2] fix(browser): a click that opens a popup leaves the recording's video on it The browser log tells a recording of an action when the browser answers it, and the browser opens the popup a click makes before it answers the click, so the click could take the video back to its opener, where it stayed until the agent's next action in the popup. An action begun on a tab before a tab it opened, which the video has followed since, now leaves the video there; one begun on the opener after that still takes it back. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 3 ++- .../website_browser/recording.py | 4 +-- .../website_browser/stream.py | 24 +++++++++++------ .../tests/test_browser_recording.py | 26 +++++++++++++++++++ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 94cd518fe..cd70d4e0f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1204,7 +1204,8 @@ are the profile's template. The base class adds: - **a video** of the tab its context's agent is on, at `fps` (10 by default, 15 at most), encoded as `computer-recording/v1`'s is and uploaded a segment at a time under `/video/`. It follows the tab an agent last acted on in that context (CDP commands and MCP calls, not agentenv's own), a tab that one opens as - soon as it opens, and once it closes the tab followed before it, else the context's newest; it's blank while the + soon as it opens, which an action begun on the opener before then, as the click that opened it, doesn't take it + back from, and once it closes the tab followed before it, else the context's newest; it's blank while the context has no tab. The browser's own connection screencasts that tab, and captures it instead while it sends no frame, as a tab behind another in its window does; `video.followed` says which tab showed when, and how. - **its log**: at the end, the lines of its context since its start, actions and tab events on its tabs and a diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py index f2419bbf4..0a00cca61 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py @@ -312,7 +312,7 @@ def tab_event(self, event: str, tab: dict) -> None: if isinstance(target := tab.get("id"), str): self.tab_contexts[target] = tab.get("context_id") for recording in self.every(): - recording.stream.tab_event(event, tab) + recording.stream.tab_event(event, tab, self.log.next_seq) def context_of_tab(self, target: object) -> Optional[str]: """The context of the tab ``target``, as it opened in, or as the browser's control has it.""" @@ -419,7 +419,7 @@ def acted(self, seq: int, tab: Optional[str], moves: bool = True) -> None: if self.ending or not self.stream.sees(tab): return if moves: - self.stream.follow(tab) + self.stream.follow(tab, seq) if not self.frames_on: return self.pending.append(seq) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py index ca2b3e213..929a87988 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py @@ -1,6 +1,7 @@ """The frames of one browser context as its recording's video takes them (browser-recording/v1): of the tab it -follows, the one an agent last acted on there, a tab that one opens as soon as it opens, and once it closes the tab -followed before it that's still open, else the context's newest; a blank frame while the context has no tab. The +follows, the one an agent last acted on there, a tab that one opens as soon as it opens, which an action begun on +the opener before it opened doesn't take the video back from, and once it closes the tab followed before it that's +still open, else the context's newest; a blank frame while the context has no tab. The browser's own connection screencasts that tab; a tab that sends no frame for ``SILENT_S`` and isn't visible, as one behind another in its window, is captured instead, at most ``FALLBACK_FPS`` times a second, until it sends one again, while a visible one is a still page its last frame shows. Every ``1/fps`` seconds the latest frame is handed on, so a @@ -56,23 +57,30 @@ def __init__(self, control: BrowserControl, context_id: Optional[str], blank_siz self._seeding: Optional[asyncio.Future] = None self._max_edge: Optional[int] = None self._via: Optional[str] = None + #: Each tab that opened one followed, and the log's next sequence number then: an action on it numbered below + #: began before that tab opened. + self._opened_from: dict[str, int] = {} def sees(self, target: Optional[str]) -> bool: """Whether the tab ``target`` is in its context.""" return bool(target) and self.control.in_context(self.control.context_of_tab(target), self.context_id) - def follow(self, target: Optional[str]) -> None: - """An agent acted on the tab ``target``: one in its context is followed from now.""" - if self.sees(target) and target != self.tab: + def follow(self, target: Optional[str], seq: int) -> None: + """An agent's action ``seq`` in the log acted on the tab ``target``: one in its context is followed from now, + unless the action began before a tab that ``target`` opened, which the video has followed since.""" + if self.sees(target) and target != self.tab and seq >= self._opened_from.get(target, 0): self._move(target) - def tab_event(self, event: str, tab: dict) -> None: - """A tab opened or closed in the browser: one the followed tab opens, or the first its context has, is - followed; once the followed tab closes, the one followed before it that's still open, else the newest.""" + def tab_event(self, event: str, tab: dict, next_seq: int) -> None: + """A tab opened or closed in the browser while ``next_seq`` was the log's next sequence number: one the + followed tab opens, or the first its context has, is followed; once the followed tab closes, the one followed + before it that's still open, else the newest.""" target = tab.get("id") if not self.control.in_context(tab.get("context_id"), self.context_id): return if event == "opened" and (self.tab is None or tab.get("opener") == self.tab): + if self.tab is not None: + self._opened_from[self.tab] = next_seq self._move(target) elif event == "closed" and target == self.tab: self._move(next((t for t in reversed(self._history) if t in self.control.tabs and self.sees(t)), None) or diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 06d3e7685..6e9ef69e7 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -228,6 +228,32 @@ async def test_the_video_follows_an_agents_tab_a_popup_it_opens_and_goes_back_wh assert "green" not in colors[colors.index("yellow"):], colors +@pytest.mark.asyncio +async def test_a_click_that_opens_a_popup_leaves_the_video_on_it_though_the_click_is_answered_after(browser): + """The browser opens the popup a click makes before it answers the click, which the log hears of then; an action + on the opener begun after it opened takes the video back.""" + pipe, control, contexts, network, recordings, uploads = browser + pipe.open("T3", YELLOW) + pipe.focused = "T3" + started = await recordings.start("run:a", _grant(), network=False) + await asyncio.sleep(0.3) + click = recordings.log.begin({"via": "cdp", "who": "ada", "method": "Input.dispatchMouseEvent", "acts": True, + "tab": "T3"}) + pipe.open("T4", RED, opener="T3") + await asyncio.sleep(0.1) + recordings.log.finish(click, ok=True) + await asyncio.sleep(0.3) + followed_in_the_popup = [seen["tab"] for seen in recordings.every()[0].stream.followed] + _acted(recordings.log, "T3") + await asyncio.sleep(0.3) + manifest = BrowserRecordingManifest.model_validate(await recordings.end(started["recording_id"])) + + assert followed_in_the_popup == ["T3", "T4"] + assert [seen.tab for seen in manifest.video.followed] == ["T3", "T4", "T3"] + colors = uploads.colors(f"{ROOT}/{manifest.recording_id}") + assert colors[0] == "yellow" and "red" in colors and colors[-1] == "yellow", colors + + @pytest.mark.asyncio async def test_a_clients_setup_on_every_page_it_attaches_to_doesnt_move_the_video_and_its_input_does(browser): pipe, control, contexts, network, recordings, uploads = browser From 4e0a975a2367d414e1efaccad161fa9f9d57ffb3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 14:48:21 -0800 Subject: [PATCH 2/2] fix(browser): a popup a click opens runs at once rather than 30 s later BrowserControl held each page it attached to until it had enabled it and set its viewport, and only then told it to run. Chromium answers Page.enable on a popup a click opened, paused before its first navigation, only once it runs, so the control waited out Page.enable's 30 s timeout first: on the released Chromium with a viewport, every popup showed its page 30 s after the click, unless another client let it run. The calls now go together, in the order the browser handles them, so the viewport still applies before the page runs: 0.05-0.08 s on the same Chromium. A browser recording followed the popup blank meanwhile. Co-Authored-By: Claude Opus 5.5 --- .../website_browser/control.py | 27 +++++++------ .../tests/test_website_browser.py | 40 +++++++++++++++++-- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index b4ac27a1a..2f43e12ce 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -1172,21 +1172,24 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N async def _let_run(self, session: str) -> None: """A page it attached to, enabled and, alone, held to its window's size, then told to run: a page attached as - it opens, a popup say, or any new page alone, waits for that before it runs.""" + it opens, a popup say, or any new page alone, waits for that before it runs. The calls go together, in order, + which the browser handles them in: a popup a click opened answers Page.enable only once it runs.""" target = self._sessions.get(session) + calls: list[tuple[str, Optional[dict]]] = [("Page.enable", None)] + if self.viewport is not None: + width, height = self.viewport + calls += [("Emulation.setDeviceMetricsOverride", + {"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}), + ("Page.setInterceptFileChooserDialog", {"enabled": True})] + calls.append(("Runtime.runIfWaitingForDebugger", None)) try: - await self.devtools.call("Page.enable", session_id=session) - if self.viewport is not None: - width, height = self.viewport - await self.devtools.call("Emulation.setDeviceMetricsOverride", { - "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) - await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session) + answers = await asyncio.gather(*(self.devtools.call(method, params, session_id=session) + for method, params in calls), return_exceptions=True) finally: - try: - await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) - finally: - if target is not None: - self._pinned.setdefault(target, asyncio.Event()).set() + if target is not None: + self._pinned.setdefault(target, asyncio.Event()).set() + if failed := next((answer for answer in answers if isinstance(answer, BaseException)), None): + raise failed def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: self._screencasts.pop(params["sessionId"], None) diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 111a72247..0a920859a 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -566,7 +566,7 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to control = BrowserControl(devtools, "agent") await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) - await asyncio.sleep(0) + await asyncio.sleep(0.01) assert ("Page.enable", {}, "s-popup") in devtools.calls assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls @@ -581,7 +581,7 @@ async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_ru control = BrowserControl(devtools, "agent", (1024, 768)) await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) - await asyncio.sleep(0) + await asyncio.sleep(0.01) sent = [method for method, _, session in devtools.calls if session == "s-popup"] assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ @@ -601,10 +601,42 @@ def refused(params, session): control = BrowserControl(devtools, "agent", (1024, 768)) await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0.01) + + assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ + "Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog", + "Runtime.runIfWaitingForDebugger"] + + +class PausedUntilRun(StubDevTools): + """A browser whose popup, paused as it opened, answers Page.enable only once it's told to run, as Chromium does a + popup a click opened before its first navigation.""" + + def __init__(self): + super().__init__() + self.ran = asyncio.Event() + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + self.calls.append((method, params or {}, session_id)) + if method == "Runtime.runIfWaitingForDebugger": + self.ran.set() + elif method == "Page.enable" and session_id == "s-popup": + await asyncio.wait_for(self.ran.wait(), timeout_s or 30) + return {} + + +@pytest.mark.asyncio +async def test_a_popup_that_answers_page_enable_only_once_it_runs_is_told_to_run_at_once_held_to_its_window_first(): + devtools = PausedUntilRun() + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.wait_for(devtools.ran.wait(), 1) await asyncio.sleep(0) assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ - "Page.enable", "Runtime.runIfWaitingForDebugger"] + "Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog", + "Runtime.runIfWaitingForDebugger"] class Navigating(StubDevTools): @@ -1248,7 +1280,7 @@ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_ devtools.send("Page.javascriptDialogOpening", {"type": kind, "message": "Sure?", "url": "https://a.test/", "defaultPrompt": "yes"}, "s-a") - await asyncio.sleep(0) + await asyncio.sleep(0.01) devtools.send("Page.javascriptDialogClosed", {"result": answer == "accepted"}, "s-a") assert ("Page.enable", {}, "s-a") in devtools.calls