diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 94cd518fe..cd70d4e0f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1204,7 +1204,8 @@ are the profile's template. The base class adds: - **a video** of the tab its context's agent is on, at `fps` (10 by default, 15 at most), encoded as `computer-recording/v1`'s is and uploaded a segment at a time under `/video/`. It follows the tab an agent last acted on in that context (CDP commands and MCP calls, not agentenv's own), a tab that one opens as - soon as it opens, and once it closes the tab followed before it, else the context's newest; it's blank while the + soon as it opens, which an action begun on the opener before then, as the click that opened it, doesn't take it + back from, and once it closes the tab followed before it, else the context's newest; it's blank while the context has no tab. The browser's own connection screencasts that tab, and captures it instead while it sends no frame, as a tab behind another in its window does; `video.followed` says which tab showed when, and how. - **its log**: at the end, the lines of its context since its start, actions and tab events on its tabs and a diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index b4ac27a1a..2f43e12ce 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -1172,21 +1172,24 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N async def _let_run(self, session: str) -> None: """A page it attached to, enabled and, alone, held to its window's size, then told to run: a page attached as - it opens, a popup say, or any new page alone, waits for that before it runs.""" + it opens, a popup say, or any new page alone, waits for that before it runs. The calls go together, in order, + which the browser handles them in: a popup a click opened answers Page.enable only once it runs.""" target = self._sessions.get(session) + calls: list[tuple[str, Optional[dict]]] = [("Page.enable", None)] + if self.viewport is not None: + width, height = self.viewport + calls += [("Emulation.setDeviceMetricsOverride", + {"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}), + ("Page.setInterceptFileChooserDialog", {"enabled": True})] + calls.append(("Runtime.runIfWaitingForDebugger", None)) try: - await self.devtools.call("Page.enable", session_id=session) - if self.viewport is not None: - width, height = self.viewport - await self.devtools.call("Emulation.setDeviceMetricsOverride", { - "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) - await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session) + answers = await asyncio.gather(*(self.devtools.call(method, params, session_id=session) + for method, params in calls), return_exceptions=True) finally: - try: - await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) - finally: - if target is not None: - self._pinned.setdefault(target, asyncio.Event()).set() + if target is not None: + self._pinned.setdefault(target, asyncio.Event()).set() + if failed := next((answer for answer in answers if isinstance(answer, BaseException)), None): + raise failed def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: self._screencasts.pop(params["sessionId"], None) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py index f2419bbf4..0a00cca61 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/recording.py @@ -312,7 +312,7 @@ def tab_event(self, event: str, tab: dict) -> None: if isinstance(target := tab.get("id"), str): self.tab_contexts[target] = tab.get("context_id") for recording in self.every(): - recording.stream.tab_event(event, tab) + recording.stream.tab_event(event, tab, self.log.next_seq) def context_of_tab(self, target: object) -> Optional[str]: """The context of the tab ``target``, as it opened in, or as the browser's control has it.""" @@ -419,7 +419,7 @@ def acted(self, seq: int, tab: Optional[str], moves: bool = True) -> None: if self.ending or not self.stream.sees(tab): return if moves: - self.stream.follow(tab) + self.stream.follow(tab, seq) if not self.frames_on: return self.pending.append(seq) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py index ca2b3e213..929a87988 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/stream.py @@ -1,6 +1,7 @@ """The frames of one browser context as its recording's video takes them (browser-recording/v1): of the tab it -follows, the one an agent last acted on there, a tab that one opens as soon as it opens, and once it closes the tab -followed before it that's still open, else the context's newest; a blank frame while the context has no tab. The +follows, the one an agent last acted on there, a tab that one opens as soon as it opens, which an action begun on +the opener before it opened doesn't take the video back from, and once it closes the tab followed before it that's +still open, else the context's newest; a blank frame while the context has no tab. The browser's own connection screencasts that tab; a tab that sends no frame for ``SILENT_S`` and isn't visible, as one behind another in its window, is captured instead, at most ``FALLBACK_FPS`` times a second, until it sends one again, while a visible one is a still page its last frame shows. Every ``1/fps`` seconds the latest frame is handed on, so a @@ -56,23 +57,30 @@ def __init__(self, control: BrowserControl, context_id: Optional[str], blank_siz self._seeding: Optional[asyncio.Future] = None self._max_edge: Optional[int] = None self._via: Optional[str] = None + #: Each tab that opened one followed, and the log's next sequence number then: an action on it numbered below + #: began before that tab opened. + self._opened_from: dict[str, int] = {} def sees(self, target: Optional[str]) -> bool: """Whether the tab ``target`` is in its context.""" return bool(target) and self.control.in_context(self.control.context_of_tab(target), self.context_id) - def follow(self, target: Optional[str]) -> None: - """An agent acted on the tab ``target``: one in its context is followed from now.""" - if self.sees(target) and target != self.tab: + def follow(self, target: Optional[str], seq: int) -> None: + """An agent's action ``seq`` in the log acted on the tab ``target``: one in its context is followed from now, + unless the action began before a tab that ``target`` opened, which the video has followed since.""" + if self.sees(target) and target != self.tab and seq >= self._opened_from.get(target, 0): self._move(target) - def tab_event(self, event: str, tab: dict) -> None: - """A tab opened or closed in the browser: one the followed tab opens, or the first its context has, is - followed; once the followed tab closes, the one followed before it that's still open, else the newest.""" + def tab_event(self, event: str, tab: dict, next_seq: int) -> None: + """A tab opened or closed in the browser while ``next_seq`` was the log's next sequence number: one the + followed tab opens, or the first its context has, is followed; once the followed tab closes, the one followed + before it that's still open, else the newest.""" target = tab.get("id") if not self.control.in_context(tab.get("context_id"), self.context_id): return if event == "opened" and (self.tab is None or tab.get("opener") == self.tab): + if self.tab is not None: + self._opened_from[self.tab] = next_seq self._move(target) elif event == "closed" and target == self.tab: self._move(next((t for t in reversed(self._history) if t in self.control.tabs and self.sees(t)), None) or diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 06d3e7685..6e9ef69e7 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -228,6 +228,32 @@ async def test_the_video_follows_an_agents_tab_a_popup_it_opens_and_goes_back_wh assert "green" not in colors[colors.index("yellow"):], colors +@pytest.mark.asyncio +async def test_a_click_that_opens_a_popup_leaves_the_video_on_it_though_the_click_is_answered_after(browser): + """The browser opens the popup a click makes before it answers the click, which the log hears of then; an action + on the opener begun after it opened takes the video back.""" + pipe, control, contexts, network, recordings, uploads = browser + pipe.open("T3", YELLOW) + pipe.focused = "T3" + started = await recordings.start("run:a", _grant(), network=False) + await asyncio.sleep(0.3) + click = recordings.log.begin({"via": "cdp", "who": "ada", "method": "Input.dispatchMouseEvent", "acts": True, + "tab": "T3"}) + pipe.open("T4", RED, opener="T3") + await asyncio.sleep(0.1) + recordings.log.finish(click, ok=True) + await asyncio.sleep(0.3) + followed_in_the_popup = [seen["tab"] for seen in recordings.every()[0].stream.followed] + _acted(recordings.log, "T3") + await asyncio.sleep(0.3) + manifest = BrowserRecordingManifest.model_validate(await recordings.end(started["recording_id"])) + + assert followed_in_the_popup == ["T3", "T4"] + assert [seen.tab for seen in manifest.video.followed] == ["T3", "T4", "T3"] + colors = uploads.colors(f"{ROOT}/{manifest.recording_id}") + assert colors[0] == "yellow" and "red" in colors and colors[-1] == "yellow", colors + + @pytest.mark.asyncio async def test_a_clients_setup_on_every_page_it_attaches_to_doesnt_move_the_video_and_its_input_does(browser): pipe, control, contexts, network, recordings, uploads = browser diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 111a72247..0a920859a 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -566,7 +566,7 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to control = BrowserControl(devtools, "agent") await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) - await asyncio.sleep(0) + await asyncio.sleep(0.01) assert ("Page.enable", {}, "s-popup") in devtools.calls assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls @@ -581,7 +581,7 @@ async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_ru control = BrowserControl(devtools, "agent", (1024, 768)) await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) - await asyncio.sleep(0) + await asyncio.sleep(0.01) sent = [method for method, _, session in devtools.calls if session == "s-popup"] assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ @@ -601,10 +601,42 @@ def refused(params, session): control = BrowserControl(devtools, "agent", (1024, 768)) await control.attach() devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0.01) + + assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ + "Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog", + "Runtime.runIfWaitingForDebugger"] + + +class PausedUntilRun(StubDevTools): + """A browser whose popup, paused as it opened, answers Page.enable only once it's told to run, as Chromium does a + popup a click opened before its first navigation.""" + + def __init__(self): + super().__init__() + self.ran = asyncio.Event() + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + self.calls.append((method, params or {}, session_id)) + if method == "Runtime.runIfWaitingForDebugger": + self.ran.set() + elif method == "Page.enable" and session_id == "s-popup": + await asyncio.wait_for(self.ran.wait(), timeout_s or 30) + return {} + + +@pytest.mark.asyncio +async def test_a_popup_that_answers_page_enable_only_once_it_runs_is_told_to_run_at_once_held_to_its_window_first(): + devtools = PausedUntilRun() + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.wait_for(devtools.ran.wait(), 1) await asyncio.sleep(0) assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ - "Page.enable", "Runtime.runIfWaitingForDebugger"] + "Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog", + "Runtime.runIfWaitingForDebugger"] class Navigating(StubDevTools): @@ -1248,7 +1280,7 @@ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_ devtools.send("Page.javascriptDialogOpening", {"type": kind, "message": "Sure?", "url": "https://a.test/", "defaultPrompt": "yes"}, "s-a") - await asyncio.sleep(0) + await asyncio.sleep(0.01) devtools.send("Page.javascriptDialogClosed", {"result": answer == "accepted"}, "s-a") assert ("Page.enable", {}, "s-a") in devtools.calls