Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1204,7 +1204,8 @@ are the profile's template. The base class adds:
- **a video** of the tab its context's agent is on, at `fps` (10 by default, 15 at most), encoded as
`computer-recording/v1`'s is and uploaded a segment at a time under `<recording_id>/video/`. It follows the tab an
agent last acted on in that context (CDP commands and MCP calls, not agentenv's own), a tab that one opens as
soon as it opens, and once it closes the tab followed before it, else the context's newest; it's blank while the
soon as it opens, which an action begun on the opener before then, as the click that opened it, doesn't take it
back from, and once it closes the tab followed before it, else the context's newest; it's blank while the
context has no tab. The browser's own connection screencasts that tab, and captures it instead while it sends no
frame, as a tab behind another in its window does; `video.followed` says which tab showed when, and how.
- **its log**: at the end, the lines of its context since its start, actions and tab events on its tabs and a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1172,21 +1172,24 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N

async def _let_run(self, session: str) -> None:
"""A page it attached to, enabled and, alone, held to its window's size, then told to run: a page attached as
it opens, a popup say, or any new page alone, waits for that before it runs."""
it opens, a popup say, or any new page alone, waits for that before it runs. The calls go together, in order,
which the browser handles them in: a popup a click opened answers Page.enable only once it runs."""
target = self._sessions.get(session)
calls: list[tuple[str, Optional[dict]]] = [("Page.enable", None)]
if self.viewport is not None:
width, height = self.viewport
calls += [("Emulation.setDeviceMetricsOverride",
{"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}),
("Page.setInterceptFileChooserDialog", {"enabled": True})]
calls.append(("Runtime.runIfWaitingForDebugger", None))
try:
await self.devtools.call("Page.enable", session_id=session)
if self.viewport is not None:
width, height = self.viewport
await self.devtools.call("Emulation.setDeviceMetricsOverride", {
"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session)
await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session)
answers = await asyncio.gather(*(self.devtools.call(method, params, session_id=session)
for method, params in calls), return_exceptions=True)
finally:
try:
await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session)
finally:
if target is not None:
self._pinned.setdefault(target, asyncio.Event()).set()
if target is not None:
self._pinned.setdefault(target, asyncio.Event()).set()
if failed := next((answer for answer in answers if isinstance(answer, BaseException)), None):
raise failed

def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None:
self._screencasts.pop(params["sessionId"], None)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ def tab_event(self, event: str, tab: dict) -> None:
if isinstance(target := tab.get("id"), str):
self.tab_contexts[target] = tab.get("context_id")
for recording in self.every():
recording.stream.tab_event(event, tab)
recording.stream.tab_event(event, tab, self.log.next_seq)

def context_of_tab(self, target: object) -> Optional[str]:
"""The context of the tab ``target``, as it opened in, or as the browser's control has it."""
Expand Down Expand Up @@ -419,7 +419,7 @@ def acted(self, seq: int, tab: Optional[str], moves: bool = True) -> None:
if self.ending or not self.stream.sees(tab):
return
if moves:
self.stream.follow(tab)
self.stream.follow(tab, seq)
if not self.frames_on:
return
self.pending.append(seq)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""The frames of one browser context as its recording's video takes them (browser-recording/v1): of the tab it
follows, the one an agent last acted on there, a tab that one opens as soon as it opens, and once it closes the tab
followed before it that's still open, else the context's newest; a blank frame while the context has no tab. The
follows, the one an agent last acted on there, a tab that one opens as soon as it opens, which an action begun on
the opener before it opened doesn't take the video back from, and once it closes the tab followed before it that's
still open, else the context's newest; a blank frame while the context has no tab. The
browser's own connection screencasts that tab; a tab that sends no frame for ``SILENT_S`` and isn't visible, as one
behind another in its window, is captured instead, at most ``FALLBACK_FPS`` times a second, until it sends one again,
while a visible one is a still page its last frame shows. Every ``1/fps`` seconds the latest frame is handed on, so a
Expand Down Expand Up @@ -56,23 +57,30 @@ def __init__(self, control: BrowserControl, context_id: Optional[str], blank_siz
self._seeding: Optional[asyncio.Future] = None
self._max_edge: Optional[int] = None
self._via: Optional[str] = None
#: Each tab that opened one followed, and the log's next sequence number then: an action on it numbered below
#: began before that tab opened.
self._opened_from: dict[str, int] = {}

def sees(self, target: Optional[str]) -> bool:
"""Whether the tab ``target`` is in its context."""
return bool(target) and self.control.in_context(self.control.context_of_tab(target), self.context_id)

def follow(self, target: Optional[str]) -> None:
"""An agent acted on the tab ``target``: one in its context is followed from now."""
if self.sees(target) and target != self.tab:
def follow(self, target: Optional[str], seq: int) -> None:
"""An agent's action ``seq`` in the log acted on the tab ``target``: one in its context is followed from now,
unless the action began before a tab that ``target`` opened, which the video has followed since."""
if self.sees(target) and target != self.tab and seq >= self._opened_from.get(target, 0):
self._move(target)

def tab_event(self, event: str, tab: dict) -> None:
"""A tab opened or closed in the browser: one the followed tab opens, or the first its context has, is
followed; once the followed tab closes, the one followed before it that's still open, else the newest."""
def tab_event(self, event: str, tab: dict, next_seq: int) -> None:
"""A tab opened or closed in the browser while ``next_seq`` was the log's next sequence number: one the
followed tab opens, or the first its context has, is followed; once the followed tab closes, the one followed
before it that's still open, else the newest."""
target = tab.get("id")
if not self.control.in_context(tab.get("context_id"), self.context_id):
return
if event == "opened" and (self.tab is None or tab.get("opener") == self.tab):
if self.tab is not None:
self._opened_from[self.tab] = next_seq
self._move(target)
elif event == "closed" and target == self.tab:
self._move(next((t for t in reversed(self._history) if t in self.control.tabs and self.sees(t)), None) or
Expand Down
26 changes: 26 additions & 0 deletions packages/agentenv-protocol/tests/test_browser_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,32 @@ async def test_the_video_follows_an_agents_tab_a_popup_it_opens_and_goes_back_wh
assert "green" not in colors[colors.index("yellow"):], colors


@pytest.mark.asyncio
async def test_a_click_that_opens_a_popup_leaves_the_video_on_it_though_the_click_is_answered_after(browser):
"""The browser opens the popup a click makes before it answers the click, which the log hears of then; an action
on the opener begun after it opened takes the video back."""
pipe, control, contexts, network, recordings, uploads = browser
pipe.open("T3", YELLOW)
pipe.focused = "T3"
started = await recordings.start("run:a", _grant(), network=False)
await asyncio.sleep(0.3)
click = recordings.log.begin({"via": "cdp", "who": "ada", "method": "Input.dispatchMouseEvent", "acts": True,
"tab": "T3"})
pipe.open("T4", RED, opener="T3")
await asyncio.sleep(0.1)
recordings.log.finish(click, ok=True)
await asyncio.sleep(0.3)
followed_in_the_popup = [seen["tab"] for seen in recordings.every()[0].stream.followed]
_acted(recordings.log, "T3")
await asyncio.sleep(0.3)
manifest = BrowserRecordingManifest.model_validate(await recordings.end(started["recording_id"]))

assert followed_in_the_popup == ["T3", "T4"]
assert [seen.tab for seen in manifest.video.followed] == ["T3", "T4", "T3"]
colors = uploads.colors(f"{ROOT}/{manifest.recording_id}")
assert colors[0] == "yellow" and "red" in colors and colors[-1] == "yellow", colors


@pytest.mark.asyncio
async def test_a_clients_setup_on_every_page_it_attaches_to_doesnt_move_the_video_and_its_input_does(browser):
pipe, control, contexts, network, recordings, uploads = browser
Expand Down
40 changes: 36 additions & 4 deletions packages/agentenv-protocol/tests/test_website_browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,7 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to
control = BrowserControl(devtools, "agent")
await control.attach()
devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")})
await asyncio.sleep(0)
await asyncio.sleep(0.01)

assert ("Page.enable", {}, "s-popup") in devtools.calls
assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls
Expand All @@ -581,7 +581,7 @@ async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_ru
control = BrowserControl(devtools, "agent", (1024, 768))
await control.attach()
devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")})
await asyncio.sleep(0)
await asyncio.sleep(0.01)

sent = [method for method, _, session in devtools.calls if session == "s-popup"]
assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[
Expand All @@ -601,10 +601,42 @@ def refused(params, session):
control = BrowserControl(devtools, "agent", (1024, 768))
await control.attach()
devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")})
await asyncio.sleep(0.01)

assert [method for method, _, session in devtools.calls if session == "s-popup"] == [
"Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog",
"Runtime.runIfWaitingForDebugger"]


class PausedUntilRun(StubDevTools):
"""A browser whose popup, paused as it opened, answers Page.enable only once it's told to run, as Chromium does a
popup a click opened before its first navigation."""

def __init__(self):
super().__init__()
self.ran = asyncio.Event()

async def call(self, method, params=None, *, session_id=None, timeout_s=None):
self.calls.append((method, params or {}, session_id))
if method == "Runtime.runIfWaitingForDebugger":
self.ran.set()
elif method == "Page.enable" and session_id == "s-popup":
await asyncio.wait_for(self.ran.wait(), timeout_s or 30)
return {}


@pytest.mark.asyncio
async def test_a_popup_that_answers_page_enable_only_once_it_runs_is_told_to_run_at_once_held_to_its_window_first():
devtools = PausedUntilRun()
control = BrowserControl(devtools, "agent", (1024, 768))
await control.attach()
devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")})
await asyncio.wait_for(devtools.ran.wait(), 1)
await asyncio.sleep(0)

assert [method for method, _, session in devtools.calls if session == "s-popup"] == [
"Page.enable", "Runtime.runIfWaitingForDebugger"]
"Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog",
"Runtime.runIfWaitingForDebugger"]


class Navigating(StubDevTools):
Expand Down Expand Up @@ -1248,7 +1280,7 @@ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_

devtools.send("Page.javascriptDialogOpening", {"type": kind, "message": "Sure?", "url": "https://a.test/",
"defaultPrompt": "yes"}, "s-a")
await asyncio.sleep(0)
await asyncio.sleep(0.01)
devtools.send("Page.javascriptDialogClosed", {"result": answer == "accepted"}, "s-a")

assert ("Page.enable", {}, "s-a") in devtools.calls
Expand Down
Loading