Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/agent_env/env/envs/playwright_mcp/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ set -euo pipefail
export MCP_PORT="${MCP_PORT:-18765}"
export RECORDER_PORT="${RECORDER_PORT:-18766}"
export PLAYWRIGHT_MCP_PORT="${PLAYWRIGHT_MCP_PORT:-18767}"
# The files Playwright MCP names itself go in one folder, not one it would name after the real time its first MCP
# session opened; the recorder names each screenshot or PDF a call leaves unnamed, which it would name after the real
# time too.
export PLAYWRIGHT_MCP_OUTPUT_DIR="${PLAYWRIGHT_MCP_OUTPUT_DIR:-/tmp/playwright-mcp-output}"
(while true; do /opt/recorder/bin/python /app/recorder.py || true; sleep 1; done) &
for _ in $(seq 1 "$(( ${RECORDER_WAIT_S:-20} * 10 ))"); do
(exec 3<>"/dev/tcp/127.0.0.1/${RECORDER_PORT}") 2>/dev/null && break
Expand All @@ -25,4 +29,5 @@ exec npx @playwright/mcp@${PLAYWRIGHT_MCP_VERSION} \
--host 127.0.0.1 \
--port "${PLAYWRIGHT_MCP_PORT}" \
--allowed-hosts '*' \
--caps vision
--caps vision \
--output-dir "${PLAYWRIGHT_MCP_OUTPUT_DIR}"
79 changes: 65 additions & 14 deletions src/agent_env/env/envs/playwright_mcp/recorder.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
"""The Playwright MCP browser's recorder: computer-recording/v1 of the page Playwright MCP drives, from Chrome DevTools
screencast frames. It is the front of the browser's MCP port: it serves the browser's card and the recording there, as
any MCP server serves its card at its own address, and passes every other request, streamed, to Playwright MCP on a
loopback port of its own behind it. It serves the card on ``RECORDER_PORT`` (18766) too, where deploys read it until
they read it at the MCP port. It is idle until a recording starts, which finds the browser through the remote debugging
port Playwright opens for an ``--isolated`` Chromium and follows the page the agent is on. It runs standalone in the
browser's image, so it imports only ``agentenv-framework-protocol[computer]`` and, to talk to the browser,
``websockets``."""
loopback port of its own behind it, naming the file of each screenshot or PDF a call leaves unnamed, which Playwright
MCP would name after the real time (``page-{timestamp}``), with a counter instead. It serves the card on
``RECORDER_PORT`` (18766) too, where deploys read it until they read it at the MCP port. It is idle until a recording
starts, which finds the browser through the remote debugging port Playwright opens for an ``--isolated`` Chromium and
follows the page the agent is on. It runs standalone in the browser's image, so it imports only
``agentenv-framework-protocol[computer]`` and, to talk to the browser, ``websockets``."""
from __future__ import annotations

import asyncio
Expand Down Expand Up @@ -49,6 +50,14 @@
#: How long passing a request to Playwright MCP may wait to connect; its answer may take as long as it does. How often
#: the front looks for Playwright MCP listening before it opens the MCP port.
CONNECT_S, LISTENING_POLL_S = 10.0, 0.2
#: Where Playwright MCP saves the files it names itself unless PLAYWRIGHT_MCP_OUTPUT_DIR says elsewhere (entrypoint.sh's
#: --output-dir), and the tools that save one named after the real time when the call names none, each with the
#: extension of what it saves by the call's arguments.
DEFAULT_OUTPUT_DIR = "/tmp/playwright-mcp-output"
SAVED_EXTENSIONS: dict[str, Callable[[dict], Optional[str]]] = {
"browser_take_screenshot": lambda arguments: {None: "png", "png": "png", "jpeg": "jpeg"}.get(arguments.get("type")),
"browser_pdf_save": lambda arguments: "pdf",
}
#: The headers that belong to one connection, which the front doesn't pass on; Host goes on as the caller sent it.
HOP_HEADERS = frozenset({"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer",
"transfer-encoding", "upgrade"})
Expand Down Expand Up @@ -301,26 +310,68 @@ def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]:
return self.driver.frames(hints)


def create_app(recorder: BrowserRecorder, behind: Optional[httpx.AsyncClient] = None) -> Any:
def create_app(recorder: BrowserRecorder, behind: Optional[httpx.AsyncClient] = None,
output_dir: Optional[Path] = None) -> Any:
"""The recorder's card and recording, and, as the front of the MCP port, every other request passed to ``behind``,
a client whose base URL is Playwright MCP's."""
a client whose base URL is Playwright MCP's, which saves the files it names itself in ``output_dir``
(PLAYWRIGHT_MCP_OUTPUT_DIR's by default)."""
app = Starlette()
AgentEnvStarletteApplication(recorder._build_card(), recorder).add_routes_to_app(app)
if behind is not None:
app.router.routes.append(Route("/{path:path}", _passing_to(behind), methods=[
names = FileNames(output_dir or Path(os.environ.get("PLAYWRIGHT_MCP_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)))
app.router.routes.append(Route("/{path:path}", _passing_to(behind, names), methods=[
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]))
return app


def _passing_to(behind: httpx.AsyncClient) -> Callable:
class FileNames:
"""Names for the files Playwright MCP saves for a call that names none: ``page-1.png``, ``page-2.pdf`` and on, each
one no file in ``folder`` has, so a recorder that restarted names none Playwright MCP saved before."""

def __init__(self, folder: Path) -> None:
self.folder = folder
self._next = itertools.count(1)

def next(self, extension: str) -> str:
for n in self._next:
name = f"page-{n}.{extension}"
if not (self.folder / name).exists():
return name


def named(body: bytes, names: FileNames) -> bytes:
"""``body``, a JSON-RPC message or batch to Playwright MCP, with a file name from ``names`` given to each call of a
tool in ``SAVED_EXTENSIONS`` that names none; as it came when there's none, or it isn't JSON."""
try:
message = json.loads(body)
except ValueError:
return body
changed = False
for call in message if isinstance(message, list) else [message]:
params = call.get("params") if isinstance(call, dict) and call.get("method") == "tools/call" else None
if not isinstance(params, dict) or params.get("name") not in SAVED_EXTENSIONS:
continue
arguments = params.get("arguments") or {}
if not isinstance(arguments, dict) or arguments.get("filename"):
continue
extension = SAVED_EXTENSIONS[params["name"]](arguments)
if extension is not None:
params["arguments"], changed = {**arguments, "filename": names.next(extension)}, True
return json.dumps(message).encode() if changed else body


def _passing_to(behind: httpx.AsyncClient, names: FileNames) -> Callable:
async def passed(request: Any) -> Any:
"""The request passed to Playwright MCP as it came, its body and its answer streamed, or 502 when Playwright
MCP doesn't answer."""
MCP doesn't answer; a JSON body is read whole, for ``named``."""
has_body = request.method not in ("GET", "HEAD", "OPTIONS") or "content-length" in request.headers
sent = behind.build_request(
request.method, request.url.path, params=request.url.query or None,
headers=[(key, value) for key, value in request.headers.items() if key.lower() not in HOP_HEADERS],
content=request.stream() if has_body else None)
headers = [(key, value) for key, value in request.headers.items() if key.lower() not in HOP_HEADERS]
content: Any = request.stream() if has_body else None
if has_body and request.headers.get("content-type", "").split(";")[0].strip().lower() == "application/json":
content = named(await request.body(), names)
headers = [(key, value) for key, value in headers if key.lower() != "content-length"]
sent = behind.build_request(request.method, request.url.path, params=request.url.query or None,
headers=headers, content=content)
try:
answer = await behind.send(sent, stream=True)
except httpx.TransportError as e:
Expand Down
17 changes: 14 additions & 3 deletions src/agent_env/task_step/task_steps/sync_env_clock.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@
(``agent_env.a2a_agent.clock``), so every program it runs reads the env's time and later clock writes can wait for it.
An agent that can't is skipped, as every agent is until its image runs the follower, or fails the step with
require_agent_clock.
Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. Place right
before prompt_agent so the virtual timeline starts at agent-start; when setup needs the clock armed, arm it at rate 0
and start it with set_env_clock_rate right before prompt_agent.
Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. A multi env's website
browser that is Playwright MCP (``playwright-mcp``) stays on real time, so its pages read the real date: the step warns
and records it (``real_time_browsers``); the browser computer (``browser_envs = ["chromium"]``) follows the clock.
Place right before prompt_agent so the virtual timeline starts at agent-start; when setup needs the clock armed, arm it
at rate 0 and start it with set_env_clock_rate right before prompt_agent.
"""

from __future__ import annotations
Expand All @@ -26,6 +28,7 @@
from agentenv_protocol import client as protocol_v1

from agent_env.env.env import EnvCapabilityUnsupported
from agent_env.env.envs.playwright_mcp import PLAYWRIGHT_MCP_ENVIRONMENT_NAME
from agent_env.env.gateway.constants import EXT_CLOCK_URI
from agent_env.task_step.context import StepContextError, TaskStepContext
from agent_env.entity.refs import EntityRef
Expand Down Expand Up @@ -182,6 +185,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
if self.timezone is not None:
configuration.update(timezone=self.timezone, armed_at=self.armed_virtual_time,
get_time_zone=state.get("time_zone"))
real_time_browsers = [s["service"] for s in skipped
if s["service"] == PLAYWRIGHT_MCP_ENVIRONMENT_NAME and not s.get("website")]
if real_time_browsers:
configuration["real_time_browsers"] = real_time_browsers
logger.warning(f"sync_env_clock env={self.env_id}: its website browser {real_time_browsers[0]!r}, "
"Playwright MCP, stays on real time, so the pages it opens read the real date in their "
"JavaScript; name the browser computer as the env's website browser "
"(browser_envs = [\"chromium\"]) for a browser on the env's clock")
context.metadata.setdefault("clock_configurations", []).append(configuration)
logger.info(f"sync_env_clock env={self.env_id}: armed rate={self.virtual_seconds_per_real_second}; "
f"synced={[s['service'] for s in synced]} skipped={[s['service'] for s in skipped]} "
Expand Down
21 changes: 19 additions & 2 deletions tst/integration/env/envs/playwright_mcp_local_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@
browser's card is served there, as every MCP server serves its card, and on the recorder's own port too, where deploys
read it until they read it at the MCP port, the card first and the MCP port only once Playwright MCP answers behind it,
which is what the gateway's health check asks; an MCP client lists and calls the browser's tools through the front; a
recording through the card on the MCP port films the page the client opens; and a killed recorder comes back, the front
with it, while the MCP server behind it stays up. Needs Docker."""
recording through the card on the MCP port films the page the client opens; a screenshot the client leaves unnamed is
saved and named to it with no real time in the name or its folder; and a killed recorder comes back, the front with it,
while the MCP server behind it stays up. Needs Docker."""

import asyncio
import json
import re
import subprocess
import time
import uuid
Expand Down Expand Up @@ -183,6 +185,21 @@ async def test_a_recording_through_the_card_on_the_mcp_port_films_the_page_an_ag
assert sum(segment["frames"] for segment in manifest["segments"]) >= 3, manifest


@pytest.mark.asyncio
async def test_a_screenshot_the_client_leaves_unnamed_is_named_with_a_counter_in_one_folder_and_no_real_time(browser):
"""Playwright MCP names one after the real time (``page-{ISO time}.png`` in ``<epoch ms>/``) and says so."""
_, mcp, _ = browser
async with streamable_http_client(f"{mcp}/mcp") as (read, write, _), ClientSession(read, write) as session:
await session.initialize()
await session.call_tool("browser_navigate", {"url": PAGE})
said = ["".join(getattr(part, "text", "") for part in (await session.call_tool(
"browser_take_screenshot", arguments)).content) for arguments in ({}, {"type": "jpeg"})]

assert "saved it as /tmp/playwright-mcp-output/page-1.png" in said[0], said[0]
assert "saved it as /tmp/playwright-mcp-output/page-2.jpeg" in said[1], said[1]
assert not [text for text in said if re.search(r"\d{4}-\d\d-\d\dT|\b\d{13}\b", text)], said


@pytest.mark.asyncio
async def test_a_killed_recorder_comes_back_the_front_with_it_and_the_mcp_server_behind_it_stays_up(browser):
name, mcp, _ = browser
Expand Down
47 changes: 46 additions & 1 deletion tst/unit/env/envs/test_playwright_mcp_recorder.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
through DevTools target events, holds a still page's frame so the recording doesn't split, and serves
computer-recording/v1 on a card of its own that the gateway nests as the playwright-mcp child env. It fronts the MCP
port: the card and recording there, every other request passed to Playwright MCP behind it, streamed, and 502 when
Playwright MCP doesn't answer."""
Playwright MCP doesn't answer; a screenshot or PDF a call leaves unnamed, which Playwright MCP would name after the real
time, is named with a counter, in the one folder Playwright MCP is given for its files."""

from __future__ import annotations

Expand All @@ -11,6 +12,7 @@
import contextlib
import io
import itertools
import json
import re
import socket
from pathlib import Path
Expand Down Expand Up @@ -242,6 +244,13 @@ def default(name: str) -> int:
assert "--isolated" in entrypoint # the remote debugging port the recorder finds comes with it


def test_the_image_gives_playwright_mcp_one_folder_for_its_files_where_the_front_looks():
entrypoint = (Path(playwright_mcp.__file__).parent / "entrypoint.sh").read_text()

assert f'PLAYWRIGHT_MCP_OUTPUT_DIR="${{PLAYWRIGHT_MCP_OUTPUT_DIR:-{recorder.DEFAULT_OUTPUT_DIR}}}"' in entrypoint
assert '--output-dir "${PLAYWRIGHT_MCP_OUTPUT_DIR}"' in entrypoint


def _behind(app) -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://playwright-mcp")

Expand Down Expand Up @@ -412,3 +421,39 @@ async def stream(reader, writer):

assert answer.headers["content-type"] == "text/event-stream" and first == "event: message"
assert rest[:1] == ["data: 1"]


def _call(tool: str, call_id: int = 1, **arguments) -> dict:
return {"jsonrpc": "2.0", "id": call_id, "method": "tools/call", "params": {"name": tool, "arguments": arguments}}


@pytest.mark.asyncio
async def test_the_front_names_each_screenshot_or_pdf_a_call_leaves_unnamed_with_a_counter_not_the_time(tmp_path):
"""Playwright MCP names such a file ``page-{timestamp}`` from the real time and says so to the model."""
(tmp_path / "page-4.png").write_bytes(b"saved before the recorder restarted")
passed = []

async def mcp(request):
body = await request.body()
passed.append((body, int(request.headers["content-length"]) == len(body)))
return JSONResponse({"jsonrpc": "2.0", "id": 1, "result": {}})

sent = [_call("browser_take_screenshot"), _call("browser_take_screenshot", type="jpeg", fullPage=True),
_call("browser_pdf_save"), _call("browser_take_screenshot", filename="mine.png"),
_call("browser_navigate", url="https://shop.test/"), _call("browser_take_screenshot", type="gif"),
[_call("browser_take_screenshot", 7), {"jsonrpc": "2.0", "method": "notifications/initialized"}]]
async with _behind(Starlette(routes=[Route("/mcp", mcp, methods=["POST"])])) as behind:
front = recorder.create_app(_recorder(_Browser()), behind, output_dir=tmp_path)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=front), base_url="http://front") as http:
for body in sent:
await http.post("/mcp", json=body)
await http.post("/mcp", content=b"not json", headers={"content-type": "application/json"})

received = [json.loads(body) for body, _ in passed[:-1]]
names = [[call.get("params", {}).get("arguments", {}).get("filename") for call in
(body if isinstance(body, list) else [body])] for body in received]
assert names == [["page-1.png"], ["page-2.jpeg"], ["page-3.pdf"], ["mine.png"], [None], [None],
["page-5.png", None]]
assert received[1]["params"]["arguments"] == {"type": "jpeg", "fullPage": True, "filename": "page-2.jpeg"}
assert all(length_matches for _, length_matches in passed)
assert passed[-1][0] == b"not json"
19 changes: 19 additions & 0 deletions tst/unit/task_step/sync_env_clock_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,25 @@ async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch)
assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email"]


@pytest.mark.asyncio
async def test_a_website_browser_on_real_time_is_named_in_a_warning_and_the_record(monkeypatch, caplog):
"""Playwright MCP serves no clock/v1, so the pages it opens read the real date while every server reads the env's."""
_mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL})
monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email"))
browsing = TaskStepContext(deployed_envs=[_carded(await _composed_card(
{"mcp-email": [_CLOCK_EXT], "mcp-playwright-mcp": []}))])
alone = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))])

with caplog.at_level("WARNING"):
await _step().execute(browsing)
await _step().execute(alone)

assert browsing.metadata["clock_configurations"][0]["real_time_browsers"] == ["playwright-mcp"]
assert "real_time_browsers" not in alone.metadata["clock_configurations"][0]
assert caplog.text.count("its website browser 'playwright-mcp', Playwright MCP, stays on real time") == 1
assert 'browser_envs = ["chromium"]' in caplog.text


@pytest.mark.asyncio
async def test_execute_keeps_the_arm_failure_message(monkeypatch):
_mock_http(monkeypatch, set_time_status=409)
Expand Down
Loading