diff --git a/src/agent_env/env/envs/playwright_mcp/entrypoint.sh b/src/agent_env/env/envs/playwright_mcp/entrypoint.sh index 519027ec5..c14527882 100644 --- a/src/agent_env/env/envs/playwright_mcp/entrypoint.sh +++ b/src/agent_env/env/envs/playwright_mcp/entrypoint.sh @@ -11,6 +11,10 @@ set -euo pipefail export MCP_PORT="${MCP_PORT:-18765}" export RECORDER_PORT="${RECORDER_PORT:-18766}" export PLAYWRIGHT_MCP_PORT="${PLAYWRIGHT_MCP_PORT:-18767}" +# The files Playwright MCP names itself go in one folder, not one it would name after the real time its first MCP +# session opened; the recorder names each screenshot or PDF a call leaves unnamed, which it would name after the real +# time too. +export PLAYWRIGHT_MCP_OUTPUT_DIR="${PLAYWRIGHT_MCP_OUTPUT_DIR:-/tmp/playwright-mcp-output}" (while true; do /opt/recorder/bin/python /app/recorder.py || true; sleep 1; done) & for _ in $(seq 1 "$(( ${RECORDER_WAIT_S:-20} * 10 ))"); do (exec 3<>"/dev/tcp/127.0.0.1/${RECORDER_PORT}") 2>/dev/null && break @@ -25,4 +29,5 @@ exec npx @playwright/mcp@${PLAYWRIGHT_MCP_VERSION} \ --host 127.0.0.1 \ --port "${PLAYWRIGHT_MCP_PORT}" \ --allowed-hosts '*' \ - --caps vision + --caps vision \ + --output-dir "${PLAYWRIGHT_MCP_OUTPUT_DIR}" diff --git a/src/agent_env/env/envs/playwright_mcp/recorder.py b/src/agent_env/env/envs/playwright_mcp/recorder.py index 25ec8bc6e..8f94e10f7 100644 --- a/src/agent_env/env/envs/playwright_mcp/recorder.py +++ b/src/agent_env/env/envs/playwright_mcp/recorder.py @@ -1,11 +1,12 @@ """The Playwright MCP browser's recorder: computer-recording/v1 of the page Playwright MCP drives, from Chrome DevTools screencast frames. It is the front of the browser's MCP port: it serves the browser's card and the recording there, as any MCP server serves its card at its own address, and passes every other request, streamed, to Playwright MCP on a -loopback port of its own behind it. It serves the card on ``RECORDER_PORT`` (18766) too, where deploys read it until -they read it at the MCP port. It is idle until a recording starts, which finds the browser through the remote debugging -port Playwright opens for an ``--isolated`` Chromium and follows the page the agent is on. It runs standalone in the -browser's image, so it imports only ``agentenv-framework-protocol[computer]`` and, to talk to the browser, -``websockets``.""" +loopback port of its own behind it, naming the file of each screenshot or PDF a call leaves unnamed, which Playwright +MCP would name after the real time (``page-{timestamp}``), with a counter instead. It serves the card on +``RECORDER_PORT`` (18766) too, where deploys read it until they read it at the MCP port. It is idle until a recording +starts, which finds the browser through the remote debugging port Playwright opens for an ``--isolated`` Chromium and +follows the page the agent is on. It runs standalone in the browser's image, so it imports only +``agentenv-framework-protocol[computer]`` and, to talk to the browser, ``websockets``.""" from __future__ import annotations import asyncio @@ -49,6 +50,14 @@ #: How long passing a request to Playwright MCP may wait to connect; its answer may take as long as it does. How often #: the front looks for Playwright MCP listening before it opens the MCP port. CONNECT_S, LISTENING_POLL_S = 10.0, 0.2 +#: Where Playwright MCP saves the files it names itself unless PLAYWRIGHT_MCP_OUTPUT_DIR says elsewhere (entrypoint.sh's +#: --output-dir), and the tools that save one named after the real time when the call names none, each with the +#: extension of what it saves by the call's arguments. +DEFAULT_OUTPUT_DIR = "/tmp/playwright-mcp-output" +SAVED_EXTENSIONS: dict[str, Callable[[dict], Optional[str]]] = { + "browser_take_screenshot": lambda arguments: {None: "png", "png": "png", "jpeg": "jpeg"}.get(arguments.get("type")), + "browser_pdf_save": lambda arguments: "pdf", +} #: The headers that belong to one connection, which the front doesn't pass on; Host goes on as the caller sent it. HOP_HEADERS = frozenset({"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade"}) @@ -301,26 +310,68 @@ def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: return self.driver.frames(hints) -def create_app(recorder: BrowserRecorder, behind: Optional[httpx.AsyncClient] = None) -> Any: +def create_app(recorder: BrowserRecorder, behind: Optional[httpx.AsyncClient] = None, + output_dir: Optional[Path] = None) -> Any: """The recorder's card and recording, and, as the front of the MCP port, every other request passed to ``behind``, - a client whose base URL is Playwright MCP's.""" + a client whose base URL is Playwright MCP's, which saves the files it names itself in ``output_dir`` + (PLAYWRIGHT_MCP_OUTPUT_DIR's by default).""" app = Starlette() AgentEnvStarletteApplication(recorder._build_card(), recorder).add_routes_to_app(app) if behind is not None: - app.router.routes.append(Route("/{path:path}", _passing_to(behind), methods=[ + names = FileNames(output_dir or Path(os.environ.get("PLAYWRIGHT_MCP_OUTPUT_DIR", DEFAULT_OUTPUT_DIR))) + app.router.routes.append(Route("/{path:path}", _passing_to(behind, names), methods=[ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"])) return app -def _passing_to(behind: httpx.AsyncClient) -> Callable: +class FileNames: + """Names for the files Playwright MCP saves for a call that names none: ``page-1.png``, ``page-2.pdf`` and on, each + one no file in ``folder`` has, so a recorder that restarted names none Playwright MCP saved before.""" + + def __init__(self, folder: Path) -> None: + self.folder = folder + self._next = itertools.count(1) + + def next(self, extension: str) -> str: + for n in self._next: + name = f"page-{n}.{extension}" + if not (self.folder / name).exists(): + return name + + +def named(body: bytes, names: FileNames) -> bytes: + """``body``, a JSON-RPC message or batch to Playwright MCP, with a file name from ``names`` given to each call of a + tool in ``SAVED_EXTENSIONS`` that names none; as it came when there's none, or it isn't JSON.""" + try: + message = json.loads(body) + except ValueError: + return body + changed = False + for call in message if isinstance(message, list) else [message]: + params = call.get("params") if isinstance(call, dict) and call.get("method") == "tools/call" else None + if not isinstance(params, dict) or params.get("name") not in SAVED_EXTENSIONS: + continue + arguments = params.get("arguments") or {} + if not isinstance(arguments, dict) or arguments.get("filename"): + continue + extension = SAVED_EXTENSIONS[params["name"]](arguments) + if extension is not None: + params["arguments"], changed = {**arguments, "filename": names.next(extension)}, True + return json.dumps(message).encode() if changed else body + + +def _passing_to(behind: httpx.AsyncClient, names: FileNames) -> Callable: async def passed(request: Any) -> Any: """The request passed to Playwright MCP as it came, its body and its answer streamed, or 502 when Playwright - MCP doesn't answer.""" + MCP doesn't answer; a JSON body is read whole, for ``named``.""" has_body = request.method not in ("GET", "HEAD", "OPTIONS") or "content-length" in request.headers - sent = behind.build_request( - request.method, request.url.path, params=request.url.query or None, - headers=[(key, value) for key, value in request.headers.items() if key.lower() not in HOP_HEADERS], - content=request.stream() if has_body else None) + headers = [(key, value) for key, value in request.headers.items() if key.lower() not in HOP_HEADERS] + content: Any = request.stream() if has_body else None + if has_body and request.headers.get("content-type", "").split(";")[0].strip().lower() == "application/json": + content = named(await request.body(), names) + headers = [(key, value) for key, value in headers if key.lower() != "content-length"] + sent = behind.build_request(request.method, request.url.path, params=request.url.query or None, + headers=headers, content=content) try: answer = await behind.send(sent, stream=True) except httpx.TransportError as e: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index d3f2a0777..fe5d60e3e 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -9,9 +9,11 @@ (``agent_env.a2a_agent.clock``), so every program it runs reads the env's time and later clock writes can wait for it. An agent that can't is skipped, as every agent is until its image runs the follower, or fails the step with require_agent_clock. -Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. Place right -before prompt_agent so the virtual timeline starts at agent-start; when setup needs the clock armed, arm it at rate 0 -and start it with set_env_clock_rate right before prompt_agent. +Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. A multi env's website +browser that is Playwright MCP (``playwright-mcp``) stays on real time, so its pages read the real date: the step warns +and records it (``real_time_browsers``); the browser computer (``browser_envs = ["chromium"]``) follows the clock. +Place right before prompt_agent so the virtual timeline starts at agent-start; when setup needs the clock armed, arm it +at rate 0 and start it with set_env_clock_rate right before prompt_agent. """ from __future__ import annotations @@ -26,6 +28,7 @@ from agentenv_protocol import client as protocol_v1 from agent_env.env.env import EnvCapabilityUnsupported +from agent_env.env.envs.playwright_mcp import PLAYWRIGHT_MCP_ENVIRONMENT_NAME from agent_env.env.gateway.constants import EXT_CLOCK_URI from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef @@ -182,6 +185,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if self.timezone is not None: configuration.update(timezone=self.timezone, armed_at=self.armed_virtual_time, get_time_zone=state.get("time_zone")) + real_time_browsers = [s["service"] for s in skipped + if s["service"] == PLAYWRIGHT_MCP_ENVIRONMENT_NAME and not s.get("website")] + if real_time_browsers: + configuration["real_time_browsers"] = real_time_browsers + logger.warning(f"sync_env_clock env={self.env_id}: its website browser {real_time_browsers[0]!r}, " + "Playwright MCP, stays on real time, so the pages it opens read the real date in their " + "JavaScript; name the browser computer as the env's website browser " + "(browser_envs = [\"chromium\"]) for a browser on the env's clock") context.metadata.setdefault("clock_configurations", []).append(configuration) logger.info(f"sync_env_clock env={self.env_id}: armed rate={self.virtual_seconds_per_real_second}; " f"synced={[s['service'] for s in synced]} skipped={[s['service'] for s in skipped]} " diff --git a/tst/integration/env/envs/playwright_mcp_local_test.py b/tst/integration/env/envs/playwright_mcp_local_test.py index 1c691f14c..d2e2e044c 100644 --- a/tst/integration/env/envs/playwright_mcp_local_test.py +++ b/tst/integration/env/envs/playwright_mcp_local_test.py @@ -2,11 +2,13 @@ browser's card is served there, as every MCP server serves its card, and on the recorder's own port too, where deploys read it until they read it at the MCP port, the card first and the MCP port only once Playwright MCP answers behind it, which is what the gateway's health check asks; an MCP client lists and calls the browser's tools through the front; a -recording through the card on the MCP port films the page the client opens; and a killed recorder comes back, the front -with it, while the MCP server behind it stays up. Needs Docker.""" +recording through the card on the MCP port films the page the client opens; a screenshot the client leaves unnamed is +saved and named to it with no real time in the name or its folder; and a killed recorder comes back, the front with it, +while the MCP server behind it stays up. Needs Docker.""" import asyncio import json +import re import subprocess import time import uuid @@ -183,6 +185,21 @@ async def test_a_recording_through_the_card_on_the_mcp_port_films_the_page_an_ag assert sum(segment["frames"] for segment in manifest["segments"]) >= 3, manifest +@pytest.mark.asyncio +async def test_a_screenshot_the_client_leaves_unnamed_is_named_with_a_counter_in_one_folder_and_no_real_time(browser): + """Playwright MCP names one after the real time (``page-{ISO time}.png`` in ``/``) and says so.""" + _, mcp, _ = browser + async with streamable_http_client(f"{mcp}/mcp") as (read, write, _), ClientSession(read, write) as session: + await session.initialize() + await session.call_tool("browser_navigate", {"url": PAGE}) + said = ["".join(getattr(part, "text", "") for part in (await session.call_tool( + "browser_take_screenshot", arguments)).content) for arguments in ({}, {"type": "jpeg"})] + + assert "saved it as /tmp/playwright-mcp-output/page-1.png" in said[0], said[0] + assert "saved it as /tmp/playwright-mcp-output/page-2.jpeg" in said[1], said[1] + assert not [text for text in said if re.search(r"\d{4}-\d\d-\d\dT|\b\d{13}\b", text)], said + + @pytest.mark.asyncio async def test_a_killed_recorder_comes_back_the_front_with_it_and_the_mcp_server_behind_it_stays_up(browser): name, mcp, _ = browser diff --git a/tst/unit/env/envs/test_playwright_mcp_recorder.py b/tst/unit/env/envs/test_playwright_mcp_recorder.py index 748df2c75..40f493a5f 100644 --- a/tst/unit/env/envs/test_playwright_mcp_recorder.py +++ b/tst/unit/env/envs/test_playwright_mcp_recorder.py @@ -2,7 +2,8 @@ through DevTools target events, holds a still page's frame so the recording doesn't split, and serves computer-recording/v1 on a card of its own that the gateway nests as the playwright-mcp child env. It fronts the MCP port: the card and recording there, every other request passed to Playwright MCP behind it, streamed, and 502 when -Playwright MCP doesn't answer.""" +Playwright MCP doesn't answer; a screenshot or PDF a call leaves unnamed, which Playwright MCP would name after the real +time, is named with a counter, in the one folder Playwright MCP is given for its files.""" from __future__ import annotations @@ -11,6 +12,7 @@ import contextlib import io import itertools +import json import re import socket from pathlib import Path @@ -242,6 +244,13 @@ def default(name: str) -> int: assert "--isolated" in entrypoint # the remote debugging port the recorder finds comes with it +def test_the_image_gives_playwright_mcp_one_folder_for_its_files_where_the_front_looks(): + entrypoint = (Path(playwright_mcp.__file__).parent / "entrypoint.sh").read_text() + + assert f'PLAYWRIGHT_MCP_OUTPUT_DIR="${{PLAYWRIGHT_MCP_OUTPUT_DIR:-{recorder.DEFAULT_OUTPUT_DIR}}}"' in entrypoint + assert '--output-dir "${PLAYWRIGHT_MCP_OUTPUT_DIR}"' in entrypoint + + def _behind(app) -> httpx.AsyncClient: return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://playwright-mcp") @@ -412,3 +421,39 @@ async def stream(reader, writer): assert answer.headers["content-type"] == "text/event-stream" and first == "event: message" assert rest[:1] == ["data: 1"] + + +def _call(tool: str, call_id: int = 1, **arguments) -> dict: + return {"jsonrpc": "2.0", "id": call_id, "method": "tools/call", "params": {"name": tool, "arguments": arguments}} + + +@pytest.mark.asyncio +async def test_the_front_names_each_screenshot_or_pdf_a_call_leaves_unnamed_with_a_counter_not_the_time(tmp_path): + """Playwright MCP names such a file ``page-{timestamp}`` from the real time and says so to the model.""" + (tmp_path / "page-4.png").write_bytes(b"saved before the recorder restarted") + passed = [] + + async def mcp(request): + body = await request.body() + passed.append((body, int(request.headers["content-length"]) == len(body))) + return JSONResponse({"jsonrpc": "2.0", "id": 1, "result": {}}) + + sent = [_call("browser_take_screenshot"), _call("browser_take_screenshot", type="jpeg", fullPage=True), + _call("browser_pdf_save"), _call("browser_take_screenshot", filename="mine.png"), + _call("browser_navigate", url="https://shop.test/"), _call("browser_take_screenshot", type="gif"), + [_call("browser_take_screenshot", 7), {"jsonrpc": "2.0", "method": "notifications/initialized"}]] + async with _behind(Starlette(routes=[Route("/mcp", mcp, methods=["POST"])])) as behind: + front = recorder.create_app(_recorder(_Browser()), behind, output_dir=tmp_path) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=front), base_url="http://front") as http: + for body in sent: + await http.post("/mcp", json=body) + await http.post("/mcp", content=b"not json", headers={"content-type": "application/json"}) + + received = [json.loads(body) for body, _ in passed[:-1]] + names = [[call.get("params", {}).get("arguments", {}).get("filename") for call in + (body if isinstance(body, list) else [body])] for body in received] + assert names == [["page-1.png"], ["page-2.jpeg"], ["page-3.pdf"], ["mine.png"], [None], [None], + ["page-5.png", None]] + assert received[1]["params"]["arguments"] == {"type": "jpeg", "fullPage": True, "filename": "page-2.jpeg"} + assert all(length_matches for _, length_matches in passed) + assert passed[-1][0] == b"not json" diff --git a/tst/unit/task_step/sync_env_clock_test.py b/tst/unit/task_step/sync_env_clock_test.py index a615101bc..9eaf5d44c 100644 --- a/tst/unit/task_step/sync_env_clock_test.py +++ b/tst/unit/task_step/sync_env_clock_test.py @@ -108,6 +108,25 @@ async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch) assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email"] +@pytest.mark.asyncio +async def test_a_website_browser_on_real_time_is_named_in_a_warning_and_the_record(monkeypatch, caplog): + """Playwright MCP serves no clock/v1, so the pages it opens read the real date while every server reads the env's.""" + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) + browsing = TaskStepContext(deployed_envs=[_carded(await _composed_card( + {"mcp-email": [_CLOCK_EXT], "mcp-playwright-mcp": []}))]) + alone = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) + + with caplog.at_level("WARNING"): + await _step().execute(browsing) + await _step().execute(alone) + + assert browsing.metadata["clock_configurations"][0]["real_time_browsers"] == ["playwright-mcp"] + assert "real_time_browsers" not in alone.metadata["clock_configurations"][0] + assert caplog.text.count("its website browser 'playwright-mcp', Playwright MCP, stays on real time") == 1 + assert 'browser_envs = ["chromium"]' in caplog.text + + @pytest.mark.asyncio async def test_execute_keeps_the_arm_failure_message(monkeypatch): _mock_http(monkeypatch, set_time_status=409)