From c02b5a4667d7ccf17c591906642166da606d05d7 Mon Sep 17 00:00:00 2001 From: hqhq1025 <78337154+hqhq1025@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:44:46 +0000 Subject: [PATCH 1/4] feat(drivers): add the Hybrid Computer Use driver Pin HCU 1.0.1 release and executable hashes, and forward its AX and GUI tools. Preserve Linux accessibility setup, add manifest regeneration, and test installation, server isolation and readiness offline. --- AGENTS.md | 2 +- scripts/dump_hcu_mcp.py | 40 ++ src/agent_env/drivers/README.md | 29 + src/agent_env/drivers/aliases.toml | 1 + src/agent_env/drivers/envs/hcu/driver.py | 158 ++++ src/agent_env/drivers/envs/hcu/env.toml | 12 + src/agent_env/drivers/envs/hcu/install.sh | 13 + src/agent_env/drivers/envs/hcu/mcp.json | 679 ++++++++++++++++++ .../drivers/envs/hcu/requirements.txt | 1 + tst/unit/bundle/shipped_bundles_test.py | 1 + tst/unit/drivers/hcu_driver_test.py | 391 ++++++++++ 11 files changed, 1326 insertions(+), 1 deletion(-) create mode 100644 scripts/dump_hcu_mcp.py create mode 100644 src/agent_env/drivers/envs/hcu/driver.py create mode 100644 src/agent_env/drivers/envs/hcu/env.toml create mode 100755 src/agent_env/drivers/envs/hcu/install.sh create mode 100644 src/agent_env/drivers/envs/hcu/mcp.json create mode 100644 src/agent_env/drivers/envs/hcu/requirements.txt create mode 100644 tst/unit/drivers/hcu_driver_test.py diff --git a/AGENTS.md b/AGENTS.md index f44d3129a..ac766e4b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -141,7 +141,7 @@ and deserialized through a registry. | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | | `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, settings as `AGENTENV_SETTING_`; it starts as root, its container form's `start_as`, whose supervisord, `root.conf`, runs the clock follower as `agentenv-clock`, the egress proxy as `agentenv-egress` (`egress/`, a Go program its Dockerfile builds and tests, on real time, through which every session program goes out, so a live site loads at any universe date: it passes a site's TLS through when the universe is inside the site's certificate, else re-signs it with the computer's own CA, minted at boot and valid from 1950 to 9999, handshaking with the site as the client did, and refuses any destination that isn't global) and the session's supervisord as the session user, which runs every session program), and `omarchy-4.5`, a disk alone built on the composing machine from Omarchy's own VM image (its `[forms.disk]` `source`) by `disk/omarchy/`'s recipe, its services systemd user units of the graphical session that start through `with-seed`, booted by the launcher as a `linux-desktop` guest (`env/envs/computer/utils/omarchy_disk.py`, on `thin_disk.py`, which Windows' disks use too), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. `artifacts/ubuntu-24.04/clock/` is the computer's clock, in the base's folder since an image builds only from its own: agent-env's build of libfaketime 0.9.12 (`libfaketime.patch` and the two `agentenv_clock_*.c` files it includes, GPL-2.0-only as libfaketime is, built by `build.sh` from upstream's sha256-pinned tarball, in a stage of the base's Dockerfile of its own) and the TLS shim, which `/etc/ld.so.preload` loads into every process and which fake only the session user's: clock reads and waits for a set clock time follow the page the follower (`clock-follower`) writes (`clockpage.py`), monotonic and relative waits staying real, with its default settings (`clock.conf`); `tst/clock` gates it. The follower long-polls the gateway's `/clock/watch` under the config core writes through the root channel when a multi env deploys the computer and after a reset (`env/envs/computer/utils/clock.py`), each time waiting on the computer's `sync_time` until it follows, a warning naming why when it doesn't (arming the clock refuses such a computer), and the driver serves `clock/v1` `sync_time` where the follower is (`agentenv_protocol.computer.clock`), not on a disk guest, whose follower core can't reach. | | `browsers/` | The `browsers` bundle: the website browsers agent-env ships, today `chromium`, a `computer_app` env (Chrome for Testing at the build Playwright pins, for amd64 and arm64, set up for unattended use by managed policies in `/etc/opt/chrome_for_testing/policies/managed`, opened at boot on its `start_urls`, its profile its data). All of it, its pin, install step, policies and command, is in `envs/chromium/chromium.py`; `tst/integration/computer/chromium_app_test.py` runs it. Like `drivers`, it has no tasks. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, the page driver aliased `page`, a browser computer's, whose screen is the browser's active tab and which serves Playwright MCP's `browser_*` tools beside `computer_*` from a Playwright MCP it runs attached to the browser over a CDP address the relay issues it (`browser_evaluate` and `browser_run_code_unsafe` hidden by default; each call an entry in the browser's log, begun and ended with `browser/v1` `calls/begin` and `calls/end`, that its CDP commands name as `for`; `scripts/upgrade_playwright_mcp.py` pins its version), and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, the page driver aliased `page`, a browser computer's, whose screen is the browser's active tab and which serves Playwright MCP's `browser_*` tools beside `computer_*` from a Playwright MCP it runs attached to the browser over a CDP address the relay issues it (`browser_evaluate` and `browser_run_code_unsafe` hidden by default; each call an entry in the browser's log, begun and ended with `browser/v1` `calls/begin` and `calls/end`, that its CDP commands name as `for`; `scripts/upgrade_playwright_mcp.py` pins its version), and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). Hybrid Computer Use (HCU) 1.0.1, aliased `hcu`, supports Linux X11 on `amd64` only. Its AX/GUI tools run beside `computer_*` from `X11Driver`. The bundle has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ### Training helpers diff --git a/scripts/dump_hcu_mcp.py b/scripts/dump_hcu_mcp.py new file mode 100644 index 000000000..cc646d393 --- /dev/null +++ b/scripts/dump_hcu_mcp.py @@ -0,0 +1,40 @@ +"""Write HCU's instructions and tools from a local release executable, without a desktop or network: + + python scripts/dump_hcu_mcp.py /path/to/computer-use src/agent_env/drivers/envs/hcu/mcp.json + +Review the manifest alongside the release's URL and sha256 pins in env.toml and driver.py. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + + +async def dump(binary: Path, output: Path) -> None: + parameters = StdioServerParameters(command=str(binary.resolve()), args=["mcp"]) + async with stdio_client(parameters) as (read, write), ClientSession(read, write) as session: + initialized = await session.initialize() + tools = (await session.list_tools()).tools + kept = ("name", "description", "inputSchema", "annotations", "outputSchema") + manifest = {"instructions": initialized.instructions or "", "tools": [ + {name: value for name, value in tool.model_dump(by_alias=True, exclude_none=True).items() if name in kept} + for tool in tools]} + output.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"{initialized.serverInfo.name} {initialized.serverInfo.version}: {', '.join(tool.name for tool in tools)}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("output", type=Path) + arguments = parser.parse_args() + asyncio.run(dump(arguments.binary, arguments.output)) + + +if __name__ == "__main__": + main() diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index baea4a7f6..c3142c7ff 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -175,6 +175,35 @@ server lists the tools `mcp.json` pins, and the TCC permissions on macOS, the in pin each tarball's URL and sha256 in `env.toml` and the executables' in `driver.py`'s `EXECUTABLES`, and take `mcp.json` again from what that version's server says at connect and lists. Name it as `agent-desktop`, the alias `aliases.toml` gives it. +`envs/hcu/` is [Hybrid Computer Use (HCU)](https://github.com/microsoft/Hybrid-Computer-Use) 1.0.1 as a driver, +for an X11 desktop on Linux (`amd64` only). The install checks the pinned +[release tarball](https://github.com/microsoft/Hybrid-Computer-Use/releases/download/v1.0.1/computer-use-1.0.1-linux-amd64.tar.gz) +by sha256 and places the extracted executable at `bin/computer-use` beside the driver. +Agents get HCU's own accessibility (AX) and graphical user interface (GUI) tools, with their instructions and schemas +pinned in `mcp.json`. The `computer_*` tools, display, input and recording come from `X11Driver`, as the reference +driver's do. Each driver instance keeps one HCU stdio process and serializes calls to that process. +`computer-ready` checks the executable's sha256 against `EXECUTABLES` and the server's instructions and tools against +the manifest, alongside the X11 readiness checks. Name it as `hcu`, the alias `aliases.toml` gives it. + +The HCU process inherits only the variables in `driver.py`'s `KEPT` and receives a fixed `PATH`. +If `XDG_RUNTIME_DIR` is unset, the driver creates a private directory with mode `0700` for HCU. +If `DBUS_SESSION_BUS_ADDRESS` is unset and `/run/agentenv/bus` exists, HCU uses that session bus. +The install sets the dconf system default `org.gnome.desktop.interface toolkit-accessibility=true` so applications +started before the first HCU call can expose their accessibility trees. + +To move it to another HCU version: + +1. Set `VERSION` in `envs/hcu/driver.py` to the new version. +2. Pin the release archive URL and sha256 in `envs/hcu/env.toml`. +3. Set `EXECUTABLES` in `envs/hcu/driver.py` to the sha256 of the extracted executable, `bin/computer-use`. +4. Regenerate `mcp.json` from the local executable with `scripts/dump_hcu_mcp.py BINARY OUT`. + The helper starts the executable, sends MCP `initialize` and `tools/list`, and writes the manifest. + From the repository root, run: + + ```bash + .venv/bin/python scripts/dump_hcu_mcp.py /path/to/computer-use src/agent_env/drivers/envs/hcu/mcp.json + ``` + `envs/macos-harness/` is browser-use's [macOS harness](https://github.com/browser-use/macos-harness) as a driver, for the same macOS computers. macos-harness 0.2.1, from PyPI into the driver's venv, reads an app as numbered text and acts on a number or a pixel, in the background, through eleven commands its skill teaches an agent to run in a shell, diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index 1a88d2fd2..c8385f72c 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -4,6 +4,7 @@ reference = ["reference"] cua-driver = ["cua-driver"] page = ["page"] agent-desktop = ["agent-desktop"] +hcu = ["hcu"] macos-harness = ["macos-harness"] codex-computer-use = ["codex-computer-use"] aside = ["aside"] diff --git a/src/agent_env/drivers/envs/hcu/driver.py b/src/agent_env/drivers/envs/hcu/driver.py new file mode 100644 index 000000000..825d669e7 --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/driver.py @@ -0,0 +1,158 @@ +"""Hybrid Computer Use (HCU) on an X11 desktop on Linux: the pinned server's instructions and AX and GUI tools, +beside X11Driver's computer_* tools. One stdio server retains its snapshots, refs and layout lease for the driver's +lifetime; calls reach it in order. A server that stops fails its pending calls, and the next call starts a fresh one.""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.types import CallToolResult, TextContent + +from agentenv_protocol import ForwardedTool, server_instructions +from agentenv_protocol.computer import ReadyCheck, X11Driver, computer_driver + +HERE = Path(__file__).resolve().parent +VERSION = "1.0.1" +EXECUTABLE = HERE / "bin" / "computer-use" +EXECUTABLES = {"linux/amd64": "3a4243b3801b4e49c3fb50d0191e31be5bcc590cc700d2c921c3ff864a7f3923"} +MANIFEST = HERE / "mcp.json" +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG", "DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", + "XDG_RUNTIME_DIR", "GTK_IM_MODULE", "QT_IM_MODULE", "XMODIFIERS") +PATH = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" +SESSION_BUS = Path("/run/agentenv/bus") + + +@computer_driver(name="hcu", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) +class HybridComputerUse(X11Driver): + def __init__(self) -> None: + super().__init__() + self._pinned: list[ReadyCheck] | None = None + self._server = _Server() + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool.listed(tool, functools.partial(self._forward, tool["name"])) + for tool in _manifest()["tools"]] + + @server_instructions + def hcu_instructions(self) -> str: + return _manifest()["instructions"] + + async def ready_checks(self) -> list[ReadyCheck]: + """Cache successful pin checks; run X11Driver's checks every time.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return [*self._pinned, *await super().ready_checks()] + + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: + try: + return await self._server.call(tool, arguments) + except Exception as error: + return CallToolResult(isError=True, content=[TextContent(type="text", text=( + f"HCU's server failed before it answered, so {tool} may have run: {_reason(error)}"))]) + + async def _pinned_checks(self) -> list[ReadyCheck]: + pinned = EXECUTABLES.get(f"{self.platform.os}/{self.platform.arch}") + digest = await asyncio.to_thread(_digest) + matches = pinned is not None and digest == pinned + checks = [ReadyCheck("hcu", matches, "the pinned computer-use" if matches else + f"bin/computer-use is {digest}, not the pinned {pinned}")] + if not matches: + return checks + try: + async with _session() as (session, initialized): + listed = (await session.list_tools()).tools + except Exception as error: + return [*checks, ReadyCheck("hcu tools", False, f"its server didn't list them: {_reason(error)}")] + running = {tool.name: tool.inputSchema for tool in listed} + pinned_tools = {tool["name"]: tool["inputSchema"] for tool in _manifest()["tools"]} + changed = ["instructions"] if (initialized.instructions or "") != _manifest()["instructions"] else [] + changed += sorted(name for name in running.keys() | pinned_tools.keys() + if running.get(name) != pinned_tools.get(name)) + return [*checks, ReadyCheck("hcu tools", not changed, f"{len(running)} tools, as pinned" if not changed + else f"these differ from {MANIFEST.name}: {', '.join(changed)}")] + + +class _Server: + """Keep the stdio client's entry and exit in one task, with a queue to serialize callers.""" + + def __init__(self) -> None: + self._calls: asyncio.Queue | None = None + self._task: asyncio.Task | None = None + + async def call(self, tool: str, arguments: dict) -> CallToolResult: + if self._task is None or self._task.done(): + self._calls = asyncio.Queue() + self._task = asyncio.create_task(self._serve(self._calls)) + answer: asyncio.Future = asyncio.get_running_loop().create_future() + await self._calls.put((tool, arguments, answer)) + return await answer + + async def _serve(self, calls: asyncio.Queue) -> None: + pending = None + try: + async with _session() as (session, _): + while True: + pending = await calls.get() + tool, arguments, answer = pending + if answer.cancelled(): + pending = None + continue + result = await session.call_tool(tool, arguments) + if not answer.done(): + answer.set_result(result) + pending = None + except BaseException as error: + failed = [pending] if pending else [] + while not calls.empty(): + failed.append(calls.get_nowait()) + for _, _, answer in failed: + if not answer.done(): + answer.set_exception(error if isinstance(error, Exception) else RuntimeError(_reason(error))) + if not isinstance(error, Exception): + raise + + +@contextlib.asynccontextmanager +async def _session(): + """AgentEnv's container desktops have no logind session, so no /run/user/: HCU's Linux provider refuses to + start without XDG_RUNTIME_DIR, and the session bus is AgentEnv's /run/agentenv/bus. Fill both only when unset. + The MCP client inherits extra environment variables; env -i gives HCU only the kept values.""" + environment = _environment() + with contextlib.ExitStack() as resources: + if "XDG_RUNTIME_DIR" not in environment: + environment["XDG_RUNTIME_DIR"] = resources.enter_context(tempfile.TemporaryDirectory(prefix="hcu-runtime-")) + server = StdioServerParameters(command="/usr/bin/env", args=[ + "-i", *(f"{name}={value}" for name, value in environment.items()), str(EXECUTABLE), "mcp"], env={}) + async with stdio_client(server) as (read, write), ClientSession(read, write) as session: + yield session, await session.initialize() + + +def _reason(error: BaseException) -> str: + while isinstance(error, BaseExceptionGroup) and len(error.exceptions) == 1: + error = error.exceptions[0] + return f"{type(error).__name__}: {error}" + + +@functools.cache +def _manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _environment() -> dict[str, str]: + environment = {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH} + if "DBUS_SESSION_BUS_ADDRESS" not in environment and SESSION_BUS.exists(): + environment["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={SESSION_BUS}" + return environment + + +def _digest() -> str | None: + return hashlib.sha256(EXECUTABLE.read_bytes()).hexdigest() if EXECUTABLE.is_file() else None diff --git a/src/agent_env/drivers/envs/hcu/env.toml b/src/agent_env/drivers/envs/hcu/env.toml new file mode 100644 index 000000000..4763aadfb --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/env.toml @@ -0,0 +1,12 @@ +type = "computer_driver" +module = "driver.py:HybridComputerUse" +port = 18765 + +[install.linux] +python = "requirements.txt" +packages = { apt = ["python3-gi", "gir1.2-atspi-2.0", "gir1.2-gtk-3.0", "gir1.2-gdkpixbuf-2.0", "xdotool", "wmctrl", "x11-utils", "dconf-cli"] } +downloads = [ + { url = "https://github.com/microsoft/Hybrid-Computer-Use/releases/download/v1.0.1/computer-use-1.0.1-linux-amd64.tar.gz", sha256 = "f078cb073b8284e158b91dcc202d47e38acf5f1ecd99245328f9b15f0a48c2fc", to = "computer-use.tar.gz", arch = "amd64" }, +] +run = [{ script = "install.sh" }] +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:HybridComputerUse" }] diff --git a/src/agent_env/drivers/envs/hcu/install.sh b/src/agent_env/drivers/envs/hcu/install.sh new file mode 100755 index 000000000..892ac3aba --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/install.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Turn the desktop's accessibility on from boot, as root, through a dconf system default. Chromium and Electron read the +# accessibility setting only when they start, and a computer's apps start at boot, before HCU's first call would switch +# it on; with org.gnome.desktop.interface toolkit-accessibility true from the start, at-spi-bus-launcher reports +# org.a11y.Status IsEnabled true and every app that starts joins the accessibility bus. +set -eu +mkdir -p bin +tar -xzf computer-use.tar.gz -C bin +rm -f computer-use.tar.gz +mkdir -p /etc/dconf/profile /etc/dconf/db/local.d +printf 'user-db:user\nsystem-db:local\n' > /etc/dconf/profile/user +printf '[org/gnome/desktop/interface]\ntoolkit-accessibility=true\n' > /etc/dconf/db/local.d/00-hcu-accessibility +dconf update diff --git a/src/agent_env/drivers/envs/hcu/mcp.json b/src/agent_env/drivers/envs/hcu/mcp.json new file mode 100644 index 000000000..a3c833a6d --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/mcp.json @@ -0,0 +1,679 @@ +{ + "instructions": "Two tools for the running desktop. AX reads and operates application windows through accessibility: it lists windows, returns controls as rN refs, acts on them, and waits with wait steps. GUI takes full-desktop screenshots and sends pixel input. Use AX for named controls, values, menus, dialogs, windows, and keys. Use GUI for pixels, canvases, dragging, and controls that AX does not expose; after one targeted AX find misses a visible control, use GUI for it. Send all desktop input (window activation, keys, text, mouse) through these tools, not shell input utilities or scripted accessibility calls: the tools check the target window before input and report what changed, and outside input makes their refs and diffs stale. After GUI input, read AX again before reusing refs.", + "tools": [ + { + "name": "AX", + "description": "Accessibility for application windows. Linux: observe with root rN and read_text reads bounded pages of full name, value (AT-SPI Text), or description; follow next_read. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", + "inputSchema": { + "$defs": { + "query": { + "additionalProperties": false, + "description": "At least one filter; ANDed, case-insensitive.", + "properties": { + "name_contains": { + "pattern": "\\S", + "type": "string" + }, + "name_equals": { + "pattern": "\\S", + "type": "string" + }, + "role": { + "pattern": "\\S", + "type": "string" + }, + "state": { + "pattern": "\\S", + "type": "string" + }, + "text_contains": { + "pattern": "\\S", + "type": "string" + }, + "text_equals": { + "pattern": "\\S", + "type": "string" + } + }, + "type": "object" + } + }, + "additionalProperties": false, + "not": { + "required": [ + "app", + "window_ref" + ] + }, + "properties": { + "actions": { + "description": "act: ordered steps; a failed step stops the batch. Wait timeouts total at most 60000 ms; an omitted timeout counts as 5000 ms.", + "items": { + "additionalProperties": false, + "properties": { + "allow_global_input": { + "default": true, + "description": "Linux click/scroll only. false forbids driver focus/raise and pointer/keyboard synthesis. Semantic actions may themselves change app focus. Unsupported uses reject the batch before input.", + "type": "boolean" + }, + "direction": { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + "gone": { + "description": "wait until it is absent.", + "type": "boolean" + }, + "keys": { + "description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", + "items": { + "maxLength": 64, + "pattern": "^\\S(.*\\S)?$", + "type": "string" + }, + "maxItems": 8, + "minItems": 1, + "type": "array" + }, + "pages": { + "exclusiveMinimum": 0, + "maximum": 100, + "type": "number" + }, + "query": { + "$ref": "#/$defs/query" + }, + "ref": { + "description": "rN element; wN for activate/close. Omit for keys/text to the batch target (focused window by default).", + "pattern": "^[rw][1-9][0-9]*$", + "type": "string" + }, + "settle": { + "description": "wait: true waits until the UI changes after the step before it, then stops changing.", + "type": "boolean" + }, + "state": { + "description": "wait: ref state, e.g. checked.", + "pattern": "\\S", + "type": "string" + }, + "text": { + "description": "wait: text in the ref element, or anywhere in the window. select: substring of ref to select.", + "maxLength": 10000, + "minLength": 1, + "type": "string" + }, + "timeout_ms": { + "description": "wait: default 5000.", + "maximum": 60000, + "minimum": 1, + "type": "integer" + }, + "title": { + "description": "activate/close: unique window-title substring, instead of wN.", + "pattern": "\\S", + "type": "string" + }, + "to_ref": { + "description": "select: range end; ref..to_ref spans text or cells.", + "pattern": "^r[1-9][0-9]*$", + "type": "string" + }, + "type": { + "enum": [ + "click", + "type", + "keypress", + "set_value", + "select", + "scroll", + "activate", + "close", + "wait" + ], + "type": "string" + }, + "value": { + "maxLength": 10000, + "type": "string" + }, + "window": { + "description": "wait: a top-level window whose title contains this, on the whole desktop.", + "pattern": "\\S", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "app": { + "description": "App name, app id, or PID; use once, then window_ref.", + "pattern": "\\S", + "type": "string" + }, + "cursor": { + "description": "find: next_cursor from a truncated result; send with mode find only.", + "pattern": "^f[1-9][0-9]*$", + "type": "string" + }, + "limit": { + "description": "find page size; default 20.", + "maximum": 50, + "minimum": 1, + "type": "integer" + }, + "mode": { + "description": "apps lists confirmed top-level windows (wN, title, app, active) and reports incomplete inventories; observe returns a tree or diff with rN; find returns matching rN; act runs actions.", + "enum": [ + "apps", + "observe", + "find", + "act" + ], + "type": "string" + }, + "observe": { + "description": "Returned evidence. act: default diff; explicit tree returns the complete window tree, full adds the full-desktop GUI screenshot. observe: default tree. none only with act.", + "enum": [ + "diff", + "tree", + "full", + "screenshot", + "none" + ], + "type": "string" + }, + "query": { + "$ref": "#/$defs/query" + }, + "read_text": { + "additionalProperties": false, + "description": "Linux observe only: read one live root rN without input or a tree. The ref identifies its window; omit app, scope and observe. Offsets count Unicode code points. Follow next_read to continue; pages are live reads, not a frozen document.", + "properties": { + "field": { + "description": "value reads the node's AT-SPI Text interface; unavailable text returns an error. name and description read those attributes.", + "enum": [ + "name", + "value", + "description" + ], + "type": "string" + }, + "limit": { + "description": "Maximum code points; default 4000.", + "maximum": 4000, + "minimum": 1, + "type": "integer" + }, + "start": { + "description": "Start offset, inclusive; default 0.", + "maximum": 2147483647, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "field" + ], + "type": "object" + }, + "root": { + "description": "observe: subtree at rN or the focused element.", + "pattern": "^(focused|r[1-9][0-9]*)$", + "type": "string" + }, + "scope": { + "description": "observe/find; default window.", + "enum": [ + "window", + "content" + ], + "type": "string" + }, + "window_ref": { + "description": "observe/find: window to read. act: omit when actions have refs, which name the window; otherwise sets the initial target for ref-less steps instead of the focused window.", + "pattern": "^w[1-9][0-9]*$", + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": "object" + }, + "annotations": { + "title": "Accessibility", + "destructiveHint": true, + "openWorldHint": false + } + }, + { + "name": "GUI", + "description": "Full-desktop screenshot and pixel input. {\"actions\":[{\"type\":\"screenshot\"}]} alone observes; {\"type\":\"zoom\",\"region\":[x0,y0,x1,y1]} alone enlarges a frame region to read small text and icons. Any other batch is checked before input and returns one final screenshot. x/y are pixels of the latest full-desktop image (screen width x height) from GUI or AX full/screenshot, which return the same frame. type/keypress go to the current focus, so click the field first. Batch the actions planned from one screenshot; end the batch after an action that may open, close, or move a window, and plan the rest from the returned image. GUI cannot address windows; AX activates and closes them by title. wait is one second with no observation, at most 20 s per batch. Errors return code and hint like AX; gui_frame_stale means nothing was sent: replan from the returned screenshot. Example: {\"actions\":[{\"type\":\"click\",\"x\":640,\"y\":412},{\"type\":\"type\",\"text\":\"hello\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}", + "inputSchema": { + "description": "One screenshot or zoom action observes; otherwise run an input batch.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "type": { + "const": "screenshot", + "description": "Capture the final full desktop; this must be the only action.", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "region": { + "description": "[x0,y0,x1,y1] frame pixels; sides >= 8.", + "items": { + "minimum": 0, + "type": "number" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + "type": { + "const": "zoom", + "description": "Look only: enlarge a frame region; alone.", + "type": "string" + } + }, + "required": [ + "type", + "region" + ], + "type": "object" + } + ] + }, + "maxItems": 1, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "actions" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "actions": { + "description": "Ordered GUI actions.", + "items": { + "description": "One GUI action.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Defaults to left.", + "enum": [ + "left", + "right", + "wheel", + "back", + "forward" + ], + "type": "string" + }, + "keys": { + "description": "Held modifiers.", + "items": { + "enum": [ + "CTRL", + "CONTROL", + "SHIFT", + "ALT", + "SUPER" + ], + "type": "string" + }, + "maxItems": 8, + "type": "array" + }, + "type": { + "const": "click", + "description": "Click.", + "type": "string" + }, + "x": { + "description": "Screenshot X pixel.", + "type": "number" + }, + "y": { + "description": "Screenshot Y pixel.", + "type": "number" + } + }, + "required": [ + "type", + "x", + "y" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "keys": { + "description": "Held modifiers.", + "items": { + "enum": [ + "CTRL", + "CONTROL", + "SHIFT", + "ALT", + "SUPER" + ], + "type": "string" + }, + "maxItems": 8, + "type": "array" + }, + "type": { + "const": "double_click", + "description": "Double click.", + "type": "string" + }, + "x": { + "description": "Screenshot X pixel.", + "type": "number" + }, + "y": { + "description": "Screenshot Y pixel.", + "type": "number" + } + }, + "required": [ + "type", + "x", + "y" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "keys": { + "description": "Held modifiers.", + "items": { + "enum": [ + "CTRL", + "CONTROL", + "SHIFT", + "ALT", + "SUPER" + ], + "type": "string" + }, + "maxItems": 8, + "type": "array" + }, + "type": { + "const": "move", + "description": "Move pointer.", + "type": "string" + }, + "x": { + "description": "Screenshot X pixel.", + "type": "number" + }, + "y": { + "description": "Screenshot Y pixel.", + "type": "number" + } + }, + "required": [ + "type", + "x", + "y" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "keys": { + "description": "Held modifiers.", + "items": { + "enum": [ + "CTRL", + "CONTROL", + "SHIFT", + "ALT", + "SUPER" + ], + "type": "string" + }, + "maxItems": 8, + "type": "array" + }, + "path": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "description": "X pixel in the exact returned full-desktop image", + "type": "number" + }, + "y": { + "description": "Y pixel in the exact returned full-desktop image", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + "maxItems": 256, + "minItems": 2, + "type": "array" + }, + "type": { + "const": "drag", + "description": "Drag path.", + "type": "string" + } + }, + "required": [ + "type", + "path" + ], + "type": "object" + }, + { + "additionalProperties": false, + "anyOf": [ + { + "properties": { + "scroll_x": { + "not": { + "const": 0 + } + } + }, + "required": [ + "scroll_x" + ] + }, + { + "properties": { + "scroll_y": { + "not": { + "const": 0 + } + } + }, + "required": [ + "scroll_y" + ] + } + ], + "properties": { + "keys": { + "description": "Held modifiers.", + "items": { + "enum": [ + "CTRL", + "CONTROL", + "SHIFT", + "ALT", + "SUPER" + ], + "type": "string" + }, + "maxItems": 8, + "type": "array" + }, + "scroll_x": { + "maximum": 2000, + "minimum": -2000, + "type": "number" + }, + "scroll_y": { + "maximum": 2000, + "minimum": -2000, + "type": "number" + }, + "type": { + "const": "scroll", + "description": "Scroll at a point.", + "type": "string" + }, + "x": { + "description": "Screenshot X pixel.", + "type": "number" + }, + "y": { + "description": "Screenshot Y pixel.", + "type": "number" + } + }, + "required": [ + "type", + "x", + "y" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "text": { + "maxLength": 10000, + "type": "string" + }, + "type": { + "const": "type", + "description": "Type literal text.", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "keys": { + "description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", + "items": { + "maxLength": 64, + "pattern": "\\S", + "type": "string" + }, + "maxItems": 8, + "minItems": 1, + "type": "array" + }, + "type": { + "const": "keypress", + "description": "Press one key chord.", + "type": "string" + } + }, + "required": [ + "type", + "keys" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "type": { + "const": "wait", + "description": "Wait exactly one second without an intermediate observation; do not stack waits.", + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + } + ] + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "settle_ms": { + "description": "Visual stillness: default 800ms, max 3000ms. Final screenshot stillness, not semantic success and not extra sleep. After wait, usually omit it. At the cap inspect the screenshot or use an AX wait step.", + "maximum": 3000, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "actions" + ], + "type": "object" + } + ], + "properties": { + "actions": {}, + "settle_ms": {} + }, + "type": "object" + }, + "annotations": { + "title": "Desktop screenshot and input", + "destructiveHint": true, + "openWorldHint": false + } + } + ] +} diff --git a/src/agent_env/drivers/envs/hcu/requirements.txt b/src/agent_env/drivers/envs/hcu/requirements.txt new file mode 100644 index 000000000..3e9c4606e --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/requirements.txt @@ -0,0 +1 @@ +agentenv-framework-protocol[computer,x11]>=0.1.469 diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 666041e8d..b7055f68f 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -62,6 +62,7 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference", "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "page": "@local/agentenv-framework/drivers/page", "agent-desktop": "@local/agentenv-framework/drivers/agent-desktop", + "hcu": "@local/agentenv-framework/drivers/hcu", "macos-harness": "@local/agentenv-framework/drivers/macos-harness", "codex-computer-use": "@local/agentenv-framework/drivers/codex-computer-use", "aside": "@local/agentenv-framework/drivers/aside", diff --git a/tst/unit/drivers/hcu_driver_test.py b/tst/unit/drivers/hcu_driver_test.py new file mode 100644 index 000000000..f5193d164 --- /dev/null +++ b/tst/unit/drivers/hcu_driver_test.py @@ -0,0 +1,391 @@ +"""HCU's pinned MCP contract, isolated server environment, retained session, failures and readiness, without a desktop. +The installer runs against a local tarball with its dconf paths redirected to the test directory.""" +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import importlib.util +import json +import os +import stat +import subprocess +import sys +import tarfile +import textwrap +import tomllib +import types +from pathlib import Path + +import pytest +import pytest_asyncio +from mcp.types import CallToolResult, TextContent + +import agent_env +from agentenv_protocol.computer import Platform, ReadyCheck + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "hcu" / "driver.py" +ROOT = Path(__file__).resolve().parents[3] +PLATFORM = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") +FAKE = textwrap.dedent('''\ + import json + import os + import sys + from pathlib import Path + + import anyio + from mcp import types + from mcp.server.lowlevel import Server + from mcp.server.stdio import stdio_server + + state = Path.home() / "fake" + with (state / "starts.jsonl").open("a") as output: + output.write(json.dumps({"argv": sys.argv[1:], "env": dict(os.environ), "pid": os.getpid()}) + "\\n") + if (state / "die").exists(): + sys.exit(1) + manifest = json.loads((state / "mcp.json").read_text()) + server = Server("hcu", instructions=manifest["instructions"]) + calls = 0 + + @server.list_tools() + async def tools(): + return [types.Tool(**tool) for tool in manifest["tools"]] + + @server.call_tool(validate_input=False) + async def call(name, arguments): + global calls + calls += 1 + if arguments.get("mode") == "exit": + os._exit(1) + if arguments.get("mode") == "error": + return types.CallToolResult(isError=True, content=[types.TextContent(type="text", text="ref expired")]) + answer = {"tool": name, "arguments": arguments, "pid": os.getpid(), "calls": calls} + content = [types.TextContent(type="text", text=json.dumps(answer))] + if name == "GUI": + content.append(types.ImageContent(type="image", mimeType="image/png", data="cG5n")) + return types.CallToolResult(content=content) + + async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + anyio.run(main) +''') + + +def _fake_x11(monkeypatch) -> None: + def module(name: str, **attributes) -> types.ModuleType: + loaded = types.ModuleType(name) + loaded.__dict__.update(attributes) + monkeypatch.setitem(sys.modules, name, loaded) + return loaded + + constants = module("Xlib.X", Button1Mask=1 << 8, Button2Mask=1 << 9, Button3Mask=1 << 10) + xtest = module("Xlib.ext.xtest") + module("Xlib", X=constants, XK=module("Xlib.XK"), display=module("Xlib.display")) + module("Xlib.error", ConnectionClosedError=OSError, DisplayConnectionError=OSError) + module("Xlib.ext", xtest=xtest) + module("mss") + monkeypatch.delitem(sys.modules, "agentenv_protocol.computer._x11", raising=False) + + +async def _xtest(self) -> list[ReadyCheck]: + return [ReadyCheck("xtest", True, "XTEST available")] + + +@pytest.fixture +def module(monkeypatch, tmp_path): + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("LANG", "C.UTF-8") + monkeypatch.setenv("TMPDIR", str(tmp_path)) + monkeypatch.setenv("DISPLAY", ":42") + monkeypatch.setenv("XAUTHORITY", str(tmp_path / "Xauthority")) + monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "must-not-reach-hcu") + monkeypatch.setenv("PYTHONPATH", "must-not-reach-hcu") + monkeypatch.setenv("SHELL", "/private/shell") + monkeypatch.setenv("TERM", "must-not-reach-hcu") + for name in ("XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: PLATFORM) + _fake_x11(monkeypatch) + spec = importlib.util.spec_from_file_location("hcu_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + monkeypatch.setattr(loaded.X11Driver, "ready_checks", _xtest) + monkeypatch.setattr(loaded.tempfile, "tempdir", str(tmp_path)) + state = tmp_path / "fake" + state.mkdir() + (state / "mcp.json").write_text(loaded.MANIFEST.read_text()) + executable = tmp_path / "computer-use" + executable.write_text(f"#!{sys.executable}\n{FAKE}") + executable.chmod(0o755) + monkeypatch.setattr(loaded, "EXECUTABLE", executable) + bus = tmp_path / "bus" + bus.touch() + monkeypatch.setattr(loaded, "SESSION_BUS", bus) + return loaded + + +async def _stop(driver) -> None: + task = driver._server._task + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +@pytest_asyncio.fixture +async def driver(module): + instance = module.HybridComputerUse() + yield instance + await _stop(instance) + + +def _starts(home: Path) -> list[dict]: + return [json.loads(line) for line in (home / "fake" / "starts.jsonl").read_text().splitlines()] + + +def _pin_fake(module, monkeypatch) -> None: + monkeypatch.setitem(module.EXECUTABLES, "linux/amd64", hashlib.sha256(module.EXECUTABLE.read_bytes()).hexdigest()) + + +def test_install_downloads_the_version_and_platform_the_driver_pins(module): + configuration = tomllib.loads((DRIVER.parent / "env.toml").read_text()) + assert configuration["module"] == "driver.py:HybridComputerUse" + assert module.HybridComputerUse.supports == [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}] + assert set(configuration["install"]) == {"linux"} + downloads = configuration["install"]["linux"]["downloads"] + assert {f"linux/{download['arch']}" for download in downloads} == set(module.EXECUTABLES) + assert len(downloads) == 1 + download = downloads[0] + assert download["url"] == ("https://github.com/microsoft/Hybrid-Computer-Use/releases/download/" + f"v{module.VERSION}/computer-use-{module.VERSION}-linux-{download['arch']}.tar.gz") + assert download["to"] == "computer-use.tar.gz" + assert download["sha256"] == "f078cb073b8284e158b91dcc202d47e38acf5f1ecd99245328f9b15f0a48c2fc" + assert module.EXECUTABLES["linux/amd64"] == "3a4243b3801b4e49c3fb50d0191e31be5bcc590cc700d2c921c3ff864a7f3923" + assert not (DRIVER.parent / "bin").exists() + + +def test_instructions_and_every_tool_field_are_the_pinned_servers(module): + instance = module.HybridComputerUse() + manifest = json.loads(module.MANIFEST.read_text()) + assert instance.hcu_instructions() == manifest["instructions"] + assert manifest["instructions"] + forwarded = instance.forwarded_tools() + assert [tool.tool.name for tool in forwarded] == ["AX", "GUI"] + for tool, pinned in zip(forwarded, manifest["tools"], strict=True): + assert tool.tool.name == pinned["name"] + assert tool.tool.description == pinned["description"] + assert tool.tool.inputSchema == pinned["inputSchema"] + assert tool.annotations == pinned.get("annotations") + assert tool.output_schema == pinned.get("outputSchema") + + +@pytest.mark.asyncio +async def test_calls_keep_refs_in_one_server_with_only_the_kept_environment(module, driver, tmp_path): + ax, gui = driver.forwarded_tools() + first = await ax.call(mode="snapshot", app="Text Editor") + second = await ax.call(mode="act", actions=[{"type": "click", "ref": "r1"}]) + screenshot = await gui.call(action="screenshot") + + first_data, second_data = (json.loads(answer.content[0].text) for answer in (first, second)) + assert not first.isError and not second.isError and not screenshot.isError + assert first_data["arguments"] == {"mode": "snapshot", "app": "Text Editor"} + assert second_data["arguments"] == {"mode": "act", "actions": [{"type": "click", "ref": "r1"}]} + assert first_data["pid"] == second_data["pid"] and second_data["calls"] == 2 + assert screenshot.content[1].type == "image" and screenshot.content[1].data == "cG5n" + (started,) = _starts(tmp_path) + assert started["argv"] == ["mcp"] + runtime = Path(started["env"]["XDG_RUNTIME_DIR"]) + assert runtime.is_dir() and stat.S_IMODE(runtime.stat().st_mode) == 0o700 + expected = {name: os.environ[name] for name in module.KEPT if name in os.environ} + assert started["env"] == {**expected, "PATH": module.PATH, "XDG_RUNTIME_DIR": str(runtime), + "DBUS_SESSION_BUS_ADDRESS": f"unix:path={module.SESSION_BUS}"} + await _stop(driver) + assert not runtime.exists() + + +@pytest.mark.asyncio +async def test_another_driver_has_its_own_server(module, driver): + other = module.HybridComputerUse() + try: + first = await driver._forward("AX", mode="apps") + second = await other._forward("AX", mode="apps") + assert json.loads(first.content[0].text)["pid"] != json.loads(second.content[0].text)["pid"] + assert json.loads(second.content[0].text)["calls"] == 1 + finally: + await _stop(other) + + +@pytest.mark.asyncio +async def test_an_existing_runtime_and_bus_are_preserved(module, driver, monkeypatch, tmp_path): + runtime = tmp_path / "existing-runtime" + runtime.mkdir(mode=0o700) + monkeypatch.setenv("XDG_RUNTIME_DIR", str(runtime)) + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:abstract=existing-session") + await driver._forward("AX", mode="apps") + (started,) = _starts(tmp_path) + assert started["env"]["XDG_RUNTIME_DIR"] == str(runtime) + assert started["env"]["DBUS_SESSION_BUS_ADDRESS"] == "unix:abstract=existing-session" + await _stop(driver) + assert runtime.is_dir() + + +def test_an_absent_session_bus_is_not_advertised(module): + module.SESSION_BUS.unlink() + assert "DBUS_SESSION_BUS_ADDRESS" not in module._environment() + + +@pytest.mark.asyncio +async def test_server_errors_are_forwarded_without_restarting(driver, tmp_path): + result = await driver._forward("AX", mode="error") + assert result.isError and result.content[0].text == "ref expired" + next_result = await driver._forward("AX", mode="apps") + assert not next_result.isError and json.loads(next_result.content[0].text)["calls"] == 2 + assert len(_starts(tmp_path)) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("when", ["startup", "call"]) +async def test_a_failed_server_fails_pending_calls_and_the_next_call_starts_fresh(driver, tmp_path, when): + marker = tmp_path / "fake" / "die" + if when == "startup": + marker.touch() + answers = await asyncio.wait_for(asyncio.gather( + driver._forward("AX", mode="exit"), driver._forward("GUI", action="screenshot")), timeout=10) + assert all(answer.isError for answer in answers) + assert "so AX may have run" in answers[0].content[0].text + assert "so GUI may have run" in answers[1].content[0].text + await driver._server._task + marker.unlink(missing_ok=True) + result = await driver._forward("AX", mode="apps") + assert not result.isError and json.loads(result.content[0].text)["calls"] == 1 + assert len(_starts(tmp_path)) == 2 + + +@pytest.mark.asyncio +async def test_concurrent_calls_are_serial_and_a_cancelled_queued_call_is_not_sent(module, driver, monkeypatch): + entered, release = asyncio.Event(), asyncio.Event() + seen = [] + + class Session: + async def call_tool(self, tool, arguments): + seen.append(arguments["number"]) + if arguments["number"] == 1: + entered.set() + await release.wait() + return CallToolResult(content=[TextContent(type="text", text=tool)]) + + @contextlib.asynccontextmanager + async def session(): + yield Session(), None + + monkeypatch.setattr(module, "_session", session) + first = asyncio.create_task(driver._forward("AX", number=1)) + await entered.wait() + queued = asyncio.Event() + original_put = driver._server._calls.put + + async def put(item): + await original_put(item) + queued.set() + + monkeypatch.setattr(driver._server._calls, "put", put) + cancelled = asyncio.create_task(driver._forward("AX", number=2)) + await queued.wait() + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled + assert seen == [1] + queued.clear() + last = asyncio.create_task(driver._forward("GUI", number=3)) + await queued.wait() + assert seen == [1] + release.set() + assert all(not result.isError for result in await asyncio.gather(first, last)) + assert seen == [1, 3] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["schema", "missing", "extra", "instructions"]) +async def test_readiness_checks_the_executable_then_instructions_and_tools(module, driver, tmp_path, monkeypatch, change): + wrong, xtest = await driver.ready_checks() + assert not wrong.ok and xtest.ok + assert not (tmp_path / "fake" / "starts.jsonl").exists() + _pin_fake(module, monkeypatch) + ready = await driver.ready_checks() + assert [(check.name, check.ok) for check in ready] == [("hcu", True), ("hcu tools", True), ("xtest", True)] + assert all(check.ok for check in await driver.ready_checks()) + assert len(_starts(tmp_path)) == 1 + manifest = json.loads(module.MANIFEST.read_text()) + if change == "schema": + manifest["tools"][0]["inputSchema"] = {"type": "object", "properties": {}} + elif change == "missing": + manifest["tools"].pop(0) + elif change == "extra": + manifest["tools"].append({"name": "extra", "inputSchema": {"type": "object"}}) + else: + manifest["instructions"] += " changed" + (tmp_path / "fake" / "mcp.json").write_text(json.dumps(manifest)) + driver._pinned = None + _, changed, _ = await driver.ready_checks() + expected = {"schema": "AX", "missing": "AX", "extra": "extra", "instructions": "instructions"}[change] + assert not changed.ok and changed.detail == f"these differ from mcp.json: {expected}" + + +@pytest.mark.asyncio +async def test_missing_executable_and_unpinned_platform_fail_readiness(module, driver, monkeypatch, tmp_path): + module.EXECUTABLE.unlink() + assert module._digest() is None + assert not (await driver.ready_checks())[0].ok + monkeypatch.setattr(driver, "platform", Platform(os="linux", arch="arm64", display="x11")) + assert not (await driver.ready_checks())[0].ok + assert not (tmp_path / "fake" / "starts.jsonl").exists() + + +@pytest.mark.asyncio +async def test_readiness_reports_a_server_that_cannot_list_tools(module, driver, monkeypatch, tmp_path): + _pin_fake(module, monkeypatch) + (tmp_path / "fake" / "die").touch() + executable, tools, xtest = await driver.ready_checks() + assert executable.ok and xtest.ok and not tools.ok + assert tools.detail.startswith("its server didn't list them:") + + +def test_install_unpacks_the_archive_and_sets_accessibility_before_apps_start(tmp_path): + source = tmp_path / "computer-use" + source.write_bytes(b"executable from the release") + source.chmod(0o755) + archive = tmp_path / "computer-use.tar.gz" + with tarfile.open(archive, "w:gz") as output: + output.add(source, arcname="computer-use") + script = (DRIVER.parent / "install.sh").read_text() + dconf = tmp_path / "etc" / "dconf" + installer = tmp_path / "install.sh" + installer.write_text(script.replace("/etc/dconf", str(dconf))) + commands = tmp_path / "commands" + commands.mkdir() + command = commands / "dconf" + command.write_text('#!/bin/sh\n[ "$1" = update ]\nprintf updated > dconf-updated\n') + command.chmod(0o755) + result = subprocess.run(["sh", str(installer)], cwd=tmp_path, env={"PATH": f"{commands}:/usr/bin:/bin"}, + capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + installed = tmp_path / "bin" / "computer-use" + assert installed.read_bytes() == source.read_bytes() and os.access(installed, os.X_OK) + assert not archive.exists() + assert (dconf / "profile" / "user").read_text() == "user-db:user\nsystem-db:local\n" + assert (dconf / "db" / "local.d" / "00-hcu-accessibility").read_text() == ( + "[org/gnome/desktop/interface]\ntoolkit-accessibility=true\n") + assert (tmp_path / "dconf-updated").read_text() == "updated" + + +def test_manifest_helper_reads_the_servers_initialize_and_list_tools(module, tmp_path): + output = tmp_path / "regenerated.json" + environment = {name: value for name, value in os.environ.items() if name != "PYTHONPATH"} + result = subprocess.run([sys.executable, str(ROOT / "scripts" / "dump_hcu_mcp.py"), + str(module.EXECUTABLE), str(output)], env=environment, + capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + assert json.loads(output.read_text()) == json.loads(module.MANIFEST.read_text()) From 17071315f8fc866177b3e176e71129fa007db725 Mon Sep 17 00:00:00 2001 From: Haoqing Wang <78337154+hqhq1025@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:02:13 +0000 Subject: [PATCH 2/4] feat(drivers): add HCU Windows adapter and platform contracts --- scripts/dump_hcu_mcp.py | 14 +- src/agent_env/drivers/README.md | 37 ++- src/agent_env/drivers/envs/hcu/driver.py | 69 ++-- src/agent_env/drivers/envs/hcu/env.toml | 5 + .../drivers/envs/hcu/install-windows.ps1 | 17 + .../drivers/envs/hcu/mcp-windows.json | 1 + .../drivers/envs/hcu/requirements-windows.txt | 1 + tst/unit/bundle/computer_driver_env_test.py | 28 ++ tst/unit/drivers/hcu_driver_test.py | 310 ++++++++++++++++-- 9 files changed, 422 insertions(+), 60 deletions(-) create mode 100644 src/agent_env/drivers/envs/hcu/install-windows.ps1 create mode 100644 src/agent_env/drivers/envs/hcu/mcp-windows.json create mode 100644 src/agent_env/drivers/envs/hcu/requirements-windows.txt diff --git a/scripts/dump_hcu_mcp.py b/scripts/dump_hcu_mcp.py index cc646d393..e296599ef 100644 --- a/scripts/dump_hcu_mcp.py +++ b/scripts/dump_hcu_mcp.py @@ -2,6 +2,9 @@ python scripts/dump_hcu_mcp.py /path/to/computer-use src/agent_env/drivers/envs/hcu/mcp.json +On Windows, run computer-use.exe with the Windows provider and input-acknowledgement DLL beside it, and write +mcp-windows.json. Linux uses mcp.json. Pass --version to refuse an executable from a different release. + Review the manifest alongside the release's URL and sha256 pins in env.toml and driver.py. """ from __future__ import annotations @@ -15,16 +18,18 @@ from mcp.client.stdio import stdio_client -async def dump(binary: Path, output: Path) -> None: +async def dump(binary: Path, output: Path, version: str | None = None) -> None: parameters = StdioServerParameters(command=str(binary.resolve()), args=["mcp"]) async with stdio_client(parameters) as (read, write), ClientSession(read, write) as session: initialized = await session.initialize() + if version is not None and initialized.serverInfo.version != version: + raise ValueError(f"Expected HCU {version}, got {initialized.serverInfo.version}") tools = (await session.list_tools()).tools - kept = ("name", "description", "inputSchema", "annotations", "outputSchema") + kept = ("name", "title", "description", "inputSchema", "annotations", "outputSchema") manifest = {"instructions": initialized.instructions or "", "tools": [ {name: value for name, value in tool.model_dump(by_alias=True, exclude_none=True).items() if name in kept} for tool in tools]} - output.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + output.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n") print(f"{initialized.serverInfo.name} {initialized.serverInfo.version}: {', '.join(tool.name for tool in tools)}") @@ -32,8 +37,9 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("binary", type=Path) parser.add_argument("output", type=Path) + parser.add_argument("--version", help="Require this server version before writing the manifest") arguments = parser.parse_args() - asyncio.run(dump(arguments.binary, arguments.output)) + asyncio.run(dump(arguments.binary, arguments.output, arguments.version)) if __name__ == "__main__": diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index c3142c7ff..6c671a1f6 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -175,35 +175,52 @@ server lists the tools `mcp.json` pins, and the TCC permissions on macOS, the in pin each tarball's URL and sha256 in `env.toml` and the executables' in `driver.py`'s `EXECUTABLES`, and take `mcp.json` again from what that version's server says at connect and lists. Name it as `agent-desktop`, the alias `aliases.toml` gives it. -`envs/hcu/` is [Hybrid Computer Use (HCU)](https://github.com/microsoft/Hybrid-Computer-Use) 1.0.1 as a driver, -for an X11 desktop on Linux (`amd64` only). The install checks the pinned +`envs/hcu/` is [Hybrid Computer Use (HCU)](https://github.com/microsoft/Hybrid-Computer-Use) as a driver +for Linux X11 and Windows desktops (`amd64` only). Windows installation awaits a published, pinned archive. +The Linux install checks the pinned 1.0.1 [release tarball](https://github.com/microsoft/Hybrid-Computer-Use/releases/download/v1.0.1/computer-use-1.0.1-linux-amd64.tar.gz) by sha256 and places the extracted executable at `bin/computer-use` beside the driver. Agents get HCU's own accessibility (AX) and graphical user interface (GUI) tools, with their instructions and schemas -pinned in `mcp.json`. The `computer_*` tools, display, input and recording come from `X11Driver`, as the reference -driver's do. Each driver instance keeps one HCU stdio process and serializes calls to that process. +pinned in `mcp.json` on Linux and `mcp-windows.json` on Windows. The `computer_*` tools, display, input and recording +come from `X11Driver` on Linux and `WindowsDriver` on Windows. Each driver instance keeps one HCU stdio process and +serializes calls to that process. `computer-ready` checks the executable's sha256 against `EXECUTABLES` and the server's instructions and tools against -the manifest, alongside the X11 readiness checks. Name it as `hcu`, the alias `aliases.toml` gives it. +the manifest, alongside the desktop driver's readiness checks. Name it as `hcu`, the alias `aliases.toml` gives it. -The HCU process inherits only the variables in `driver.py`'s `KEPT` and receives a fixed `PATH`. +On Linux, the HCU process inherits only the variables in `driver.py`'s `KEPT` and receives a fixed `PATH`. If `XDG_RUNTIME_DIR` is unset, the driver creates a private directory with mode `0700` for HCU. If `DBUS_SESSION_BUS_ADDRESS` is unset and `/run/agentenv/bus` exists, HCU uses that session bus. The install sets the dconf system default `org.gnome.desktop.interface toolkit-accessibility=true` so applications started before the first HCU call can expose their accessibility trees. +On Windows, the adapter uses `computer = "windows-11"` and `driver = "hcu"`. +No public Windows download is configured yet. Development bundles must supply `computer-use.zip` +through their file collection, matching the installer's SHA256 pin; the installer refuses a missing or different archive. +`install-windows.ps1` extracts `computer-use.exe`, `computer-use-windows-provider.exe`, +`computer-use-input-ack.dll` and their licenses into `bin`. The three runtime files must stay together. +The provider includes its .NET runtime; HCU needs no Python provider. +The driver's venv installs the protocol's `computer` and `windows` extras. +`agentenv-session` starts the driver in the signed-in desktop session. HCU receives the session and system variables +in `WINDOWS_KEPT` and a fixed system `PATH`, without deployment credentials or provider overrides. +`computer-ready` checks all three runtime hashes, the Windows MCP contract and access to the session's input desktop. + To move it to another HCU version: 1. Set `VERSION` in `envs/hcu/driver.py` to the new version. -2. Pin the release archive URL and sha256 in `envs/hcu/env.toml`. -3. Set `EXECUTABLES` in `envs/hcu/driver.py` to the sha256 of the extracted executable, `bin/computer-use`. -4. Regenerate `mcp.json` from the local executable with `scripts/dump_hcu_mcp.py BINARY OUT`. +2. Pin each platform's release archive URL and sha256 in `envs/hcu/env.toml`. +3. Update `EXECUTABLES`, `PROVIDER_SHA256` and `INPUT_ACK_SHA256` in `envs/hcu/driver.py`. + Update the Windows archive pin in `install-windows.ps1` as well. +4. Check the manifest on each platform with `scripts/dump_hcu_mcp.py BINARY OUT --version VERSION`. The helper starts the executable, sends MCP `initialize` and `tools/list`, and writes the manifest. From the repository root, run: ```bash - .venv/bin/python scripts/dump_hcu_mcp.py /path/to/computer-use src/agent_env/drivers/envs/hcu/mcp.json + .venv/bin/python scripts/dump_hcu_mcp.py /path/to/computer-use src/agent_env/drivers/envs/hcu/mcp.json --version 1.0.1 ``` + On Windows, use the driver's Python, pass `bin/computer-use.exe` and write `mcp-windows.json`. + The helper writes UTF-8 with LF line endings on both platforms and refuses a different server version. + `envs/macos-harness/` is browser-use's [macOS harness](https://github.com/browser-use/macos-harness) as a driver, for the same macOS computers. macos-harness 0.2.1, from PyPI into the driver's venv, reads an app as numbered text and acts on a number or a pixel, in the background, through eleven commands its skill teaches an agent to run in a shell, diff --git a/src/agent_env/drivers/envs/hcu/driver.py b/src/agent_env/drivers/envs/hcu/driver.py index 825d669e7..f03ecc723 100644 --- a/src/agent_env/drivers/envs/hcu/driver.py +++ b/src/agent_env/drivers/envs/hcu/driver.py @@ -1,5 +1,5 @@ -"""Hybrid Computer Use (HCU) on an X11 desktop on Linux: the pinned server's instructions and AX and GUI tools, -beside X11Driver's computer_* tools. One stdio server retains its snapshots, refs and layout lease for the driver's +"""Hybrid Computer Use (HCU) on Linux and Windows: the pinned server's instructions and AX and GUI tools, +beside the protocol driver's computer_* tools. One stdio server retains its snapshots, refs and layout lease for the driver's lifetime; calls reach it in order. A server that stops fails its pending calls, and the next call starts a fresh one.""" from __future__ import annotations @@ -9,6 +9,7 @@ import hashlib import json import os +import sys import tempfile from pathlib import Path from typing import Any @@ -17,22 +18,36 @@ from mcp.client.stdio import stdio_client from mcp.types import CallToolResult, TextContent -from agentenv_protocol import ForwardedTool, server_instructions -from agentenv_protocol.computer import ReadyCheck, X11Driver, computer_driver +from agentenv_protocol import ForwardedTool, computer, server_instructions +from agentenv_protocol.computer import ReadyCheck, computer_driver HERE = Path(__file__).resolve().parent VERSION = "1.0.1" -EXECUTABLE = HERE / "bin" / "computer-use" -EXECUTABLES = {"linux/amd64": "3a4243b3801b4e49c3fb50d0191e31be5bcc590cc700d2c921c3ff864a7f3923"} -MANIFEST = HERE / "mcp.json" +WINDOWS = sys.platform == "win32" +OSDriver = getattr(computer, "WindowsDriver" if WINDOWS else "X11Driver") +EXECUTABLE = HERE / "bin" / ("computer-use.exe" if WINDOWS else "computer-use") +EXECUTABLES = {"linux/amd64": "3a4243b3801b4e49c3fb50d0191e31be5bcc590cc700d2c921c3ff864a7f3923", + "windows/amd64": "76bfd73ba9a0a783b018a782df5d8638d3a6411a5e45068973ab86786fab8d08"} +PROVIDER = HERE / "bin" / "computer-use-windows-provider.exe" +PROVIDER_SHA256 = "0f41f120ee1e042b3a0b1a311443caecc7c38066e8cb933c083297f1a7a53f35" +INPUT_ACK = HERE / "bin" / "computer-use-input-ack.dll" +INPUT_ACK_SHA256 = "6d4b4384fec3ed49447e807c1566ee616731b7f600f2950bf967c8888fff63f4" +MANIFEST = HERE / ("mcp-windows.json" if WINDOWS else "mcp.json") KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG", "DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR", "GTK_IM_MODULE", "QT_IM_MODULE", "XMODIFIERS") PATH = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" +WINDOWS_KEPT = ("USERNAME", "USERDOMAIN", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", "APPDATA", "LOCALAPPDATA", + "TEMP", "TMP", "SYSTEMROOT", "SYSTEMDRIVE", "WINDIR", "COMSPEC", "PATHEXT", "PROGRAMDATA", + "PROGRAMFILES", "PROGRAMFILES(X86)", "PROGRAMW6432", "COMMONPROGRAMFILES", "PUBLIC", + "ALLUSERSPROFILE", "COMPUTERNAME", "PROCESSOR_ARCHITECTURE") +WINDOWS_PATH = (r"{SYSTEMROOT}\System32;{SYSTEMROOT};{SYSTEMROOT}\System32\Wbem;" + r"{SYSTEMROOT}\System32\WindowsPowerShell\v1.0") SESSION_BUS = Path("/run/agentenv/bus") -@computer_driver(name="hcu", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) -class HybridComputerUse(X11Driver): +@computer_driver(name="hcu", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}, + {"os": {"type": "windows", "display": ["dwm"]}, "arch": ["amd64"]}]) +class HybridComputerUse(OSDriver): def __init__(self) -> None: super().__init__() self._pinned: list[ReadyCheck] | None = None @@ -47,7 +62,7 @@ def hcu_instructions(self) -> str: return _manifest()["instructions"] async def ready_checks(self) -> list[ReadyCheck]: - """Cache successful pin checks; run X11Driver's checks every time.""" + """Cache successful pin checks; run the desktop driver's checks every time.""" if self._pinned is None or not all(check.ok for check in self._pinned): self._pinned = await self._pinned_checks() return [*self._pinned, *await super().ready_checks()] @@ -64,8 +79,17 @@ async def _pinned_checks(self) -> list[ReadyCheck]: digest = await asyncio.to_thread(_digest) matches = pinned is not None and digest == pinned checks = [ReadyCheck("hcu", matches, "the pinned computer-use" if matches else - f"bin/computer-use is {digest}, not the pinned {pinned}")] - if not matches: + f"bin/{EXECUTABLE.name} is {digest}, not the pinned {pinned}")] + if WINDOWS: + provider_digest = await asyncio.to_thread(_digest, PROVIDER) + checks.append(ReadyCheck("hcu provider", provider_digest == PROVIDER_SHA256, + "the pinned Windows provider" if provider_digest == PROVIDER_SHA256 else + f"bin/{PROVIDER.name} is {provider_digest}, not the pinned {PROVIDER_SHA256}")) + ack_digest = await asyncio.to_thread(_digest, INPUT_ACK) + checks.append(ReadyCheck("hcu input acknowledgement", ack_digest == INPUT_ACK_SHA256, + "the pinned input acknowledgement helper" if ack_digest == INPUT_ACK_SHA256 else + f"bin/{INPUT_ACK.name} is {ack_digest}, not the pinned {INPUT_ACK_SHA256}")) + if not all(check.ok for check in checks): return checks try: async with _session() as (session, initialized): @@ -128,10 +152,13 @@ async def _session(): The MCP client inherits extra environment variables; env -i gives HCU only the kept values.""" environment = _environment() with contextlib.ExitStack() as resources: - if "XDG_RUNTIME_DIR" not in environment: - environment["XDG_RUNTIME_DIR"] = resources.enter_context(tempfile.TemporaryDirectory(prefix="hcu-runtime-")) - server = StdioServerParameters(command="/usr/bin/env", args=[ - "-i", *(f"{name}={value}" for name, value in environment.items()), str(EXECUTABLE), "mcp"], env={}) + if WINDOWS: + server = StdioServerParameters(command=str(EXECUTABLE), args=["mcp"], env=environment) + else: + if "XDG_RUNTIME_DIR" not in environment: + environment["XDG_RUNTIME_DIR"] = resources.enter_context(tempfile.TemporaryDirectory(prefix="hcu-runtime-")) + server = StdioServerParameters(command="/usr/bin/env", args=[ + "-i", *(f"{name}={value}" for name, value in environment.items()), str(EXECUTABLE), "mcp"], env={}) async with stdio_client(server) as (read, write), ClientSession(read, write) as session: yield session, await session.initialize() @@ -144,15 +171,19 @@ def _reason(error: BaseException) -> str: @functools.cache def _manifest() -> dict: - return json.loads(MANIFEST.read_text()) + return json.loads(MANIFEST.read_text(encoding="utf-8")) def _environment() -> dict[str, str]: + if WINDOWS: + environment = {name: os.environ[name] for name in WINDOWS_KEPT if name in os.environ} + return {**environment, "PATH": WINDOWS_PATH.format_map(environment)} environment = {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH} if "DBUS_SESSION_BUS_ADDRESS" not in environment and SESSION_BUS.exists(): environment["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={SESSION_BUS}" return environment -def _digest() -> str | None: - return hashlib.sha256(EXECUTABLE.read_bytes()).hexdigest() if EXECUTABLE.is_file() else None +def _digest(path: Path | None = None) -> str | None: + path = EXECUTABLE if path is None else path + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None diff --git a/src/agent_env/drivers/envs/hcu/env.toml b/src/agent_env/drivers/envs/hcu/env.toml index 4763aadfb..3ed1b17dd 100644 --- a/src/agent_env/drivers/envs/hcu/env.toml +++ b/src/agent_env/drivers/envs/hcu/env.toml @@ -10,3 +10,8 @@ downloads = [ ] run = [{ script = "install.sh" }] services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:HybridComputerUse" }] + +[install.windows] +python = "requirements-windows.txt" +run = [{ script = "install-windows.ps1" }] +services = [{ name = "driver", command = "venv/Scripts/python.exe -m agentenv_protocol.computer.serve driver.py:HybridComputerUse" }] diff --git a/src/agent_env/drivers/envs/hcu/install-windows.ps1 b/src/agent_env/drivers/envs/hcu/install-windows.ps1 new file mode 100644 index 000000000..255e27c44 --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/install-windows.ps1 @@ -0,0 +1,17 @@ +$ErrorActionPreference = 'Stop' +$archive = Join-Path $env:PREFIX 'computer-use.zip' +if (-not (Test-Path -LiteralPath $archive -PathType Leaf)) { + throw 'The pinned HCU Windows archive has not been staged. Windows installation requires a published release archive.' +} +$expected = 'feb7c8699d2552fe2ae14da54485c9bca2ec8aecf77fc8f029b4a937dbbe1d6d' +$actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLower() +if ($actual -ne $expected) { + throw "HCU Windows archive has sha256 $actual, not the pinned $expected" +} +Expand-Archive -Force -LiteralPath $archive -DestinationPath "$env:PREFIX\bin" +foreach ($executable in @('computer-use.exe', 'computer-use-windows-provider.exe', 'computer-use-input-ack.dll')) { + if (-not (Test-Path -LiteralPath "$env:PREFIX\bin\$executable" -PathType Leaf)) { + throw "HCU release is missing $executable" + } +} +Remove-Item -LiteralPath $archive diff --git a/src/agent_env/drivers/envs/hcu/mcp-windows.json b/src/agent_env/drivers/envs/hcu/mcp-windows.json new file mode 100644 index 000000000..3d93b11d1 --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/mcp-windows.json @@ -0,0 +1 @@ +{"instructions": "Two tools for the running desktop. AX reads and operates application windows through accessibility: it lists windows, returns controls as rN refs, acts on them, and waits with wait steps. GUI takes full-desktop screenshots and sends pixel input. Use AX for named controls, values, menus, dialogs, windows, and keys. Use GUI for pixels, canvases, dragging, and controls that AX does not expose; after one targeted AX find misses a visible control, use GUI for it. Send all desktop input (window activation, keys, text, mouse) through these tools, not shell input utilities or scripted accessibility calls: the tools check the target window before input and report what changed, and outside input makes their refs and diffs stale. After GUI input, read AX again before reusing refs.", "tools": [{"name": "AX", "title": "Accessibility", "description": "Accessibility for application windows. Linux and Windows: observe with root rN and read_text reads bounded pages of full name, value, or description; follow next_read. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", "inputSchema": {"$defs": {"query": {"additionalProperties": false, "description": "At least one filter; ANDed, case-insensitive.", "properties": {"name_contains": {"pattern": "\\S", "type": "string"}, "name_equals": {"pattern": "\\S", "type": "string"}, "role": {"pattern": "\\S", "type": "string"}, "state": {"pattern": "\\S", "type": "string"}, "text_contains": {"pattern": "\\S", "type": "string"}, "text_equals": {"pattern": "\\S", "type": "string"}}, "type": "object"}}, "additionalProperties": false, "not": {"required": ["app", "window_ref"]}, "properties": {"actions": {"description": "act: ordered steps; a failed step stops the batch. Wait timeouts total at most 60000 ms; an omitted timeout counts as 5000 ms.", "items": {"additionalProperties": false, "properties": {"allow_global_input": {"default": true, "description": "Linux click/scroll only. false forbids driver focus/raise and pointer/keyboard synthesis. Semantic actions may themselves change app focus. Unsupported uses reject the batch before input.", "type": "boolean"}, "direction": {"enum": ["up", "down", "left", "right"], "type": "string"}, "gone": {"description": "wait until it is absent.", "type": "boolean"}, "keys": {"description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", "items": {"maxLength": 64, "pattern": "^\\S(.*\\S)?$", "type": "string"}, "maxItems": 8, "minItems": 1, "type": "array"}, "pages": {"exclusiveMinimum": 0, "maximum": 100, "type": "number"}, "query": {"$ref": "#/$defs/query"}, "ref": {"description": "rN element; wN for activate/close. Omit for keys/text to the batch target (focused window by default).", "pattern": "^[rw][1-9][0-9]*$", "type": "string"}, "settle": {"description": "wait: true waits until the UI changes after the step before it, then stops changing.", "type": "boolean"}, "state": {"description": "wait: ref state, e.g. checked.", "pattern": "\\S", "type": "string"}, "text": {"description": "wait: text in the ref element, or anywhere in the window. select: substring of ref to select.", "maxLength": 10000, "minLength": 1, "type": "string"}, "timeout_ms": {"description": "wait: default 5000.", "maximum": 60000, "minimum": 1, "type": "integer"}, "title": {"description": "activate/close: unique window-title substring, instead of wN.", "pattern": "\\S", "type": "string"}, "to_ref": {"description": "select: range end; ref..to_ref spans text or cells.", "pattern": "^r[1-9][0-9]*$", "type": "string"}, "type": {"enum": ["click", "type", "keypress", "set_value", "select", "scroll", "activate", "close", "wait"], "type": "string"}, "value": {"maxLength": 10000, "type": "string"}, "window": {"description": "wait: a top-level window whose title contains this, on the whole desktop.", "pattern": "\\S", "type": "string"}}, "required": ["type"], "type": "object"}, "maxItems": 64, "minItems": 1, "type": "array"}, "app": {"description": "App name, app id, or PID; use once, then window_ref.", "pattern": "\\S", "type": "string"}, "cursor": {"description": "find: next_cursor from a truncated result; send with mode find only.", "pattern": "^f[1-9][0-9]*$", "type": "string"}, "limit": {"description": "find page size; default 20.", "maximum": 50, "minimum": 1, "type": "integer"}, "mode": {"description": "apps lists confirmed top-level windows (wN, title, app, active) and reports incomplete inventories; observe returns a tree or diff with rN; find returns matching rN; act runs actions.", "enum": ["apps", "observe", "find", "act"], "type": "string"}, "observe": {"description": "Returned evidence. act: default diff; explicit tree returns the complete window tree, full adds the full-desktop GUI screenshot. observe: default tree. none only with act.", "enum": ["diff", "tree", "full", "screenshot", "none"], "type": "string"}, "query": {"$ref": "#/$defs/query"}, "read_text": {"additionalProperties": false, "description": "Linux and Windows observe only: read one live root rN without input or a tree. The ref identifies its window; omit app, scope and observe. Offsets count Unicode code points. Follow next_read to continue; pages are live reads, not a frozen document.", "properties": {"field": {"description": "value reads AT-SPI Text on Linux or live UIA, IAccessible2, or native control text on Windows; unavailable or protected text returns an error. name and description read those attributes.", "enum": ["name", "value", "description"], "type": "string"}, "limit": {"description": "Maximum code points; default 4000.", "maximum": 4000, "minimum": 1, "type": "integer"}, "start": {"description": "Start offset, inclusive; default 0.", "maximum": 2147483647, "minimum": 0, "type": "integer"}}, "required": ["field"], "type": "object"}, "root": {"description": "observe: subtree at rN or the focused element.", "pattern": "^(focused|r[1-9][0-9]*)$", "type": "string"}, "scope": {"description": "observe/find; default window.", "enum": ["window", "content"], "type": "string"}, "window_ref": {"description": "observe/find: window to read. act: omit when actions have refs, which name the window; otherwise sets the initial target for ref-less steps instead of the focused window.", "pattern": "^w[1-9][0-9]*$", "type": "string"}}, "required": ["mode"], "type": "object"}, "annotations": {"title": "Accessibility", "destructiveHint": true, "openWorldHint": false}}, {"name": "GUI", "title": "Desktop screenshot and input", "description": "Full-desktop screenshot and pixel input. {\"actions\":[{\"type\":\"screenshot\"}]} alone observes; {\"type\":\"zoom\",\"region\":[x0,y0,x1,y1]} alone enlarges a frame region. Each input batch is checked before input and returns one final screenshot. x/y are pixels of the latest full-desktop image from GUI or AX full/screenshot, which return the same frame. type/keypress go to the current focus; click the field first. Batch actions planned from one screenshot; end after an action that may open, close, or move a window, then plan the rest from the returned image. GUI cannot address windows; AX activates and closes them by title. wait is one second with no observation, at most 20 s per batch. Errors return code and hint; gui_frame_stale means no input: replan from the returned screenshot. Example: {\"actions\":[{\"type\":\"type\",\"text\":\"hello\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}", "inputSchema": {"description": "One screenshot or zoom action observes; otherwise run an input batch.", "oneOf": [{"additionalProperties": false, "properties": {"actions": {"items": {"oneOf": [{"additionalProperties": false, "properties": {"type": {"const": "screenshot", "description": "Capture the final full desktop; this must be the only action.", "type": "string"}}, "required": ["type"], "type": "object"}, {"additionalProperties": false, "properties": {"region": {"description": "[x0,y0,x1,y1] frame pixels; sides >= 8.", "items": {"minimum": 0, "type": "number"}, "maxItems": 4, "minItems": 4, "type": "array"}, "type": {"const": "zoom", "description": "Look only: enlarge a frame region; alone.", "type": "string"}}, "required": ["type", "region"], "type": "object"}]}, "maxItems": 1, "minItems": 1, "type": "array"}}, "required": ["actions"], "type": "object"}, {"additionalProperties": false, "properties": {"actions": {"description": "Ordered GUI actions.", "items": {"description": "One GUI action.", "oneOf": [{"additionalProperties": false, "properties": {"button": {"description": "Mouse button. Defaults to left.", "enum": ["left", "right", "wheel", "back", "forward"], "type": "string"}, "keys": {"description": "Held modifiers.", "items": {"enum": ["CTRL", "CONTROL", "SHIFT", "ALT", "SUPER"], "type": "string"}, "maxItems": 8, "type": "array"}, "type": {"const": "click", "description": "Click.", "type": "string"}, "x": {"description": "Screenshot X pixel.", "type": "number"}, "y": {"description": "Screenshot Y pixel.", "type": "number"}}, "required": ["type", "x", "y"], "type": "object"}, {"additionalProperties": false, "properties": {"keys": {"description": "Held modifiers.", "items": {"enum": ["CTRL", "CONTROL", "SHIFT", "ALT", "SUPER"], "type": "string"}, "maxItems": 8, "type": "array"}, "type": {"const": "double_click", "description": "Double click.", "type": "string"}, "x": {"description": "Screenshot X pixel.", "type": "number"}, "y": {"description": "Screenshot Y pixel.", "type": "number"}}, "required": ["type", "x", "y"], "type": "object"}, {"additionalProperties": false, "properties": {"keys": {"description": "Held modifiers.", "items": {"enum": ["CTRL", "CONTROL", "SHIFT", "ALT", "SUPER"], "type": "string"}, "maxItems": 8, "type": "array"}, "type": {"const": "move", "description": "Move pointer.", "type": "string"}, "x": {"description": "Screenshot X pixel.", "type": "number"}, "y": {"description": "Screenshot Y pixel.", "type": "number"}}, "required": ["type", "x", "y"], "type": "object"}, {"additionalProperties": false, "properties": {"keys": {"description": "Held modifiers.", "items": {"enum": ["CTRL", "CONTROL", "SHIFT", "ALT", "SUPER"], "type": "string"}, "maxItems": 8, "type": "array"}, "path": {"items": {"additionalProperties": false, "properties": {"x": {"description": "X pixel in the exact returned full-desktop image", "type": "number"}, "y": {"description": "Y pixel in the exact returned full-desktop image", "type": "number"}}, "required": ["x", "y"], "type": "object"}, "maxItems": 256, "minItems": 2, "type": "array"}, "type": {"const": "drag", "description": "Drag path.", "type": "string"}}, "required": ["type", "path"], "type": "object"}, {"additionalProperties": false, "anyOf": [{"properties": {"scroll_x": {"not": {"const": 0}}}, "required": ["scroll_x"]}, {"properties": {"scroll_y": {"not": {"const": 0}}}, "required": ["scroll_y"]}], "properties": {"keys": {"description": "Held modifiers.", "items": {"enum": ["CTRL", "CONTROL", "SHIFT", "ALT", "SUPER"], "type": "string"}, "maxItems": 8, "type": "array"}, "scroll_x": {"maximum": 2000, "minimum": -2000, "type": "number"}, "scroll_y": {"maximum": 2000, "minimum": -2000, "type": "number"}, "type": {"const": "scroll", "description": "Scroll at a point.", "type": "string"}, "x": {"description": "Screenshot X pixel.", "type": "number"}, "y": {"description": "Screenshot Y pixel.", "type": "number"}}, "required": ["type", "x", "y"], "type": "object"}, {"additionalProperties": false, "properties": {"text": {"maxLength": 10000, "type": "string"}, "type": {"const": "type", "description": "Type literal text.", "type": "string"}}, "required": ["type", "text"], "type": "object"}, {"additionalProperties": false, "properties": {"keys": {"description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", "items": {"maxLength": 64, "pattern": "\\S", "type": "string"}, "maxItems": 8, "minItems": 1, "type": "array"}, "type": {"const": "keypress", "description": "Press one key chord.", "type": "string"}}, "required": ["type", "keys"], "type": "object"}, {"additionalProperties": false, "properties": {"type": {"const": "wait", "description": "Wait exactly one second without an intermediate observation; do not stack waits.", "type": "string"}}, "required": ["type"], "type": "object"}]}, "maxItems": 64, "minItems": 1, "type": "array"}, "settle_ms": {"description": "Visual stillness: default 800ms, max 3000ms. Final screenshot stillness, not semantic success and not extra sleep. After wait, usually omit it. At the cap inspect the screenshot or use an AX wait step.", "maximum": 3000, "minimum": 0, "type": "integer"}}, "required": ["actions"], "type": "object"}], "properties": {"actions": {}, "settle_ms": {}}, "type": "object"}, "annotations": {"title": "Desktop screenshot and input", "destructiveHint": true, "openWorldHint": false}}]} \ No newline at end of file diff --git a/src/agent_env/drivers/envs/hcu/requirements-windows.txt b/src/agent_env/drivers/envs/hcu/requirements-windows.txt new file mode 100644 index 000000000..2ff599ef3 --- /dev/null +++ b/src/agent_env/drivers/envs/hcu/requirements-windows.txt @@ -0,0 +1 @@ +agentenv-framework-protocol[computer,windows]>=0.1.503 diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py index 2b3eac873..85d7ea3ea 100644 --- a/tst/unit/bundle/computer_driver_env_test.py +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -269,3 +269,31 @@ def test_a_run_naming_the_reference_driver_writes_the_shipped_one(tmp_path, monk "urn:agentenv:computer-exec/v1"} <= set(env.declaration.platforms[0].provides) assert (env.port, env.install["linux"].python) == (18765, "requirements.txt") assert env.install["linux"].services[0].command.endswith("agentenv_protocol.computer.serve driver.py:ReferenceDriver") + + +def test_the_hcu_alias_registers_linux_and_windows_contracts_and_installations(tmp_path, monkeypatch, local_stores, + cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + layout(tmp_path / "uses", {"tasks/t.json": json.dumps([ + {"id": "agent", "type": "deploy_agent", "env_ids": ["hcu"]}])}) + plan = plan_bundle(resolve_bundle(parse_bundle(tmp_path / "uses")), installed=installed_bundles) + + written = {done.write.id for done in materialize(plan).writes} + env = Env.get("@local/agentenv-framework/drivers/hcu") + + assert env.id in written and env.declaration.name == "hcu" + platforms = {declared.platform.os: declared for declared in env.declaration.platforms} + assert {name: (declared.platform.arch, declared.platform.display) for name, declared in platforms.items()} == { + "linux": ("amd64", "x11"), "windows": ("amd64", "dwm")} + assert set(env.install) == {"linux", "windows"} + for name, declared in platforms.items(): + assert {"urn:agentenv:computer-pointer/v1", "urn:agentenv:computer-keyboard/v1", + "urn:agentenv:computer-exec/v1"} <= set(declared.provides) + assert {"AX", "GUI"} <= set(declared.tools) + install = env.install[name] + assert install.python == ("requirements-windows.txt" if name == "windows" else "requirements.txt") + assert install.run[0].script == ("install-windows.ps1" if name == "windows" else "install.sh") + assert all(download.arch == "amd64" for download in install.downloads or []) + assert {"mcp.json", "mcp-windows.json", "install-windows.ps1", "requirements-windows.txt"} <= set( + env.files.file_artifact_ids) + assert ComputerDriverEnv.from_dict(env.to_dict()).to_dict() == env.to_dict() diff --git a/tst/unit/drivers/hcu_driver_test.py b/tst/unit/drivers/hcu_driver_test.py index f5193d164..d76f86f37 100644 --- a/tst/unit/drivers/hcu_driver_test.py +++ b/tst/unit/drivers/hcu_driver_test.py @@ -1,4 +1,5 @@ -"""HCU's pinned MCP contract, isolated server environment, retained session, failures and readiness, without a desktop. +"""HCU's Linux and Windows MCP contracts, isolated server environment, retained session, failures and readiness, +without a desktop or native Windows imports. The installer runs against a local tarball with its dconf paths redirected to the test directory.""" from __future__ import annotations @@ -20,13 +21,16 @@ import pytest import pytest_asyncio from mcp.types import CallToolResult, TextContent +from mcp.client import stdio import agent_env -from agentenv_protocol.computer import Platform, ReadyCheck +from agentenv_protocol import computer +from agentenv_protocol.computer import ComputerDriverEnvironment, Platform, ReadyCheck DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "hcu" / "driver.py" ROOT = Path(__file__).resolve().parents[3] PLATFORM = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") +WINDOWS_PLATFORM = Platform(os="windows", version="11", arch="amd64", display="dwm") FAKE = textwrap.dedent('''\ import json import os @@ -38,13 +42,13 @@ from mcp.server.lowlevel import Server from mcp.server.stdio import stdio_server - state = Path.home() / "fake" + state = Path(os.environ.get("USERPROFILE", str(Path.home()))) / "fake" with (state / "starts.jsonl").open("a") as output: output.write(json.dumps({"argv": sys.argv[1:], "env": dict(os.environ), "pid": os.getpid()}) + "\\n") if (state / "die").exists(): sys.exit(1) manifest = json.loads((state / "mcp.json").read_text()) - server = Server("hcu", instructions=manifest["instructions"]) + server = Server("hcu", version="1.0.1", instructions=manifest["instructions"]) calls = 0 @server.list_tools() @@ -93,8 +97,46 @@ async def _xtest(self) -> list[ReadyCheck]: return [ReadyCheck("xtest", True, "XTEST available")] -@pytest.fixture -def module(monkeypatch, tmp_path): +def _fake_windows(monkeypatch) -> None: + class WindowsDriver(ComputerDriverEnvironment): + @computer.display + def screen(self): + raise AssertionError("no Windows desktop in this test") + + @computer.capture + def capture_screen(self): + raise AssertionError("no Windows desktop in this test") + + @computer.click + def click_at(self, *args): + raise AssertionError("no Windows desktop in this test") + + @computer.drag + def drag_along(self, *args): + raise AssertionError("no Windows desktop in this test") + + @computer.scroll + def scroll_at(self, *args): + raise AssertionError("no Windows desktop in this test") + + @computer.type_text + def type_in(self, text): + raise AssertionError("no Windows desktop in this test") + + @computer.press + def press_keys(self, keys): + raise AssertionError("no Windows desktop in this test") + + async def ready_checks(self) -> list[ReadyCheck]: + return [ReadyCheck("input_desktop", True, "the input desktop is this session's")] + + loaded = types.ModuleType("agentenv_protocol.computer._windows") + loaded.WindowsDriver = WindowsDriver + monkeypatch.setitem(sys.modules, loaded.__name__, loaded) + + +@pytest.fixture(params=["linux", "windows"]) +def module(request, monkeypatch, tmp_path): monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("LANG", "C.UTF-8") @@ -105,22 +147,50 @@ def module(monkeypatch, tmp_path): monkeypatch.setenv("PYTHONPATH", "must-not-reach-hcu") monkeypatch.setenv("SHELL", "/private/shell") monkeypatch.setenv("TERM", "must-not-reach-hcu") + for name in ("AGENTENV_APP_SETTINGS", "AGENTENV_MODEL_API_KEY", + "COMPUTER_USE_WINDOWS_PROVIDER", "COMPUTER_USE_WINDOWS_PROVIDER_TRACE", + "COMPUTER_USE_CHANNELS", "HCU_SOCKET", "HCU_SESSION", "DOTNET_STARTUP_HOOKS", "LD_PRELOAD"): + monkeypatch.setenv(name, "must-not-reach-hcu") + monkeypatch.delenv("AGENTENV_TOOL_PREFIX", raising=False) + monkeypatch.delenv("USERPROFILE", raising=False) for name in ("XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS"): monkeypatch.delenv(name, raising=False) - monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: PLATFORM) - _fake_x11(monkeypatch) + platform = WINDOWS_PLATFORM if request.param == "windows" else PLATFORM + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: platform) + if request.param == "windows": + _fake_windows(monkeypatch) + for name, value in {"SYSTEMROOT": r"C:\Windows", "WINDIR": r"C:\Windows", "SYSTEMDRIVE": "C:", + "USERNAME": "user", "USERPROFILE": str(tmp_path), "TEMP": str(tmp_path), + "TMP": str(tmp_path), "APPDATA": str(tmp_path / "AppData" / "Roaming"), + "LOCALAPPDATA": str(tmp_path / "AppData" / "Local"), + "COMSPEC": r"C:\Windows\System32\cmd.exe", "PATHEXT": ".COM;.EXE;.BAT;.CMD"}.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr(stdio, "DEFAULT_INHERITED_ENV_VARS", [ + "APPDATA", "HOMEDRIVE", "HOMEPATH", "LOCALAPPDATA", "PATH", "PATHEXT", "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", "SYSTEMROOT", "TEMP", "USERNAME", "USERPROFILE"]) + else: + _fake_x11(monkeypatch) spec = importlib.util.spec_from_file_location("hcu_driver", DRIVER) loaded = importlib.util.module_from_spec(spec) - spec.loader.exec_module(loaded) - monkeypatch.setattr(loaded.X11Driver, "ready_checks", _xtest) + with monkeypatch.context() as importing: + importing.setattr(sys, "platform", "win32" if request.param == "windows" else "linux") + spec.loader.exec_module(loaded) + if request.param == "linux": + monkeypatch.setattr(loaded.OSDriver, "ready_checks", _xtest) monkeypatch.setattr(loaded.tempfile, "tempdir", str(tmp_path)) state = tmp_path / "fake" state.mkdir() (state / "mcp.json").write_text(loaded.MANIFEST.read_text()) - executable = tmp_path / "computer-use" + executable = tmp_path / loaded.EXECUTABLE.name executable.write_text(f"#!{sys.executable}\n{FAKE}") executable.chmod(0o755) monkeypatch.setattr(loaded, "EXECUTABLE", executable) + provider = tmp_path / "computer-use-windows-provider.exe" + provider.write_bytes(b"Windows provider from the release") + monkeypatch.setattr(loaded, "PROVIDER", provider) + input_ack = tmp_path / "computer-use-input-ack.dll" + input_ack.write_bytes(b"Windows input acknowledgement helper") + monkeypatch.setattr(loaded, "INPUT_ACK", input_ack) bus = tmp_path / "bus" bus.touch() monkeypatch.setattr(loaded, "SESSION_BUS", bus) @@ -147,16 +217,22 @@ def _starts(home: Path) -> list[dict]: def _pin_fake(module, monkeypatch) -> None: - monkeypatch.setitem(module.EXECUTABLES, "linux/amd64", hashlib.sha256(module.EXECUTABLE.read_bytes()).hexdigest()) + platform = "windows/amd64" if module.WINDOWS else "linux/amd64" + monkeypatch.setitem(module.EXECUTABLES, platform, hashlib.sha256(module.EXECUTABLE.read_bytes()).hexdigest()) + if module.WINDOWS: + monkeypatch.setattr(module, "PROVIDER_SHA256", hashlib.sha256(module.PROVIDER.read_bytes()).hexdigest()) + monkeypatch.setattr(module, "INPUT_ACK_SHA256", hashlib.sha256(module.INPUT_ACK.read_bytes()).hexdigest()) def test_install_downloads_the_version_and_platform_the_driver_pins(module): configuration = tomllib.loads((DRIVER.parent / "env.toml").read_text()) assert configuration["module"] == "driver.py:HybridComputerUse" - assert module.HybridComputerUse.supports == [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}] - assert set(configuration["install"]) == {"linux"} + assert module.HybridComputerUse.supports == [ + {"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}, + {"os": {"type": "windows", "display": ["dwm"]}, "arch": ["amd64"]}] + assert set(configuration["install"]) == {"linux", "windows"} downloads = configuration["install"]["linux"]["downloads"] - assert {f"linux/{download['arch']}" for download in downloads} == set(module.EXECUTABLES) + assert set(module.EXECUTABLES) == {"linux/amd64", "windows/amd64"} assert len(downloads) == 1 download = downloads[0] assert download["url"] == ("https://github.com/microsoft/Hybrid-Computer-Use/releases/download/" @@ -164,9 +240,28 @@ def test_install_downloads_the_version_and_platform_the_driver_pins(module): assert download["to"] == "computer-use.tar.gz" assert download["sha256"] == "f078cb073b8284e158b91dcc202d47e38acf5f1ecd99245328f9b15f0a48c2fc" assert module.EXECUTABLES["linux/amd64"] == "3a4243b3801b4e49c3fb50d0191e31be5bcc590cc700d2c921c3ff864a7f3923" + windows = configuration["install"]["windows"] + for digest in (module.EXECUTABLES["windows/amd64"], module.PROVIDER_SHA256, module.INPUT_ACK_SHA256): + assert len(digest) == 64 and int(digest, 16) > 0 + assert windows["python"] == "requirements-windows.txt" + assert (DRIVER.parent / windows["python"]).read_text().strip() == ( + "agentenv-framework-protocol[computer,windows]>=0.1.503") + assert windows["run"] == [{"script": "install-windows.ps1"}] + assert windows["services"] == [{"name": "driver", "command": ( + "venv/Scripts/python.exe -m agentenv_protocol.computer.serve driver.py:HybridComputerUse")}] + assert (DRIVER.parent / windows["run"][0]["script"]).is_file() + assert not configuration.get("known_failures") assert not (DRIVER.parent / "bin").exists() +def test_platform_selects_the_desktop_base_executable_and_manifest(module): + expected = ("WindowsDriver", "computer-use.exe", "mcp-windows.json", WINDOWS_PLATFORM) if module.WINDOWS else ( + "X11Driver", "computer-use", "mcp.json", PLATFORM) + assert (module.OSDriver.__name__, module.EXECUTABLE.name, module.MANIFEST.name, + module.HybridComputerUse().platform) == expected + assert module.HybridComputerUse.__bases__ == (module.OSDriver,) + + def test_instructions_and_every_tool_field_are_the_pinned_servers(module): instance = module.HybridComputerUse() manifest = json.loads(module.MANIFEST.read_text()) @@ -183,6 +278,7 @@ def test_instructions_and_every_tool_field_are_the_pinned_servers(module): @pytest.mark.asyncio +@pytest.mark.parametrize("module", ["linux"], indirect=True) async def test_calls_keep_refs_in_one_server_with_only_the_kept_environment(module, driver, tmp_path): ax, gui = driver.forwarded_tools() first = await ax.call(mode="snapshot", app="Text Editor") @@ -206,6 +302,81 @@ async def test_calls_keep_refs_in_one_server_with_only_the_kept_environment(modu assert not runtime.exists() +@pytest.mark.asyncio +@pytest.mark.parametrize("module", ["windows"], indirect=True) +async def test_windows_calls_use_stdio_directly_and_keep_only_the_windows_environment(module, driver, tmp_path, + monkeypatch): + servers = [] + original = module.stdio_client + + @contextlib.asynccontextmanager + async def client(server): + servers.append(server) + async with original(server) as streams: + yield streams + + def no_runtime_directory(**kwargs): + raise AssertionError("Windows must not create an X11 runtime directory") + + monkeypatch.setattr(module, "stdio_client", client) + monkeypatch.setattr(module.tempfile, "TemporaryDirectory", no_runtime_directory) + ax, gui = driver.forwarded_tools() + snapshot = await ax.call(mode="apps") + action = await ax.call(mode="act", actions=[{"type": "click", "ref": "r1"}]) + screenshot = await gui.call(actions=[{"type": "screenshot"}]) + assert not snapshot.isError and not action.isError and not screenshot.isError + assert json.loads(snapshot.content[0].text)["arguments"] == {"mode": "apps"} + action_data = json.loads(action.content[0].text) + assert action_data["arguments"] == {"mode": "act", "actions": [{"type": "click", "ref": "r1"}]} + assert action_data["pid"] == json.loads(snapshot.content[0].text)["pid"] and action_data["calls"] == 2 + assert screenshot.content[1].type == "image" and screenshot.content[1].data == "cG5n" + assert json.loads(screenshot.content[0].text)["arguments"] == {"actions": [{"type": "screenshot"}]} + (server,) = servers + assert (server.command, server.args) == (str(module.EXECUTABLE), ["mcp"]) + expected = {name: os.environ[name] for name in module.WINDOWS_KEPT if name in os.environ} + expected["PATH"] = (r"C:\Windows\System32;C:\Windows;C:\Windows\System32\Wbem;" + r"C:\Windows\System32\WindowsPowerShell\v1.0") + assert server.env == expected + (started,) = _starts(tmp_path) + assert started["argv"] == ["mcp"] + actual = started["env"] + assert actual.pop("LC_CTYPE", "C.UTF-8") == "C.UTF-8" + assert actual == expected + assert expected["USERPROFILE"] == str(tmp_path) + assert expected["APPDATA"] == str(tmp_path / "AppData" / "Roaming") + assert expected["LOCALAPPDATA"] == str(tmp_path / "AppData" / "Local") + assert expected["COMSPEC"] == r"C:\Windows\System32\cmd.exe" + assert "must-not-reach-hcu" not in actual.values() + assert not {"DISPLAY", "XAUTHORITY", "XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS", "HOME", "SHELL", "TERM"} & actual.keys() + + +def test_deploy_secrets_and_provider_overrides_never_reach_hcu(module, monkeypatch): + for name in ("AGENTENV_COMPUTER_SETTINGS", "AGENTENV_CONTROL_TOKEN", "AGENTENV_APP_SETTINGS", + "AGENTENV_TOOL_PREFIX", "COMPUTER_USE_WINDOWS_PROVIDER", "COMPUTER_USE_WINDOWS_PROVIDER_TRACE", + "COMPUTER_USE_CHANNELS", "HCU_SOCKET", "HCU_SESSION", "PYTHONPATH", "DOTNET_STARTUP_HOOKS"): + monkeypatch.setenv(name, "must-not-reach-hcu") + monkeypatch.setenv("PATH", "/untrusted/bin") + environment = module._environment() + assert "must-not-reach-hcu" not in environment.values() + assert "/untrusted/bin" not in environment["PATH"] + + +def test_the_manifest_is_read_as_utf8(module, monkeypatch, tmp_path): + manifest = json.loads(module.MANIFEST.read_text(encoding="utf-8")) + manifest["instructions"] += "\n读取桌面" + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + original = Path.read_text + + def read_text(self, encoding=None, errors=None): + return original(self, encoding=encoding or "cp1252", errors=errors) + + monkeypatch.setattr(Path, "read_text", read_text) + monkeypatch.setattr(module, "MANIFEST", path) + module._manifest.cache_clear() + assert module.HybridComputerUse().hcu_instructions() == manifest["instructions"] + + @pytest.mark.asyncio async def test_another_driver_has_its_own_server(module, driver): other = module.HybridComputerUse() @@ -219,6 +390,7 @@ async def test_another_driver_has_its_own_server(module, driver): @pytest.mark.asyncio +@pytest.mark.parametrize("module", ["linux"], indirect=True) async def test_an_existing_runtime_and_bus_are_preserved(module, driver, monkeypatch, tmp_path): runtime = tmp_path / "existing-runtime" runtime.mkdir(mode=0o700) @@ -232,6 +404,7 @@ async def test_an_existing_runtime_and_bus_are_preserved(module, driver, monkeyp assert runtime.is_dir() +@pytest.mark.parametrize("module", ["linux"], indirect=True) def test_an_absent_session_bus_is_not_advertised(module): module.SESSION_BUS.unlink() assert "DBUS_SESSION_BUS_ADDRESS" not in module._environment() @@ -310,12 +483,16 @@ async def put(item): @pytest.mark.asyncio @pytest.mark.parametrize("change", ["schema", "missing", "extra", "instructions"]) async def test_readiness_checks_the_executable_then_instructions_and_tools(module, driver, tmp_path, monkeypatch, change): - wrong, xtest = await driver.ready_checks() - assert not wrong.ok and xtest.ok + unpinned = await driver.ready_checks() + assert not unpinned[0].ok and unpinned[-1].ok + if module.WINDOWS: + assert not unpinned[1].ok and unpinned[1].name == "hcu provider" assert not (tmp_path / "fake" / "starts.jsonl").exists() _pin_fake(module, monkeypatch) ready = await driver.ready_checks() - assert [(check.name, check.ok) for check in ready] == [("hcu", True), ("hcu tools", True), ("xtest", True)] + names = ["hcu", *(["hcu provider", "hcu input acknowledgement"] if module.WINDOWS else []), "hcu tools", + "input_desktop" if module.WINDOWS else "xtest"] + assert [(check.name, check.ok) for check in ready] == [(name, True) for name in names] assert all(check.ok for check in await driver.ready_checks()) assert len(_starts(tmp_path)) == 1 manifest = json.loads(module.MANIFEST.read_text()) @@ -329,18 +506,33 @@ async def test_readiness_checks_the_executable_then_instructions_and_tools(modul manifest["instructions"] += " changed" (tmp_path / "fake" / "mcp.json").write_text(json.dumps(manifest)) driver._pinned = None - _, changed, _ = await driver.ready_checks() + changed = (await driver.ready_checks())[-2] expected = {"schema": "AX", "missing": "AX", "extra": "extra", "instructions": "instructions"}[change] - assert not changed.ok and changed.detail == f"these differ from mcp.json: {expected}" + assert not changed.ok and changed.detail == f"these differ from {module.MANIFEST.name}: {expected}" + + +@pytest.mark.asyncio +async def test_readiness_refuses_the_other_platforms_contract(module, driver, tmp_path, monkeypatch): + _pin_fake(module, monkeypatch) + other = DRIVER.parent / ("mcp.json" if module.WINDOWS else "mcp-windows.json") + (tmp_path / "fake" / "mcp.json").write_bytes(other.read_bytes()) + checks = await driver.ready_checks() + failed, = [check for check in checks if not check.ok] + assert failed.name == "hcu tools" + assert "AX" in failed.detail and module.MANIFEST.name in failed.detail @pytest.mark.asyncio async def test_missing_executable_and_unpinned_platform_fail_readiness(module, driver, monkeypatch, tmp_path): + _pin_fake(module, monkeypatch) + platform = driver.platform + monkeypatch.setattr(driver, "platform", Platform(os=platform.os, arch="arm64", display=platform.display)) + unsupported = (await driver.ready_checks())[0] + assert not unsupported.ok and unsupported.detail.endswith("not the pinned None") + monkeypatch.setattr(driver, "platform", platform) module.EXECUTABLE.unlink() assert module._digest() is None assert not (await driver.ready_checks())[0].ok - monkeypatch.setattr(driver, "platform", Platform(os="linux", arch="arm64", display="x11")) - assert not (await driver.ready_checks())[0].ok assert not (tmp_path / "fake" / "starts.jsonl").exists() @@ -348,9 +540,55 @@ async def test_missing_executable_and_unpinned_platform_fail_readiness(module, d async def test_readiness_reports_a_server_that_cannot_list_tools(module, driver, monkeypatch, tmp_path): _pin_fake(module, monkeypatch) (tmp_path / "fake" / "die").touch() - executable, tools, xtest = await driver.ready_checks() - assert executable.ok and xtest.ok and not tools.ok - assert tools.detail.startswith("its server didn't list them:") + checks = await driver.ready_checks() + assert all(check.ok for check in checks[:-2]) and checks[-1].ok and not checks[-2].ok + assert checks[-2].detail.startswith("its server didn't list them:") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("module", ["windows"], indirect=True) +@pytest.mark.parametrize("binary", ["EXECUTABLE", "PROVIDER", "INPUT_ACK"]) +@pytest.mark.parametrize("change", ["missing", "digest"]) +async def test_windows_checks_all_binaries_before_starting_and_retries_after_repair(module, driver, monkeypatch, + tmp_path, binary, change): + _pin_fake(module, monkeypatch) + path = getattr(module, binary) + original = path.read_bytes() + if change == "missing": + path.unlink() + else: + path.write_bytes(b"untrusted executable") + checks = await driver.ready_checks() + expected = {"EXECUTABLE": "hcu", "PROVIDER": "hcu provider", "INPUT_ACK": "hcu input acknowledgement"}[binary] + failed, = [check for check in checks if not check.ok] + assert failed.name == expected + assert path.name in failed.detail and "not the pinned" in failed.detail + assert not (tmp_path / "fake" / "starts.jsonl").exists() + assert (module._digest(path) is None) == (change == "missing") + path.write_bytes(original) + path.chmod(0o755) + assert all(check.ok for check in await driver.ready_checks()) + assert len(_starts(tmp_path)) == 1 + + +@pytest.mark.asyncio +async def test_successful_pins_are_cached_but_desktop_readiness_is_checked_every_time(module, driver, monkeypatch, + tmp_path): + _pin_fake(module, monkeypatch) + checked = [] + name = "input_desktop" if module.WINDOWS else "xtest" + + async def desktop(self): + checked.append(self) + return [ReadyCheck(name, len(checked) == 1, "current desktop state")] + + monkeypatch.setattr(module.OSDriver, "ready_checks", desktop) + assert all(check.ok for check in await driver.ready_checks()) + checks = await driver.ready_checks() + assert all(check.ok for check in checks[:-1]) + assert not checks[-1].ok and checks[-1].name == name + assert checked == [driver, driver] + assert len(_starts(tmp_path)) == 1 def test_install_unpacks_the_archive_and_sets_accessibility_before_apps_start(tmp_path): @@ -381,11 +619,29 @@ def test_install_unpacks_the_archive_and_sets_accessibility_before_apps_start(tm assert (tmp_path / "dconf-updated").read_text() == "updated" -def test_manifest_helper_reads_the_servers_initialize_and_list_tools(module, tmp_path): +@pytest.mark.parametrize("version", [None, "1.0.1"]) +def test_manifest_helper_reads_the_servers_initialize_and_list_tools(module, tmp_path, version): output = tmp_path / "regenerated.json" environment = {name: value for name, value in os.environ.items() if name != "PYTHONPATH"} result = subprocess.run([sys.executable, str(ROOT / "scripts" / "dump_hcu_mcp.py"), - str(module.EXECUTABLE), str(output)], env=environment, + str(module.EXECUTABLE), str(output), *(["--version", version] if version else [])], env=environment, capture_output=True, text=True, timeout=10) assert result.returncode == 0, result.stderr assert json.loads(output.read_text()) == json.loads(module.MANIFEST.read_text()) + assert output.read_bytes().endswith(b"\n") and b"\r\n" not in output.read_bytes() + + +@pytest.mark.parametrize("existing", [False, True]) +def test_manifest_helper_refuses_a_different_version_without_writing(module, tmp_path, existing): + output = tmp_path / "regenerated.json" + original = b"existing manifest\n" + if existing: + output.write_bytes(original) + environment = {name: value for name, value in os.environ.items() if name != "PYTHONPATH"} + result = subprocess.run([sys.executable, str(ROOT / "scripts" / "dump_hcu_mcp.py"), + str(module.EXECUTABLE), str(output), "--version", "0.0.0"], env=environment, + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 and "Expected HCU 0.0.0, got 1.0.1" in result.stderr + assert output.exists() == existing + if existing: + assert output.read_bytes() == original From 8653eee4e9db33f0522e83ab36278a56507f2052 Mon Sep 17 00:00:00 2001 From: Haoqing Wang <78337154+hqhq1025@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:10:52 +0000 Subject: [PATCH 3/4] fix(drivers): bound HCU calls and verify live accessibility --- src/agent_env/drivers/envs/hcu/driver.py | 12 ++- .../computer/hcu/envs/accessibility/app.py | 11 +++ .../computer/hcu/envs/accessibility/env.toml | 4 + .../computer/hcu/envs/accessibility/window.py | 36 +++++++++ .../computer/hcu/envs/desktop/env.toml | 4 + .../computer/hcu/tasks/desktop.json | 3 + tst/integration/computer/hcu_driver_test.py | 81 +++++++++++++++++++ tst/unit/drivers/hcu_driver_test.py | 64 +++++++++++++++ 8 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 tst/integration/computer/hcu/envs/accessibility/app.py create mode 100644 tst/integration/computer/hcu/envs/accessibility/env.toml create mode 100644 tst/integration/computer/hcu/envs/accessibility/window.py create mode 100644 tst/integration/computer/hcu/envs/desktop/env.toml create mode 100644 tst/integration/computer/hcu/tasks/desktop.json create mode 100644 tst/integration/computer/hcu_driver_test.py diff --git a/src/agent_env/drivers/envs/hcu/driver.py b/src/agent_env/drivers/envs/hcu/driver.py index 825d669e7..e1adb04d9 100644 --- a/src/agent_env/drivers/envs/hcu/driver.py +++ b/src/agent_env/drivers/envs/hcu/driver.py @@ -10,6 +10,7 @@ import json import os import tempfile +from datetime import timedelta from pathlib import Path from typing import Any @@ -29,6 +30,8 @@ "XDG_RUNTIME_DIR", "GTK_IM_MODULE", "QT_IM_MODULE", "XMODIFIERS") PATH = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" SESSION_BUS = Path("/run/agentenv/bus") +# AX batches allow a minute of waits; leave time for input and final evidence. +CALL_TIMEOUT_S = 300 @computer_driver(name="hcu", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) @@ -106,7 +109,11 @@ async def _serve(self, calls: asyncio.Queue) -> None: if answer.cancelled(): pending = None continue - result = await session.call_tool(tool, arguments) + try: + async with asyncio.timeout(CALL_TIMEOUT_S): + result = await session.call_tool(tool, arguments) + except TimeoutError as error: + raise TimeoutError(f"HCU did not answer within {CALL_TIMEOUT_S} seconds") from error if not answer.done(): answer.set_result(result) pending = None @@ -132,7 +139,8 @@ async def _session(): environment["XDG_RUNTIME_DIR"] = resources.enter_context(tempfile.TemporaryDirectory(prefix="hcu-runtime-")) server = StdioServerParameters(command="/usr/bin/env", args=[ "-i", *(f"{name}={value}" for name, value in environment.items()), str(EXECUTABLE), "mcp"], env={}) - async with stdio_client(server) as (read, write), ClientSession(read, write) as session: + async with stdio_client(server) as (read, write), ClientSession( + read, write, read_timeout_seconds=timedelta(seconds=CALL_TIMEOUT_S)) as session: yield session, await session.initialize() diff --git a/tst/integration/computer/hcu/envs/accessibility/app.py b/tst/integration/computer/hcu/envs/accessibility/app.py new file mode 100644 index 000000000..a6211f5b7 --- /dev/null +++ b/tst/integration/computer/hcu/envs/accessibility/app.py @@ -0,0 +1,11 @@ +"""A GTK app started at boot, before HCU enables accessibility on its first AX call.""" +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Packages, Ready, computer_app, linux + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}]) +@linux(packages=Packages(apt=["python3-gi", "gir1.2-gtk-3.0"]), + launch=Launch(at="boot", command=["/usr/bin/python3", "window.py"], + window={"class": "HcuAccessibility"}, ready=Ready(window=True)), + paths=[".hcu-accessibility"]) +class AccessibilityProbe(ComputerAppEnvironment): + pass diff --git a/tst/integration/computer/hcu/envs/accessibility/env.toml b/tst/integration/computer/hcu/envs/accessibility/env.toml new file mode 100644 index 000000000..d4b329716 --- /dev/null +++ b/tst/integration/computer/hcu/envs/accessibility/env.toml @@ -0,0 +1,4 @@ +type = "computer_app" +module = "app.py:AccessibilityProbe" +computer = "ubuntu-24.04" +driver = "hcu" diff --git a/tst/integration/computer/hcu/envs/accessibility/window.py b/tst/integration/computer/hcu/envs/accessibility/window.py new file mode 100644 index 000000000..c99a56adc --- /dev/null +++ b/tst/integration/computer/hcu/envs/accessibility/window.py @@ -0,0 +1,36 @@ +"""Record the startup accessibility setting, then persist an AX button activation.""" +import json +import os +from pathlib import Path + +import gi + +gi.require_version("Gtk", "3.0") +from gi.repository import Gio, Gtk + +folder = Path.home() / ".hcu-accessibility" +folder.mkdir(exist_ok=True) +started = {"pid": os.getpid(), + "toolkit_accessibility": Gio.Settings.new("org.gnome.desktop.interface").get_boolean("toolkit-accessibility")} +(folder / "startup.json").write_text(json.dumps(started)) +window = Gtk.Window(title="HCU accessibility fixture") +window.set_wmclass("hcu-accessibility", "HcuAccessibility") +window.set_default_size(420, 180) +window.connect("destroy", Gtk.main_quit) +box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=16) +box.set_border_width(24) +label = Gtk.Label(label="Waiting for HCU") +button = Gtk.Button(label="Confirm HCU action") + + +def confirmed(_button): + (folder / "result.json").write_text(json.dumps({"clicked": True, "pid": os.getpid()})) + label.set_text("HCU action confirmed") + + +button.connect("clicked", confirmed) +box.pack_start(label, True, True, 0) +box.pack_start(button, True, True, 0) +window.add(box) +window.show_all() +Gtk.main() diff --git a/tst/integration/computer/hcu/envs/desktop/env.toml b/tst/integration/computer/hcu/envs/desktop/env.toml new file mode 100644 index 000000000..4ba5d1455 --- /dev/null +++ b/tst/integration/computer/hcu/envs/desktop/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "hcu" +apps = ["accessibility"] diff --git a/tst/integration/computer/hcu/tasks/desktop.json b/tst/integration/computer/hcu/tasks/desktop.json new file mode 100644 index 000000000..67f3b3154 --- /dev/null +++ b/tst/integration/computer/hcu/tasks/desktop.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"} +] diff --git a/tst/integration/computer/hcu_driver_test.py b/tst/integration/computer/hcu_driver_test.py new file mode 100644 index 000000000..e693fa5d8 --- /dev/null +++ b/tst/integration/computer/hcu_driver_test.py @@ -0,0 +1,81 @@ +"""The shipped HCU release reads and activates a GTK control in an app started before its first AX call.""" + +import json +import platform +import re +from pathlib import Path + +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client + +from agent_env.bundle import parse_bundle, prepare_bundle +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext +from agentenv_protocol.computer import COMPUTER_READY +from tst.util.capabilities import missing_capability_reason + +pytestmark = [ + pytest.mark.integration, + pytest.mark.int_test_slow, + pytest.mark.skipif(platform.machine().lower() not in {"x86_64", "amd64"}, + reason=missing_capability_reason("hcu_linux_amd64")), +] +BUNDLE = Path(__file__).parent / "hcu" + + +@pytest.fixture +def hcu_desktop(state): + bundle = parse_bundle(BUNDLE) + with namespace_routing(): + prepare_bundle(BUNDLE, sandbox="local") + yield Env.get(f"{bundle.id_root}/desktop") + + +def _text(result): + assert not result.isError, result + return "\n".join(item.text for item in result.content if item.type == "text") + + +async def _record(deployed, name): + command = ("from pathlib import Path; " + f"print((Path.home() / '.hcu-accessibility' / {name!r}).read_text())") + result = await inside.run(deployed, ["/usr/bin/python3", "-c", command], timeout_s=10) + assert result["exit_code"] == 0, result + return json.loads(result["stdout"]) + + +@pytest.mark.asyncio +async def test_an_app_started_with_accessibility_is_read_and_acted_on_through_the_pinned_hcu(hcu_desktop): + with namespace_routing(): + deployed = await Env.get(hcu_desktop.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + startup = await _record(deployed, "startup.json") + assert startup["toolkit_accessibility"] is True + async with streamable_http_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + apps = _text(await session.call_tool("AX", {"mode": "apps"})) + window = re.search(r'(w\d+) "HCU accessibility fixture"', apps).group(1) + tree = _text(await session.call_tool("AX", {"mode": "observe", "window_ref": window})) + button = next(line for line in tree.splitlines() if '"Confirm HCU action"' in line) + ref = re.search(r"\[ref=(r\d+)\]", button).group(1) + changed = _text(await session.call_tool("AX", { + "mode": "act", "actions": [{"type": "click", "ref": ref}, + {"type": "wait", "text": "HCU action confirmed", "timeout_ms": 5000}], + "observe": "tree", + })) + saved = await _record(deployed, "result.json") + ready = await deployed.invoke(COMPUTER_READY, "status", None) + finally: + report = await teardown_run(context) + + assert saved == {"clicked": True, "pid": startup["pid"]} + assert "HCU action confirmed" in changed + checks = {check["name"]: check for check in ready["checks"]} + assert ready["ready"] and checks["hcu"]["ok"] and checks["hcu tools"]["ok"] + assert not report.failed diff --git a/tst/unit/drivers/hcu_driver_test.py b/tst/unit/drivers/hcu_driver_test.py index f5193d164..3b049eb73 100644 --- a/tst/unit/drivers/hcu_driver_test.py +++ b/tst/unit/drivers/hcu_driver_test.py @@ -55,8 +55,13 @@ async def tools(): async def call(name, arguments): global calls calls += 1 + with (state / "calls.jsonl").open("a") as output: + output.write(json.dumps({"tool": name, "arguments": arguments, "pid": os.getpid()}) + "\\n") if arguments.get("mode") == "exit": os._exit(1) + if arguments.get("mode") == "stall": + await server.request_context.session.send_log_message(level="info", data="stalled") + await anyio.sleep_forever() if arguments.get("mode") == "error": return types.CallToolResult(isError=True, content=[types.TextContent(type="text", text="ref expired")]) answer = {"tool": name, "arguments": arguments, "pid": os.getpid(), "calls": calls} @@ -307,6 +312,65 @@ async def put(item): assert seen == [1, 3] +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_caller", [False, True]) +async def test_a_stalled_call_expires_closes_its_session_and_does_not_replay_queued_input( + module, driver, monkeypatch, tmp_path, cancel_caller): + entered, queued = asyncio.Event(), asyncio.Event() + deadlines = [] + real_timeout = asyncio.timeout + real_client = module.ClientSession + + def deadline(delay): + limit = real_timeout(None) + deadlines.append(limit) + return limit + + async def logging(params): + assert params.data == "stalled" + entered.set() + + def client(*args, **kwargs): + return real_client(*args, logging_callback=logging, **kwargs) + + monkeypatch.setattr(module, "ClientSession", client) + monkeypatch.setattr(module.asyncio, "timeout", deadline) + first = asyncio.create_task(driver._forward("AX", mode="stall")) + await asyncio.wait_for(entered.wait(), timeout=10) + put = driver._server._calls.put + + async def enqueue(item): + await put(item) + queued.set() + + monkeypatch.setattr(driver._server._calls, "put", enqueue) + next_call = asyncio.create_task(driver._forward("GUI", action="screenshot")) + await queued.wait() + if cancel_caller: + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + deadlines[0].reschedule(asyncio.get_running_loop().time() - 1) + answer = await asyncio.wait_for(next_call, timeout=10) + await driver._server._task + + assert answer.isError and "TimeoutError: HCU did not answer within" in answer.content[0].text + (old,) = _starts(tmp_path) + assert not Path(old["env"]["XDG_RUNTIME_DIR"]).exists() + if not cancel_caller: + failed = await first + assert failed.isError and "so AX may have run" in failed.content[0].text + fresh = await driver._forward("AX", mode="apps") + assert not fresh.isError and json.loads(fresh.content[0].text)["calls"] == 1 + old, new = _starts(tmp_path) + assert old["pid"] != new["pid"] + called = [json.loads(line) for line in (tmp_path / "fake" / "calls.jsonl").read_text().splitlines()] + assert called == [ + {"tool": "AX", "arguments": {"mode": "stall"}, "pid": old["pid"]}, + {"tool": "AX", "arguments": {"mode": "apps"}, "pid": new["pid"]}, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize("change", ["schema", "missing", "extra", "instructions"]) async def test_readiness_checks_the_executable_then_instructions_and_tools(module, driver, tmp_path, monkeypatch, change): From 063f3363c3c8fad64e9986f3fb65d0517e0d14d5 Mon Sep 17 00:00:00 2001 From: Haoqing Wang <78337154+hqhq1025@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:27:33 +0000 Subject: [PATCH 4/4] test(drivers): verify HCU timeout cleanup on both platforms --- tst/unit/drivers/hcu_driver_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tst/unit/drivers/hcu_driver_test.py b/tst/unit/drivers/hcu_driver_test.py index 2972bf3c4..58e9d89f7 100644 --- a/tst/unit/drivers/hcu_driver_test.py +++ b/tst/unit/drivers/hcu_driver_test.py @@ -529,7 +529,12 @@ async def enqueue(item): assert answer.isError and "TimeoutError: HCU did not answer within" in answer.content[0].text (old,) = _starts(tmp_path) - assert not Path(old["env"]["XDG_RUNTIME_DIR"]).exists() + if module.WINDOWS: + assert "XDG_RUNTIME_DIR" not in old["env"] + else: + assert not Path(old["env"]["XDG_RUNTIME_DIR"]).exists() + with pytest.raises(ProcessLookupError): + os.kill(old["pid"], 0) if not cancel_caller: failed = await first assert failed.isError and "so AX may have run" in failed.content[0].text