Local control: in-SDK sidecar + desktop/browser drivers

pyproject.toml

@@ -37,10 +37,16 @@ cli = [
   "rich>=13,<16",
   "typer>=0.12,<1",
 ]
+desktop = [
+  "pyautogui>=0.9.54",
+  "pillow>=10",
+]
+browser = [
+  "selenium>=4.20",
+  "markdownify>=0.11",
+]
 all = [
-  "python-dotenv>=1.2.2,<2",
-  "rich>=13,<16",
-  "typer>=0.12,<1",
+  "hai-agents[browser,cli,desktop]",
 ]
 
 [project.scripts]
@@ -55,6 +61,9 @@ Documentation = "https://hub.hcompany.ai/computer-use-agents"
 [tool.hatch.build.targets.wheel]
 packages = ["src/hai_agents", "src/hai_agents_cli", "src/hai_agents_common"]
 
+[tool.hatch.build.targets.wheel.force-include]
+"src/hai_agents/local/selenium_browser/js" = "hai_agents/local/selenium_browser/js"
+
 [dependency-groups]
 dev = [
   "pytest>=9.0.3,<10",

src/hai_agents/client.py

@@ -7,11 +7,16 @@
 
 from __future__ import annotations
 
+import asyncio
+import functools
 import typing
 
 import typing_extensions
 
+from .agents.client import AgentsClient, AsyncAgentsClient
 from .base_client import AsyncBaseClient, BaseClient
+from .local.runtime import auto_sidecars_enabled, ensure_sidecar
+from .local.wiring import collect_sidecar_configs, localize_agent, localize_environments, localize_subagents
 from .polling import (
     AnswerT,
     AsyncSessionHandle,
@@ -24,9 +29,78 @@
 )
 from .polling import async_run_session as _async_run_session
 from .polling import run_session as _run_session
+from .sessions.client import AsyncSessionsClient, SessionsClient
 from .tools import ToolInput, as_tools
 
 
+def _wire_agent_fields(kwargs: typing.Dict[str, typing.Any], get_api_key: typing.Callable[[], str]) -> None:
+    if kwargs.get("environments"):
+        kwargs["environments"] = localize_environments(kwargs["environments"], get_api_key)
+    if kwargs.get("subagents"):
+        kwargs["subagents"] = localize_subagents(kwargs["subagents"], get_api_key)
+
+
+def _ensure_local_sidecars(agent: typing.Any, client_wrapper: typing.Any) -> None:
+    """Auto-start sidecars for the agent's user_device environments before the session is created."""
+    if not auto_sidecars_enabled():
+        return
+    for config in collect_sidecar_configs(agent, client_wrapper._get_api_key(), client_wrapper.get_base_url()):
+        ensure_sidecar(config)
+
+
+class _LocalAgentsClient(AgentsClient):
+    @functools.wraps(AgentsClient.create_agent)
+    def create_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().create_agent(**kwargs)
+
+    @functools.wraps(AgentsClient.update_agent)
+    def update_agent(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().update_agent(*args, **kwargs)
+
+    @functools.wraps(AgentsClient.patch_agent)
+    def patch_agent(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().patch_agent(*args, **kwargs)
+
+
+class _LocalSessionsClient(SessionsClient):
+    @functools.wraps(SessionsClient.create_session)
+    def create_session(self, **kwargs: typing.Any) -> typing.Any:
+        if "agent" in kwargs:
+            kwargs["agent"] = localize_agent(kwargs["agent"], self._raw_client._client_wrapper._get_api_key)
+            _ensure_local_sidecars(kwargs["agent"], self._raw_client._client_wrapper)
+        return super().create_session(**kwargs)
+
+
+class _LocalAsyncAgentsClient(AsyncAgentsClient):
+    @functools.wraps(AsyncAgentsClient.create_agent)
+    async def create_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().create_agent(**kwargs)
+
+    @functools.wraps(AsyncAgentsClient.update_agent)
+    async def update_agent(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().update_agent(*args, **kwargs)
+
+    @functools.wraps(AsyncAgentsClient.patch_agent)
+    async def patch_agent(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any:
+        _wire_agent_fields(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().patch_agent(*args, **kwargs)
+
+
+class _LocalAsyncSessionsClient(AsyncSessionsClient):
+    @functools.wraps(AsyncSessionsClient.create_session)
+    async def create_session(self, **kwargs: typing.Any) -> typing.Any:
+        if "agent" in kwargs:
+            kwargs["agent"] = localize_agent(kwargs["agent"], self._raw_client._client_wrapper._get_api_key)
+            # Chrome launch + sidecar startup can take seconds; keep the event loop free.
+            await asyncio.to_thread(_ensure_local_sidecars, kwargs["agent"], self._raw_client._client_wrapper)
+        return await super().create_session(**kwargs)
+
+
 class Client(BaseClient):
     def run_session(
         self,
@@ -75,6 +149,18 @@ def session(self, id: str) -> SessionHandle:
         """Wrap an existing session id in a handle."""
         return SessionHandle(self, id)
 
+    @property
+    def agents(self) -> _LocalAgentsClient:
+        if self._agents is None:
+            self._agents = _LocalAgentsClient(client_wrapper=self._client_wrapper)
+        return self._agents
+
+    @property
+    def sessions(self) -> _LocalSessionsClient:
+        if self._sessions is None:
+            self._sessions = _LocalSessionsClient(client_wrapper=self._client_wrapper)
+        return self._sessions
+
 
 class AsyncClient(AsyncBaseClient):
     async def run_session(
@@ -123,3 +209,15 @@ async def start_session(
     def session(self, id: str) -> AsyncSessionHandle:
         """Wrap an existing session id in a handle."""
         return AsyncSessionHandle(self, id)
+
+    @property
+    def agents(self) -> _LocalAsyncAgentsClient:
+        if self._agents is None:
+            self._agents = _LocalAsyncAgentsClient(client_wrapper=self._client_wrapper)
+        return self._agents
+
+    @property
+    def sessions(self) -> _LocalAsyncSessionsClient:
+        if self._sessions is None:
+            self._sessions = _LocalAsyncSessionsClient(client_wrapper=self._client_wrapper)
+        return self._sessions

src/hai_agents/local/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from .chrome import ensure_local_chrome
+from .config import SidecarConfig, session_id_from_environment_id
+from .runtime import ensure_sidecar, stop_sidecars
+from .sidecar import SidecarBusyError, SidecarClient
+
+__all__ = [
+    "SidecarBusyError",
+    "SidecarClient",
+    "SidecarConfig",
+    "ensure_local_chrome",
+    "ensure_sidecar",
+    "session_id_from_environment_id",
+    "stop_sidecars",
+]

src/hai_agents/local/chrome.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import logging
+import platform
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_DEBUG_PORT = 9222
+CHROME_STARTUP_TIMEOUT_S = 20.0
+CHROME_PROFILE_DIR = Path.home() / ".hai" / "chrome-profile"
+
+_CHROME_CANDIDATES = {
+    "Darwin": (
+        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+        "/Applications/Chromium.app/Contents/MacOS/Chromium",
+    ),
+    "Windows": (
+        r"C:\Program Files\Google\Chrome\Application\chrome.exe",
+        r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
+    ),
+}
+_CHROME_COMMANDS = ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome")
+
+
+def ensure_local_chrome(port: int = DEFAULT_DEBUG_PORT) -> None:
+    """Make sure a Chrome with an open remote-debugging port is running, launching one if needed."""
+    if _debugger_listening(port):
+        return
+    binary = _find_chrome()
+    if binary is None:
+        raise RuntimeError(
+            "Google Chrome was not found. Install Chrome, or start a browser yourself with "
+            f"--remote-debugging-port={port}."
+        )
+    CHROME_PROFILE_DIR.mkdir(parents=True, exist_ok=True)
+    logger.info("launching Chrome with remote debugging on port %d (profile: %s)", port, CHROME_PROFILE_DIR)
+    subprocess.Popen(
+        [
+            binary,
+            f"--remote-debugging-port={port}",
+            f"--user-data-dir={CHROME_PROFILE_DIR}",
+            "--no-first-run",
+            "--no-default-browser-check",
+        ],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+    )
+    deadline = time.monotonic() + CHROME_STARTUP_TIMEOUT_S
+    while time.monotonic() < deadline:
+        if _debugger_listening(port):
+            return
+        time.sleep(0.25)
+    raise RuntimeError(f"Chrome did not open debugging port {port} within {CHROME_STARTUP_TIMEOUT_S:.0f}s")
+
+
+def _debugger_listening(port: int) -> bool:
+    try:
+        return httpx.get(f"http://127.0.0.1:{port}/json/version", timeout=2.0).status_code == 200
+    except httpx.HTTPError:
+        return False
+
+
+def _find_chrome() -> str | None:
+    for path in _CHROME_CANDIDATES.get(platform.system(), ()):
+        if Path(path).exists():
+            return path
+    for command in _CHROME_COMMANDS:
+        found = shutil.which(command)
+        if found:
+            return found
+    return None

src/hai_agents/local/config.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import os
+import uuid as _uuid
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+from typing_extensions import Self
+
+from ..environment import HaiAgentsEnvironment
+
+DEFAULT_BASE_URL = HaiAgentsEnvironment.EU.value
+KIND_TO_CAPABILITY = {"web": "browser", "desktop": "desktop"}
+CAPABILITIES = frozenset(KIND_TO_CAPABILITY.values())
+
+
+def session_id_from_environment_id(environment_id: str, api_key: str, capability: str) -> str:
+    return str(_uuid.uuid5(_uuid.NAMESPACE_DNS, f"{api_key}.{environment_id}.{capability}"))
+
+
+class SidecarConfig(BaseModel):
+    capability: str
+    environment_id: str
+    api_key: str = ""
+    base_url: str = DEFAULT_BASE_URL
+    session_id: str = ""
+    driver_options: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _resolve_defaults(self) -> Self:
+        if self.capability not in CAPABILITIES:
+            raise ValueError(f"unknown capability {self.capability!r}; expected one of {sorted(CAPABILITIES)}")
+        if not self.api_key:
+            self.api_key = os.getenv("HAI_API_KEY") or os.getenv("AGP_SERVICE_KEY") or os.getenv("AGP_API_KEY") or ""
+        if not self.api_key:
+            raise ValueError("api_key is required (or set HAI_API_KEY)")
+        if not self.session_id:
+            self.session_id = session_id_from_environment_id(self.environment_id, self.api_key, self.capability)
+        return self

src/hai_agents/local/pyautogui_desktop/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .driver import LocalDesktopDriver, RunCommandResponse
+
+__all__ = ["LocalDesktopDriver", "RunCommandResponse"]