diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index 76ce7a0b7..10d858f57 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -44,11 +44,17 @@ The A2A server starts with the real-time marker, so it and its records stay on r as a follower with a read token of its own and calls that `sync_time` with the follower's config, which the server writes; `deploy_agent` does the same for an agent it deploys while the clock is armed. From then on the CLI, its own "Today's date is" line included, and every command the model runs read the env's time, and a clock write that waits for -followers waits for the agent. While the clock is armed the CLI goes out through the egress proxy, which keeps a live -site's TLS working at any date the clock shows, and to the env's MCP servers and a plain-HTTP model endpoint directly. -The image's proxy reaches private networks too (`AGENTENV_EGRESS_PRIVATE`), so a model endpoint over TLS at a private -address works at any date as well. Unarmed, the page holds real time and the CLI goes out directly, as before. Started -as a user other than root, the image runs the A2A server alone, on real time, as before. +followers waits for the agent. While the clock is armed the CLI goes out through the egress proxy, which keeps TLS +working at any date the clock shows, a live site's, an env MCP server's or the model endpoint's; only loopback hosts +(the peer-agents MCP server's), `host.docker.internal`, the hosts the agent's `NO_PROXY` names and the hosts it reaches +by plain HTTP alone, which have no certificate to check, go direct. `NO_PROXY` goes by host, not scheme, so a host the +agent also reaches over TLS stays behind the proxy. The image's proxy reaches private networks too +(`AGENTENV_EGRESS_PRIVATE`), so a host over TLS at a private address works at any date as well. Unarmed, the page holds +real time and the CLI goes out directly, as before. That rule is `agent_egress.py` in `utils/agent_clock`, which only +agent images take (`agent_clock` in `shared_sources.toml`), so changing it rebuilds no computer base. The bridge +imports it, and the image and `install.sh` copy it beside the bridge with the clock's `clockpage.py`, whose layout and +check it reads the page by, so every agent put on the clock, OpenCode's too, goes out the same way. Started as a user +other than root, the image runs the A2A server alone, on real time, as before. `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements diff --git a/src/agent_env/agents/agents/claude-code/Dockerfile b/src/agent_env/agents/agents/claude-code/Dockerfile index b56404d44..9068d5d16 100644 --- a/src/agent_env/agents/agents/claude-code/Dockerfile +++ b/src/agent_env/agents/agents/claude-code/Dockerfile @@ -38,6 +38,8 @@ COPY requirements.txt model-requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -r model-requirements.txt # install.sh and run.sh are install/v1's: in the image, so they stay in its build context. COPY a2a_server.py claude_bridge.py kit.py tar_changelog.py entrypoint.sh install.sh run.sh ./ +# Every clocked agent's egress rule, which the bridge imports, and the clock page's format it reads (shared_sources.toml). +COPY agent_clock/agent_egress.py clock/clockpage.py ./ # The CLI's cwd, /app, and its state, fixed so neither follows the uid. RUN useradd --create-home --uid 10001 agent && mkdir -p /app /home/agent/.claude \ && chown agent:agent /app /home/agent/.claude && chmod 0755 entrypoint.sh diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index e1a30956d..8fb10c916 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -11,17 +11,16 @@ import re import signal import stat -import struct import time from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path from typing import Literal -from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, model_validator +from agent_egress import clock_armed, egress_env from kit import in_thread_to_its_end logger = logging.getLogger(__name__) @@ -175,74 +174,9 @@ def _model_headers(raw: str | None) -> dict[str, str]: return {str(name): str(value) for name, value in headers.items()} -#: Set by the image (root.conf) where the egress proxy runs: the CLI and the model's commands go out through it -#: while the agent follows an armed clock. -EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" -#: The clock page agent-env's libfaketime reads the CLI's time from (clock/clockpage.py, version 2): it keeps the last -#: armed clock while the follower reconnects or restarts, which the follower's status doesn't say. -CLOCK_PAGE = "/dev/shm/agentenv-clock" -_PAGE = struct.Struct(" bool: - """Whether the clock page holds an armed clock, so the CLI started now reads the env's time: the page itself, - as libfaketime reads it, retried while a write is under way (seq odd) or its check doesn't match.""" - try: - with open(page_path or CLOCK_PAGE, "rb") as page: - for _ in range(_PAGE_READ_TRIES): - page.seek(0) - raw = page.read(_PAGE.size) - if len(raw) < _PAGE.size: - return False - magic, version, seq, universe, anchor, rate, armed, _, generation, checked, mono_u, mono_a = \ - _PAGE.unpack(raw) - if magic != _PAGE_MAGIC or version != _PAGE_VERSION: - return False - rate_bits = struct.unpack(" list[str]: - """The model endpoint's host when the CLI reaches it directly on an armed clock: over plain HTTP, which has no - certificate to check at the env's date. Over TLS it goes through the egress proxy, which checks the certificate on - real time and, in an agent's image, reaches a private network's address too (``AGENTENV_EGRESS_PRIVATE``).""" - endpoint = urlsplit(base_url or "") - return [endpoint.hostname] if endpoint.hostname and endpoint.scheme == "http" else [] - - -def egress_env(environ: Mapping[str, str], direct: Iterable[str] = (), *, armed: bool = False) -> dict[str, str]: - """The egress proxy for the CLI and what it starts, where the image runs one (FD-5916) and the clock is ``armed``: - they read the env's time, so a client checking a live site's certificate against it would refuse the site once - the clock is months off, and the proxy, on real time, hands it a certificate that holds then. The env's MCP servers - (``direct``) are reached directly: the proxy refuses an address that isn't global, as a local or private gateway's - is. Unarmed, the CLI reads real time and goes out directly, as an image without the proxy does, so a model endpoint - or mirror at a private address keeps working. Hosts the agent was deployed to reach directly (``NO_PROXY``, - ``no_proxy``) stay direct.""" - proxy = environ.get(EGRESS_PROXY_ENV) - if not proxy or not armed: - return {} - configured = [host.strip() for name in ("NO_PROXY", "no_proxy") for host in (environ.get(name) or "").split(",")] - hosts = ",".join(dict.fromkeys([*_DIRECT_HOSTS, *filter(None, configured), *filter(None, direct)])) - return {**{name: proxy for base in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY") for name in (base, base.lower())}, - "NO_PROXY": hosts, "no_proxy": hosts, "NODE_USE_ENV_PROXY": "1"} - - def cli_env(environ: Mapping[str, str], *, model: str | None = None, project_id: str | None = None, task_id: str | None = None, max_thinking_tokens: int | None = None, - direct_hosts: Iterable[str] = (), clock_armed: bool = False) -> dict[str, str]: + clock_armed: bool = False, mcp_urls: Iterable[str] = ()) -> dict[str, str]: """The CLI's environment: this process's, pointed at the endpoint agent-env configured (its key as a bearer token, so an API key too would conflict), with the endpoint's headers and the run's attribution, and the model's aliases pinned to ``model`` on that endpoint. A variable already set is kept, LITELLM_* too, so a script the model @@ -271,8 +205,7 @@ def cli_env(environ: Mapping[str, str], *, model: str | None = None, project_id: # Older CLIs prepend a changing attribution block to the system prompt on custom base URLs, which defeats the # provider's prefix cache. env["CLAUDE_CODE_ATTRIBUTION_HEADER"] = "0" - model_hosts = model_direct_hosts(env.get("ANTHROPIC_BASE_URL")) if clock_armed else [] - env.update(egress_env(environ, [*direct_hosts, *model_hosts], armed=clock_armed)) + env.update(egress_env(environ, [env.get("ANTHROPIC_BASE_URL", ""), *mcp_urls], armed=clock_armed)) return env @@ -753,8 +686,7 @@ async def _spawn_claude( stderr=asyncio.subprocess.PIPE, env=cli_env(os.environ, model=model, project_id=project_id, task_id=task_id, max_thinking_tokens=max_thinking_tokens, clock_armed=clock_armed(), - direct_hosts=[urlsplit(str(entry.get("url") or "")).hostname or "" - for entry in (mcp_servers or {}).values()]), + mcp_urls=[info["url"] for info in (mcp_servers or {}).values()]), cwd=WORKSPACE_DIR, start_new_session=True, )) diff --git a/src/agent_env/agents/agents/claude-code/install.sh b/src/agent_env/agents/agents/claude-code/install.sh index a4f030723..5105a9edc 100755 --- a/src/agent_env/agents/agents/claude-code/install.sh +++ b/src/agent_env/agents/agents/claude-code/install.sh @@ -22,7 +22,7 @@ PYTHON_VERSION=3.12 START_TIMEOUT_S=120 STOP_TIMEOUT_S=10 CLAUDE_CODE_VERSION=$(sed -n 's/^ARG CLAUDE_CODE_VERSION=//p' "$here/Dockerfile") -AGENT_FILES="a2a_server.py claude_bridge.py kit.py requirements.txt run.sh tar_changelog.py" +AGENT_FILES="a2a_server.py claude_bridge.py kit.py requirements.txt run.sh tar_changelog.py agent_clock/agent_egress.py clock/clockpage.py" # put SRC DEST replaces DEST by a rename, so a binary an earlier install's agent still runs can be replaced. case $mode in @@ -123,7 +123,7 @@ put "$stage/claude" "$root/bin/claude" put "$stage/uv-$uv_cpu-unknown-linux-musl/uv" "$root/bin/uv" put "$stage/agent.env" "$root/agent.env" for file in $AGENT_FILES; do - put "$here/$file" "$root/app/$file" + put "$here/$file" "$root/app/${file##*/}" done on "chmod 0600 $r/agent.env && chmod 0755 $r/bin/claude $r/bin/uv $r/app/run.sh" diff --git a/src/agent_env/agents/agents/claude-code/shared_sources.toml b/src/agent_env/agents/agents/claude-code/shared_sources.toml index dc9d15a01..6076f3540 100644 --- a/src/agent_env/agents/agents/claude-code/shared_sources.toml +++ b/src/agent_env/agents/agents/claude-code/shared_sources.toml @@ -2,3 +2,4 @@ # copy of each, shared with the other images that take it (utils/docker/context.py). Paths are under src/agent_env. clock = "utils/faketime_clock" egress = "utils/egress_proxy" +agent_clock = "utils/agent_clock" diff --git a/src/agent_env/agents/agents/opencode/Dockerfile b/src/agent_env/agents/agents/opencode/Dockerfile index 6b2d5a3b3..fb49ee3ea 100644 --- a/src/agent_env/agents/agents/opencode/Dockerfile +++ b/src/agent_env/agents/agents/opencode/Dockerfile @@ -40,6 +40,8 @@ COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt # install.sh and run.sh are install/v1's: in the image, so they stay in its build context. COPY a2a_server.py opencode_bridge.py opencode_kit.py entrypoint.sh install.sh run.sh ./ +# Every clocked agent's egress rule, which the bridge imports, and the clock page's format it reads (shared_sources.toml). +COPY agent_clock/agent_egress.py clock/clockpage.py ./ # The CLI's cwd, /app, and its state under /data, fixed so neither follows the uid. RUN useradd --create-home --uid 10001 agent && mkdir -p /app /data \ && chown agent:agent /app /data && chmod 0755 entrypoint.sh diff --git a/src/agent_env/agents/agents/opencode/install.sh b/src/agent_env/agents/agents/opencode/install.sh index 5785d3bfe..ffb8308bd 100755 --- a/src/agent_env/agents/agents/opencode/install.sh +++ b/src/agent_env/agents/agents/opencode/install.sh @@ -24,7 +24,7 @@ START_TIMEOUT_S=120 STOP_TIMEOUT_S=10 dockerfile_arg() { sed -n "s/^ARG $1=//p" "$here/Dockerfile"; } NODE_VERSION=$(dockerfile_arg NODE_VERSION) -AGENT_FILES="a2a_server.py opencode_bridge.py opencode_kit.py requirements.txt run.sh" +AGENT_FILES="a2a_server.py opencode_bridge.py opencode_kit.py requirements.txt run.sh agent_clock/agent_egress.py clock/clockpage.py" # put SRC DEST replaces DEST by a rename, so a binary an earlier install's agent still runs can be replaced; put_dir SRC # DEST copies a folder to DEST, which isn't there. @@ -135,7 +135,7 @@ put "$stage/agent.env" "$root/agent.env" put "$here/package.json" "$root/npm/package.json" put "$here/package-lock.json" "$root/npm/package-lock.json" for file in $AGENT_FILES; do - put "$here/$file" "$root/app/$file" + put "$here/$file" "$root/app/${file##*/}" done on "chmod 0600 $r/agent.env && chmod 0755 $r/bin/uv $r/bin/opencode $r/app/run.sh" diff --git a/src/agent_env/agents/agents/opencode/opencode_bridge.py b/src/agent_env/agents/agents/opencode/opencode_bridge.py index f75a652c5..3fcabea1d 100644 --- a/src/agent_env/agents/agents/opencode/opencode_bridge.py +++ b/src/agent_env/agents/agents/opencode/opencode_bridge.py @@ -22,19 +22,18 @@ import os import re import signal -import struct import threading -import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Literal -from urllib.parse import urlsplit from jsonschema import Draft202012Validator, SchemaError, ValidationError from pydantic import BaseModel, ConfigDict, model_validator +from agent_egress import clock_armed, egress_env + logger = logging.getLogger(__name__) # The model a run uses when neither the task nor the agent-config names one; the @@ -99,80 +98,13 @@ def _context_xdg(context_id: str) -> dict[str, str]: return {"XDG_DATA_HOME": str(data_dir / "share"), "XDG_STATE_HOME": str(data_dir / "state")} -#: Set by the image (root.conf) where the egress proxy runs: the CLI and the model's commands go out through it -#: while the agent follows an armed clock. -EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" -#: The clock page agent-env's libfaketime reads the CLI's time from (clock/clockpage.py, version 2): it keeps the last -#: armed clock while the follower reconnects or restarts, which the follower's status doesn't say. -CLOCK_PAGE = "/dev/shm/agentenv-clock" -_PAGE = struct.Struct(" bool: - """Whether the clock page holds an armed clock, so the CLI started now reads the env's time: the page itself, - as libfaketime reads it, retried while a write is under way (seq odd) or its check doesn't match.""" - try: - with open(page_path or CLOCK_PAGE, "rb") as page: - for _ in range(_PAGE_READ_TRIES): - page.seek(0) - raw = page.read(_PAGE.size) - if len(raw) < _PAGE.size: - return False - magic, version, seq, universe, anchor, rate, armed, _, generation, checked, mono_u, mono_a = \ - _PAGE.unpack(raw) - if magic != _PAGE_MAGIC or version != _PAGE_VERSION: - return False - rate_bits = struct.unpack(" list[str]: - """The model endpoint's host when the CLI reaches it directly on an armed clock: over plain HTTP, which has no - certificate to check at the env's date. Over TLS it goes through the egress proxy, which checks the certificate on - real time and, in an agent's image, reaches a private network's address too (``AGENTENV_EGRESS_PRIVATE``).""" - endpoint = urlsplit(base_url or "") - return [endpoint.hostname] if endpoint.hostname and endpoint.scheme == "http" else [] - - -def egress_env(environ: Mapping[str, str], direct: Iterable[str] = (), *, armed: bool = False) -> dict[str, str]: - """The egress proxy for the CLI and what it starts, where the image runs one (FD-5916) and the clock is ``armed``: - they read the env's time, so a client checking a live site's certificate against it would refuse the site once - the clock is months off, and the proxy, on real time, hands it a certificate that holds then. The env's MCP servers - (``direct``) are reached directly: the proxy refuses an address that isn't global, as a local or private gateway's - is. Unarmed, the CLI reads real time and goes out directly, as an image without the proxy does, so a model endpoint - or mirror at a private address keeps working. Hosts the agent was deployed to reach directly (``NO_PROXY``, - ``no_proxy``) stay direct.""" - proxy = environ.get(EGRESS_PROXY_ENV) - if not proxy or not armed: - return {} - configured = [host.strip() for name in ("NO_PROXY", "no_proxy") for host in (environ.get(name) or "").split(",")] - hosts = ",".join(dict.fromkeys([*_DIRECT_HOSTS, *filter(None, configured), *filter(None, direct)])) - return {**{name: proxy for base in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY") for name in (base, base.lower())}, - "NO_PROXY": hosts, "no_proxy": hosts, "NODE_USE_ENV_PROXY": "1"} - - def cli_env(context_id: str, environ: Mapping[str, str] = os.environ, *, mcp_servers: dict[str, dict] | None = None, clock_armed: bool = False) -> dict[str, str]: - """The CLI's environment for ``context_id``: this process's, with the context's own data and state dirs. On an - armed clock (``clock_armed``) the CLI goes out through the egress proxy (``egress_env``), reaching the env's MCP - servers (``mcp_servers``), which it connects to itself, and a plain-HTTP model endpoint directly.""" - model_host = model_direct_hosts(environ.get("LITELLM_BASE_URL")) if clock_armed else [] - mcp_hosts = [url.hostname for url in (urlsplit(str(info.get("url") or "")) for info in (mcp_servers or {}).values()) if url.hostname and url.scheme == "http"] - return {**environ, **_context_xdg(context_id), - **egress_env(environ, [*mcp_hosts, *model_host], armed=clock_armed)} + """The CLI's environment for ``context_id``: this process's, with the context's own data and state dirs, and on an + armed clock the egress proxy (``egress_env``) for the model endpoint and the env's MCP servers, which the CLI + connects to itself.""" + urls = [environ.get("LITELLM_BASE_URL", ""), *(info["url"] for info in (mcp_servers or {}).values())] + return {**environ, **_context_xdg(context_id), **egress_env(environ, urls, armed=clock_armed)} async def _communicate(*cmd: str, env: Mapping[str, str], timeout_s: int) -> tuple[int, bytes, bytes]: diff --git a/src/agent_env/agents/agents/opencode/shared_sources.toml b/src/agent_env/agents/agents/opencode/shared_sources.toml index dc9d15a01..6076f3540 100644 --- a/src/agent_env/agents/agents/opencode/shared_sources.toml +++ b/src/agent_env/agents/agents/opencode/shared_sources.toml @@ -2,3 +2,4 @@ # copy of each, shared with the other images that take it (utils/docker/context.py). Paths are under src/agent_env. clock = "utils/faketime_clock" egress = "utils/egress_proxy" +agent_clock = "utils/agent_clock" diff --git a/src/agent_env/utils/agent_clock/agent_egress.py b/src/agent_env/utils/agent_clock/agent_egress.py new file mode 100644 index 000000000..8fad38ef2 --- /dev/null +++ b/src/agent_env/utils/agent_clock/agent_egress.py @@ -0,0 +1,55 @@ +"""How an agent on the env's clock goes out, imported by every clocked agent's bridge, whose image copies it and +``clockpage`` beside it: the agent's CLI checks certificates against the env's date, which can be before their issue, so +every TLS host goes through the image's egress proxy, on real time, and only hosts with nothing to check go direct.""" + +from __future__ import annotations + +import time +from collections.abc import Iterable, Mapping +from urllib.parse import urlsplit + +import clockpage + +EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" +_DIRECT_HOSTS = ("localhost", "127.0.0.1", "::1", "host.docker.internal") +_READ_TRIES, _RETRY_S = 50, 0.001 + + +def clock_armed(page_path: str | None = None) -> bool: + """Whether the clock page libfaketime reads, which keeps the last clock while the follower reconnects, holds an + armed clock, so a CLI started now reads the env's time. A page mid-write or torn through every retry is taken at its + armed flag, as the proxy relays untouched when unarmed; one that isn't a page or isn't there reads as unarmed.""" + try: + with open(page_path or clockpage.PAGE, "rb") as page: + for _ in range(_READ_TRIES): + page.seek(0) + raw = page.read(clockpage.LAYOUT.size) + if len(raw) < clockpage.LAYOUT.size: + return False + magic, version, seq, universe, anchor, rate, armed, _, generation, checked, mono_u, mono_a = \ + clockpage.LAYOUT.unpack(raw) + if (magic, version) != (clockpage.MAGIC, clockpage.VERSION): + return False + if seq % 2 == 0 and checked == clockpage.check(seq, universe, anchor, rate, armed, generation, mono_u, + mono_a): + return bool(armed) + time.sleep(_RETRY_S) + except OSError: + return False + return bool(armed) + + +def egress_env(environ: Mapping[str, str], urls: Iterable[str] = (), *, armed: bool) -> dict[str, str]: + """The proxy variables for the CLI and what it starts where the image runs the proxy and the clock is ``armed``, + else none: every host goes through the proxy but loopback, which it refuses, the hosts the deploy's NO_PROXY names, + and hosts ``urls`` name over plain HTTP alone, with no certificate to check (NO_PROXY goes by host, not scheme).""" + proxy = environ.get(EGRESS_PROXY_ENV) + if not proxy or not armed: + return {} + configured = [host.strip() for name in ("NO_PROXY", "no_proxy") for host in (environ.get(name) or "").split(",")] + parsed = [urlsplit(url) for url in urls] + tls = {parts.hostname for parts in parsed if parts.scheme != "http"} + plain = [parts.hostname for parts in parsed if parts.scheme == "http" and parts.hostname not in tls] + hosts = ",".join(dict.fromkeys(filter(None, [*_DIRECT_HOSTS, *configured, *plain]))) + return {**{name: proxy for base in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY") for name in (base, base.lower())}, + "NO_PROXY": hosts, "no_proxy": hosts, "NODE_USE_ENV_PROXY": "1"} diff --git a/tst/integration/task_step/test_agent_env_clock_local.py b/tst/integration/task_step/test_agent_env_clock_local.py index a17265238..87468d613 100644 --- a/tst/integration/task_step/test_agent_env_clock_local.py +++ b/tst/integration/task_step/test_agent_env_clock_local.py @@ -55,11 +55,11 @@ def _in_agent(container: str, *command: str, real_time: bool = False) -> str: def _routes_through_proxy(container: str, bridge: str) -> bool: - """Whether the CLI the agent starts now goes out through the egress proxy, as the image's bridge decides: every - bridge has the same ``clock_armed`` and ``egress_env``.""" + """Whether the CLI the agent starts now goes out through the egress proxy, as the image's bridge decides with the + ``clock_armed`` and ``egress_env`` every clocked bridge imports from ``agent_egress`` beside it.""" # The A2A server's environment has the proxy (root.conf); this shell's doesn't, so it's passed as the server has it. - return _in_agent(container, "python3", "-c", f"import {bridge} as b; print('HTTPS_PROXY' in b.egress_env(" - "{b.EGRESS_PROXY_ENV: 'http://127.0.0.1:3128'}, armed=b.clock_armed()))", + return _in_agent(container, "python3", "-c", f"import {bridge} as b, agent_egress as e; print('HTTPS_PROXY' in " + "b.egress_env({e.EGRESS_PROXY_ENV: 'http://127.0.0.1:3128'}, armed=b.clock_armed()))", real_time=True) == "True" diff --git a/tst/unit/agents/claude_code/agent.py b/tst/unit/agents/claude_code/agent.py index 4cdd3d02b..091845ee6 100644 --- a/tst/unit/agents/claude_code/agent.py +++ b/tst/unit/agents/claude_code/agent.py @@ -1,15 +1,21 @@ -"""The Claude Code agent's modules, imported from its folder as its image runs them: by their own names, from the -folder on the path.""" +"""The Claude Code agent's modules, imported as its image runs them: by their own names, from its folder and from the +folders its build context takes as agent_clock/ and clock/, whose modules the image copies beside the bridge.""" import importlib +import os import sys from pathlib import Path import agent_env +from tst.util.build_context import sent AGENT_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "claude-code" -sys.path.insert(0, str(AGENT_DIR)) +IMPORTED = [str(AGENT_DIR), str(sent(AGENT_DIR, "agent_clock/agent_egress.py").parent), + str(sent(AGENT_DIR, "clock/clockpage.py").parent)] +PYTHONPATH = os.pathsep.join(IMPORTED) +sys.path[:0] = IMPORTED a2a_server = importlib.import_module("a2a_server") tar_changelog = importlib.import_module("tar_changelog") claude_bridge = importlib.import_module("claude_bridge") kit = importlib.import_module("kit") +agent_egress = importlib.import_module("agent_egress") diff --git a/tst/unit/agents/claude_code/test_card_contract.py b/tst/unit/agents/claude_code/test_card_contract.py index 1a7ec6690..d905c1427 100644 --- a/tst/unit/agents/claude_code/test_card_contract.py +++ b/tst/unit/agents/claude_code/test_card_contract.py @@ -10,7 +10,7 @@ from agent_env.task_step.task_steps.install_agent import _build_param_resolvers -from tst.unit.agents.claude_code.agent import AGENT_DIR +from tst.unit.agents.claude_code.agent import AGENT_DIR, PYTHONPATH from tst.unit.agents.claude_code.support import card _CLAUDE_SKILL = { @@ -137,7 +137,7 @@ def test_install_agent_reads_the_card_and_its_install_commands_from_the_image(): "from a2a_server import AGENT_CARD; " "print(json.dumps(AGENT_CARD.model_dump()))") env = {**{k: v for k, v in os.environ.items() if not k.startswith(("LITELLM_", "ANTHROPIC_"))}, - "PYTHONPATH": str(AGENT_DIR)} + "PYTHONPATH": PYTHONPATH} out = subprocess.run([sys.executable, "-c", script], cwd=AGENT_DIR, env=env, capture_output=True, text=True, check=True) (install,) = [ext for ext in json.loads(out.stdout)["capabilities"]["extensions"] diff --git a/tst/unit/agents/claude_code/test_claude_bridge_cancel.py b/tst/unit/agents/claude_code/test_claude_bridge_cancel.py index a4ff23536..57a6eed0d 100644 --- a/tst/unit/agents/claude_code/test_claude_bridge_cancel.py +++ b/tst/unit/agents/claude_code/test_claude_bridge_cancel.py @@ -13,7 +13,7 @@ import time import pytest -from tst.unit.agents.claude_code.agent import AGENT_DIR, claude_bridge +from tst.unit.agents.claude_code.agent import PYTHONPATH, claude_bridge @@ -184,7 +184,7 @@ def test_stopping_the_agents_server_kills_the_clis_of_turns_still_running_and_th (bin_dir / "claude").write_text(f'#!/bin/sh\n{tool} > /dev/null 2>&1 < /dev/null & echo "$$ $!" > {pids}\nwait\n') (bin_dir / "claude").chmod(0o755) (tmp_path / "app").mkdir() - env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "PYTHONPATH": str(AGENT_DIR), + env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "PYTHONPATH": PYTHONPATH, "CLAUDE_CONFIG_DIR": str(tmp_path / "state"), "AGENT_WORKSPACE_DIR": str(tmp_path / "app")} with log.open("w") as out: server = subprocess.Popen([sys.executable, "-m", "uvicorn", "a2a_server:app", "--host", "127.0.0.1", diff --git a/tst/unit/agents/claude_code/test_env_clock_image.py b/tst/unit/agents/claude_code/test_env_clock_image.py index 1f5432a96..64de53fe4 100644 --- a/tst/unit/agents/claude_code/test_env_clock_image.py +++ b/tst/unit/agents/claude_code/test_env_clock_image.py @@ -1,45 +1,69 @@ -"""Claude Code's bridge on the env's clock (FD-5916): while the clock page is armed, the CLI goes out through the -egress proxy, and to the env's MCP servers, the hosts NO_PROXY names and a plain-HTTP model endpoint directly; unarmed, -it goes out as before. What every clocked image shares is checked in tst/unit/agents/test_agent_clock_images.py.""" +"""Claude Code's bridge on the env's clock: it hands the shared egress rule (tst/unit/agents/test_agent_egress.py) the +model endpoint's URL and every MCP server's, so on an armed clock an env MCP server over TLS goes through the egress +proxy and loopback and plain-HTTP hosts go direct; unarmed, the CLI goes out as before. What every clocked image shares +is checked in tst/unit/agents/test_agent_clock_images.py.""" from __future__ import annotations +import asyncio import json -import types +import re from urllib.parse import urlsplit import pytest -from tst.unit.agents.claude_code.agent import AGENT_DIR, claude_bridge +from tst.unit.agents.claude_code.agent import AGENT_DIR, agent_egress, claude_bridge from tst.util.build_context import sent PROXY = "http://127.0.0.1:3128" +ARMED_PAGE = agent_egress.clockpage.encode(2, 1_900_000_000 * 10**9, 10**18, 1.0, 1, 3, 0, 0) -def test_on_an_armed_clock_the_cli_goes_out_through_the_proxy_and_to_the_envs_servers_directly(): - env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY}, direct_hosts=["gateway", "", "gateway"], - clock_armed=True) +def test_on_an_armed_clock_the_cli_goes_out_through_the_proxy_and_to_loopback_directly(): + env = claude_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY}, clock_armed=True) assert {env[name] for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "https_proxy")} == {PROXY} - assert env["NO_PROXY"] == env["no_proxy"] == "localhost,127.0.0.1,::1,host.docker.internal,gateway" + assert env["NO_PROXY"] == env["no_proxy"] == "localhost,127.0.0.1,::1,host.docker.internal" assert env["NODE_USE_ENV_PROXY"] == "1" -def test_hosts_the_agent_was_deployed_to_reach_directly_stay_direct_on_an_armed_clock(): - # A private registry the agent's NO_PROXY names worked before the clock was armed; the proxy would refuse it. - env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY, "NO_PROXY": "registry.internal, .corp", - "no_proxy": "mirror.internal"}, direct_hosts=["gateway"], clock_armed=True) +def _spawned_env(monkeypatch, tmp_path, armed: bool) -> dict[str, str]: + """The environment _spawn_claude starts the CLI with, for an env MCP server over TLS, one over plain HTTP and the + peer-agents one.""" + spawned = {} + + async def fake_exec(*cmd, env, **_): + spawned.update(env) + raise RuntimeError("not starting the CLI") + + monkeypatch.setattr(claude_bridge.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(claude_bridge, "clock_armed", lambda: armed) + monkeypatch.setattr(claude_bridge.os, "environ", {agent_egress.EGRESS_PROXY_ENV: PROXY}) + monkeypatch.setattr(claude_bridge, "WORKSPACE_DIR", tmp_path) + servers = {"env": {"url": "https://abc-123.w.modal.host/mcp"}, "local": {"url": "http://gateway:8080/mcp"}, + "peers": {"url": "http://127.0.0.1:8000/peers/mcp"}} + with pytest.raises(RuntimeError, match="not starting the CLI"): + asyncio.run(claude_bridge._spawn_claude("hi", "s", False, mcp_servers=servers)) + return spawned + + +def test_on_an_armed_clock_an_env_mcp_server_over_tls_goes_through_the_proxy_and_the_others_direct(monkeypatch, + tmp_path): + env = _spawned_env(monkeypatch, tmp_path, armed=True) hosts = env["NO_PROXY"].split(",") - assert env["no_proxy"] == env["NO_PROXY"] - assert {"registry.internal", ".corp", "mirror.internal", "gateway", "localhost"} <= set(hosts) - assert len(hosts) == len(set(hosts)) and "" not in hosts + assert env["HTTPS_PROXY"] == env["https_proxy"] == PROXY + assert "abc-123.w.modal.host" not in hosts and {"gateway", "127.0.0.1"} <= set(hosts) + + +def test_unarmed_the_cli_reaches_every_mcp_server_directly(monkeypatch, tmp_path): + env = _spawned_env(monkeypatch, tmp_path, armed=False) + + assert not {"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"} & set(env) @pytest.mark.parametrize("base_url, direct", [ - ("http://10.0.0.5:4000", True), # plain HTTP: nothing to check at the env's date - # TLS, at a private address or a public one: its certificate is checked through the proxy, on real time, and the - # agent's proxy reaches private networks (root.conf's AGENTENV_EGRESS_PRIVATE) + ("http://10.0.0.5:4000", True), ("https://10.0.0.5:4000", False), ("https://[fd00::5]:4000", False), ("https://litellm.internal:4000", False), @@ -47,56 +71,32 @@ def test_hosts_the_agent_was_deployed_to_reach_directly_stay_direct_on_an_armed_ ]) def test_a_model_endpoint_over_plain_http_is_reached_directly_and_one_over_tls_through_the_proxy_on_an_armed_clock( base_url, direct): - env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url, + env = claude_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url, "LITELLM_API_KEY": "k"}, clock_armed=True) assert (urlsplit(base_url).hostname in env["NO_PROXY"].split(",")) is direct and env["HTTPS_PROXY"] == PROXY -@pytest.mark.parametrize("environ, armed", [({}, True), ({claude_bridge.EGRESS_PROXY_ENV: PROXY}, False)]) +@pytest.mark.parametrize("environ, armed", [({}, True), ({agent_egress.EGRESS_PROXY_ENV: PROXY}, False)]) def test_without_the_proxy_or_an_armed_clock_the_cli_goes_out_directly_as_before(environ, armed): - # A model endpoint at a private address, which the proxy refuses, keeps working until a task arms the clock. assert not {"HTTPS_PROXY", "NO_PROXY"} & set(claude_bridge.cli_env(environ, clock_armed=armed)) -def _clockpage(): - """The image's clock page writer, whose encoding libfaketime and the egress proxy read too.""" - # Run from its source, so no __pycache__ lands in agent-env's sources. - module = types.ModuleType("clockpage") - path = sent(AGENT_DIR, "clock/clockpage.py") - exec(compile(path.read_text(), str(path), "exec"), module.__dict__) - return module - - -def _page(armed: int, seq: int = 2) -> bytes: - return _clockpage().encode(seq, 1_900_000_000 * 10**9 if armed else 0, 10**18 if armed else 0, 1.0, armed, 3, 0, 0) - - -@pytest.mark.parametrize("page, armed", [ - (_page(1), True), - (_page(0), False), - # A write under way (seq odd) that never finishes: the armed flag it left is the best guess. - (_page(1, seq=3), True), - (_page(1)[:40], False), # not a page - (b"\0" * 80, False), # not one yet: no magic - (None, False), # no page: an image without the clock, or install_agent's target -]) -def test_the_clock_is_armed_when_the_page_libfaketime_reads_says_so(tmp_path, page, armed): - path = tmp_path / "agentenv-clock" - if page is not None: - path.write_bytes(page) - assert claude_bridge.clock_armed(str(path)) is armed - - @pytest.mark.parametrize("status", [ - {"connected": False, "error": "URLError: gateway restarting"}, # the follower lost its gateway - {"connected": False, "waiting": "no config yet"}, # the follower restarted, before its config is read - {"connected": True, "armed": False, "version": None}, # ... and before its first watch answers -]) + {"connected": False, "error": "URLError: gateway restarting"}, + {"connected": False, "waiting": "no config yet"}, + {"connected": True, "armed": False, "version": None}, +], ids=["lost-its-gateway", "restarted-before-its-config", "restarted-before-its-first-watch"]) def test_while_the_follower_reconnects_or_restarts_the_page_still_routes_the_cli_through_the_proxy(tmp_path, status): - # The page keeps the last armed clock in both windows, so the CLI reads the env's time and must go through egress. (tmp_path / "status.json").write_text(json.dumps(status)) page = tmp_path / "agentenv-clock" - page.write_bytes(_page(1)) - env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY}, clock_armed=claude_bridge.clock_armed(str(page))) + page.write_bytes(ARMED_PAGE) + env = claude_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY}, + clock_armed=agent_egress.clock_armed(str(page))) assert env["HTTPS_PROXY"] == PROXY + +def test_an_installed_agent_has_the_egress_rule_beside_its_bridge_as_the_image_does(): + listed = re.search(r'^AGENT_FILES="(.+)"$', (AGENT_DIR / "install.sh").read_text(), re.MULTILINE).group(1).split() + + assert {"agent_clock/agent_egress.py", "clock/clockpage.py"} <= set(listed) + assert all(sent(AGENT_DIR, name).is_file() for name in listed) diff --git a/tst/unit/agents/opencode/agent.py b/tst/unit/agents/opencode/agent.py index d131aaa9b..84d32b7d8 100644 --- a/tst/unit/agents/opencode/agent.py +++ b/tst/unit/agents/opencode/agent.py @@ -1,17 +1,23 @@ -"""The OpenCode agent's modules, imported from its folder as its image runs them: by their own names, from the folder -on the path.""" +"""The OpenCode agent's modules, imported as its image runs them: by their own names, from its folder and from the +folders its build context takes as agent_clock/ and clock/, whose modules the image copies beside the bridge.""" import importlib import importlib.util +import os import sys from pathlib import Path import agent_env +from tst.util.build_context import sent AGENT_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "opencode" -sys.path.insert(0, str(AGENT_DIR)) +IMPORTED = [str(AGENT_DIR), str(sent(AGENT_DIR, "agent_clock/agent_egress.py").parent), + str(sent(AGENT_DIR, "clock/clockpage.py").parent)] +PYTHONPATH = os.pathsep.join(IMPORTED) +sys.path[:0] = IMPORTED opencode_bridge = importlib.import_module("opencode_bridge") opencode_kit = importlib.import_module("opencode_kit") +agent_egress = importlib.import_module("agent_egress") # By its path: another agent's a2a_server, imported by the same name, may be loaded already. _spec = importlib.util.spec_from_file_location("opencode_a2a_server", AGENT_DIR / "a2a_server.py") a2a_server = importlib.util.module_from_spec(_spec) diff --git a/tst/unit/agents/opencode/test_card_contract.py b/tst/unit/agents/opencode/test_card_contract.py index 94582d36d..2b75178fc 100644 --- a/tst/unit/agents/opencode/test_card_contract.py +++ b/tst/unit/agents/opencode/test_card_contract.py @@ -12,7 +12,7 @@ import pytest -from tst.unit.agents.opencode.agent import AGENT_DIR, a2a_server, opencode_bridge +from tst.unit.agents.opencode.agent import AGENT_DIR, PYTHONPATH, a2a_server, opencode_bridge from tst.unit.agents.opencode.conftest import Harness _ENDPOINT = ["litellm_api_key", "litellm_base_url", "litellm_extra_headers", "trust_env"] @@ -92,7 +92,8 @@ def test_install_mode_reads_the_card_by_importing_agent_card(): """agent-env's install step imports AGENT_CARD from the image with no agent environment, and parses its stdout as the card.""" script = "import json; from a2a_server import AGENT_CARD; print(json.dumps(AGENT_CARD.model_dump(mode='json')))" - env = {key: value for key, value in os.environ.items() if not key.startswith("LITELLM_")} + env = {**{key: value for key, value in os.environ.items() if not key.startswith("LITELLM_")}, + "PYTHONPATH": PYTHONPATH} out = subprocess.run([sys.executable, "-c", script], cwd=AGENT_DIR, env=env, capture_output=True, text=True, check=True) extensions = {ext["uri"]: ext for ext in json.loads(out.stdout)["capabilities"]["extensions"]} diff --git a/tst/unit/agents/opencode/test_env_clock.py b/tst/unit/agents/opencode/test_env_clock.py index a6c494548..67d69e5f9 100644 --- a/tst/unit/agents/opencode/test_env_clock.py +++ b/tst/unit/agents/opencode/test_env_clock.py @@ -1,18 +1,19 @@ -"""OpenCode on the env's clock (FD-5916): while the clock page is armed the CLI goes out through the image's egress -proxy, and to the hosts NO_PROXY names, the env's MCP servers, which it connects to itself, and a plain-HTTP model -endpoint directly; unarmed, it goes out as before. What every clocked image shares is checked in -tst/unit/agents/test_agent_clock_images.py.""" +"""OpenCode on the env's clock: its bridge hands the shared egress rule (tst/unit/agents/test_agent_egress.py) the +model endpoint's URL and every MCP server's, which the CLI connects to itself, so on an armed clock it goes out through +the image's egress proxy, and to loopback, the hosts NO_PROXY names and plain-HTTP hosts directly; unarmed, it goes out +as before. What every clocked image shares is checked in tst/unit/agents/test_agent_clock_images.py.""" from __future__ import annotations +import subprocess + import pytest -from tst.unit.agents.opencode.agent import opencode_bridge +from tst.unit.agents.opencode.agent import AGENT_DIR, agent_egress, opencode_bridge PROXY = "http://127.0.0.1:3128" - @pytest.fixture(autouse=True) def _sessions(tmp_path, monkeypatch): monkeypatch.setattr(opencode_bridge, "SESSIONS_ROOT", str(tmp_path)) @@ -23,16 +24,15 @@ def _direct(env: dict[str, str]) -> set[str]: def test_on_an_armed_clock_the_cli_goes_out_through_the_proxy(): - env = opencode_bridge.cli_env("ctx", {opencode_bridge.EGRESS_PROXY_ENV: PROXY, "NO_PROXY": "registry.internal"}, + env = opencode_bridge.cli_env("ctx", {agent_egress.EGRESS_PROXY_ENV: PROXY, "NO_PROXY": "registry.internal"}, clock_armed=True) assert {env[name] for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "https_proxy")} == {PROXY} assert {"localhost", "host.docker.internal", "registry.internal"} <= _direct(env) - # Still the context's own session store. assert env["XDG_DATA_HOME"] == str(opencode_bridge.context_data_dir("ctx") / "share") -@pytest.mark.parametrize("environ, armed", [({}, True), ({opencode_bridge.EGRESS_PROXY_ENV: PROXY}, False)]) +@pytest.mark.parametrize("environ, armed", [({}, True), ({agent_egress.EGRESS_PROXY_ENV: PROXY}, False)]) def test_without_the_proxy_or_an_armed_clock_the_cli_goes_out_directly_as_before(environ, armed): env = opencode_bridge.cli_env("ctx", environ, clock_armed=armed) @@ -42,28 +42,25 @@ def test_without_the_proxy_or_an_armed_clock_the_cli_goes_out_directly_as_before @pytest.mark.parametrize("base_url, direct", [("http://10.0.0.5:4000", True), ("https://litellm.example.com", False)]) def test_a_model_endpoint_over_plain_http_is_reached_directly_on_an_armed_clock(base_url, direct): - env = opencode_bridge.cli_env("ctx", {opencode_bridge.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url}, + env = opencode_bridge.cli_env("ctx", {agent_egress.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url}, clock_armed=True) assert (base_url.split("//")[1].split(":")[0] in _direct(env)) is direct def test_the_envs_plain_http_mcp_servers_are_reached_directly_and_https_ones_through_the_proxy_on_an_armed_clock(): - servers = {"env": {"url": "http://host.docker.internal:8080/mcp"}, - "gateway": {"url": "https://mcp.gateway.internal/mcp", "headers": {"x": "y"}}} - env = opencode_bridge.cli_env("ctx", {opencode_bridge.EGRESS_PROXY_ENV: PROXY}, mcp_servers=servers, + """Bun checks an HTTPS server's certificate against the env's date, which the proxy, on real time, doesn't.""" + servers = {"env": {"url": "http://gateway:8080/mcp"}, + "remote": {"url": "https://abc-123.w.modal.host/mcp", "headers": {"x": "y"}}} + env = opencode_bridge.cli_env("ctx", {agent_egress.EGRESS_PROXY_ENV: PROXY}, mcp_servers=servers, clock_armed=True) - # Bun checks an HTTPS server's certificate against the env's date, which the proxy, on real time, doesn't. - assert "host.docker.internal" in _direct(env) and "mcp.gateway.internal" not in _direct(env) + assert "gateway" in _direct(env) and "abc-123.w.modal.host" not in _direct(env) def test_start_agent_keeps_a_providers_ca_bundle_trusted_by_the_server_and_the_proxy(tmp_path): - # Sail points SSL_CERT_FILE and the like at the bundle it signs model requests under; root.conf points the server - # at agent-env's bundles, so start-agent adds the provider's to them and to the proxy's roots, once each. - import subprocess - from pathlib import Path - - start = (Path(__file__).parents[4] / "src/agent_env/agents/agents/opencode/start-agent").read_text() + """Sail points SSL_CERT_FILE and the like at the bundle it signs model requests under; root.conf points the server + at agent-env's bundles, so start-agent adds the provider's to them and to the proxy's roots, once each.""" + start = (AGENT_DIR / "start-agent").read_text() merge = start[start.index('for bundle in "${SSL_CERT_FILE'):start.index("exec supervisord")] targets = {"/etc/agentenv/ca-bundle.pem": tmp_path / "ca-bundle.pem", "/etc/agentenv/ca.pem": tmp_path / "ca.pem", "/usr/local/lib/agentenv-egress/roots.pem": tmp_path / "roots.pem"} @@ -79,3 +76,4 @@ def test_start_agent_keeps_a_providers_ca_bundle_trusted_by_the_server_and_the_p for fake in targets.values(): text = fake.read_text() assert text.startswith("SYSTEM ROOTS\n") and text.count("SAIL") == 1, (fake.name, text) + diff --git a/tst/unit/agents/opencode/test_image_contract.py b/tst/unit/agents/opencode/test_image_contract.py index b95efaabb..356c8bb1c 100644 --- a/tst/unit/agents/opencode/test_image_contract.py +++ b/tst/unit/agents/opencode/test_image_contract.py @@ -22,11 +22,15 @@ def _copied() -> list[str]: for name in line.split()] -def _in_context(name: str) -> bool: - """Whether the build context has ``name``: from the folder, or under a folder shared_sources.toml takes from - agent-env's own sources at that name.""" +def _source(name: str): + """Where the build context's ``name`` comes from: the folder, or a folder shared_sources.toml takes from agent-env's + own sources at that name.""" first, _, rest = name.partition("/") - path = SOURCES / SHARED[first] / rest if first in SHARED else AGENT_DIR / name + return SOURCES / SHARED[first] / rest if first in SHARED else AGENT_DIR / name + + +def _in_context(name: str) -> bool: + path = _source(name) return path.is_dir() if name.endswith("/") else path.is_file() @@ -50,10 +54,19 @@ def _imported(path) -> set[str]: def test_the_image_and_an_install_carry_every_module_the_server_imports(): - modules = {path.stem: path for path in AGENT_DIR.glob("*.py")} - needed = {"a2a_server.py"} | {f"{name}.py" for path in modules.values() for name in _imported(path) & set(modules)} + """The folder's own modules and those the build context takes from agent-env's sources, the egress rule and the + clock page's format among them, each by its name in the context.""" + shared = {path.stem: f"{name}/{path.name}" for name, source in SHARED.items() + for path in (SOURCES / source).glob("*.py")} + modules = {**shared, **{path.stem: path.name for path in AGENT_DIR.glob("*.py")}} + needed, pending = set(), ["a2a_server"] + while pending: + if (name := modules[pending.pop()]) not in needed: + needed.add(name) + pending += _imported(_source(name)) & set(modules) installed = set(re.search(r'^AGENT_FILES="([^"]+)"$', INSTALL_SH, re.MULTILINE).group(1).split()) + assert {"agent_clock/agent_egress.py", "clock/clockpage.py"} <= needed assert needed <= set(_copied()) and needed <= installed, (needed, installed) diff --git a/tst/unit/agents/test_agent_clock_images.py b/tst/unit/agents/test_agent_clock_images.py index 055ed3683..971dd1d80 100644 --- a/tst/unit/agents/test_agent_clock_images.py +++ b/tst/unit/agents/test_agent_clock_images.py @@ -1,13 +1,14 @@ -"""Every agent image on the env's clock (FD-5916), whichever agent it is: each folder whose ``shared_sources.toml`` -takes the clock takes the computer bases' clock library and egress proxy from agent-env's one copy, whose libfaketime -patch has the init guard; runs the follower and the proxy, each as its own user, and the A2A server as the agent's user -on real time, under root's supervisord; preloads agent-env's libfaketime for the agent's user alone; starts as another -user as it did before; and its bridge decides the CLI's proxy with the same ``clock_armed`` and ``egress_env`` as every -other agent's.""" +"""Every agent image on the env's clock, whichever agent it is: each folder whose ``shared_sources.toml`` takes the +clock takes the computer bases' clock library and egress proxy from agent-env's one copy, whose libfaketime patch has +the init guard; runs the follower and the proxy, each as its own user, and the A2A server as the agent's user on real +time, under root's supervisord; preloads agent-env's libfaketime for the agent's user alone; starts as another user as +it did before; and its bridge takes ``clock_armed`` and ``egress_env`` from the one shared ``agent_egress``, in a folder +only agent images take, which its image copies beside it.""" from __future__ import annotations import ast +import posixpath import re import tomllib from pathlib import Path @@ -34,6 +35,17 @@ def test_the_clock_pieces_are_taken_from_the_one_copy_the_computer_bases_take(ag assert not (AGENTS_DIR / agent / part).exists(), f"{agent}/{part} is a copy: take it in shared_sources.toml instead" +@pytest.mark.parametrize("agent", CLOCKED) +def test_the_egress_rule_is_taken_from_the_one_copy_only_agent_images_take(agent): + """A computer base that took it would be rebuilt, and every computer composed on it, whenever the rule changes.""" + taken = tomllib.loads((AGENTS_DIR / agent / "shared_sources.toml").read_text()) + others = [p for p in SOURCES.glob("*/*/*/shared_sources.toml") if AGENTS_DIR not in p.parents] + + assert taken["agent_clock"] == "utils/agent_clock" + assert not (AGENTS_DIR / agent / "agent_clock").exists() + assert others and not [p for p in others if "utils/agent_clock" in tomllib.loads(p.read_text()).values()] + + def test_the_libfaketime_every_clocked_image_builds_has_the_init_guard(): # A CLI that reads the clock while libfaketime's init runs (an allocator, as dlsym allocates) has pthread_once wait # on itself: Claude Code's `--version` never returned. The guard has a clock call made during the init read the @@ -87,16 +99,50 @@ def test_started_as_another_user_the_image_runs_the_server_alone_as_before(agent assert "AGENTENV_SESSION_USER=agent AGENTENV_EGRESS_NSS=off" in start -def _clock_helpers(agent: str) -> dict[str, str]: - """The bridge's clock helpers, as source: the one module in the agent's folder that defines ``egress_env``.""" - [module] = [p for p in (AGENTS_DIR / agent).glob("*.py") if "def egress_env(" in p.read_text()] - tree = ast.parse(module.read_text()) - names = {"clock_armed", "model_direct_hosts", "egress_env"} - found = {node.name: ast.unparse(node) for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in names} - assert set(found) == names, (module, sorted(found)) - return found +SHARED = ("clock_armed", "egress_env") +SHARED_FILES = ("agent_clock/agent_egress.py", "clock/clockpage.py") + + +def _modules(agent: str) -> dict[str, ast.Module]: + return {path.name: ast.parse(path.read_text()) for path in sorted((AGENTS_DIR / agent).glob("*.py"))} + + +def _bridges(agent: str) -> list[str]: + """The agent's modules that take the egress rule from the shared module.""" + return [name for name, tree in _modules(agent).items() + if any(isinstance(node, ast.ImportFrom) and node.module == "agent_egress" + and set(SHARED) <= {alias.name for alias in node.names} for node in tree.body)] + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_every_bridge_takes_the_egress_rule_from_the_shared_module_and_defines_none_of_it(agent): + assert _bridges(agent), f"no module of {agent}'s imports {SHARED} from agent_egress" + for name, tree in _modules(agent).items(): + defined = {node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + defined |= {target.id for node in tree.body if isinstance(node, ast.Assign) + for target in node.targets if isinstance(target, ast.Name)} + own = defined & {*SHARED, "EGRESS_PROXY_ENV"} + assert not own, (name, sorted(own)) + + +def _copied_to(dockerfile: str) -> dict[str, set[str]]: + """The folders the image's last stage copies each file of its build context into, by the file's name there.""" + workdir, folders = "/", {} + stage = re.split(r"^FROM .*$", dockerfile, flags=re.MULTILINE)[-1] + for instruction, args in re.findall(r"^(WORKDIR|COPY) (.+)$", stage, re.MULTILINE): + if instruction == "WORKDIR": + workdir = posixpath.join(workdir, args) + elif not re.search(r"(^| )--from=", args): + *sources, dest = [arg for arg in args.split() if not arg.startswith("--")] + dest = posixpath.join(workdir, dest) + folder = posixpath.normpath(dest if dest.endswith("/") or len(sources) > 1 else posixpath.dirname(dest)) + for source in sources: + folders.setdefault(source, set()).add(folder) + return folders @pytest.mark.parametrize("agent", CLOCKED) -def test_every_bridge_decides_the_clis_proxy_as_claude_codes_does(agent): - assert _clock_helpers(agent) == _clock_helpers("claude-code") +def test_every_clocked_image_copies_the_shared_module_and_the_page_format_beside_its_bridge(agent): + folders = _copied_to((AGENTS_DIR / agent / "Dockerfile").read_text()) + + assert set.intersection(*(folders.get(name, set()) for name in (*_bridges(agent), *SHARED_FILES))), folders diff --git a/tst/unit/agents/test_agent_egress.py b/tst/unit/agents/test_agent_egress.py new file mode 100644 index 000000000..49e02ec19 --- /dev/null +++ b/tst/unit/agents/test_agent_egress.py @@ -0,0 +1,118 @@ +"""The egress rule every agent on the env's clock goes out by (utils/agent_clock/agent_egress.py), imported by its own +name with clockpage beside it, as each clocked agent's image copies them beside its bridge: on an armed clock, every +host but loopback, the deploy's NO_PROXY hosts and plain-HTTP hosts goes through the proxy; unarmed, nothing changes.""" + +from __future__ import annotations + +import importlib +import sys +import time +from pathlib import Path +from urllib.parse import urlsplit + +import pytest + +import agent_env + +sys.path[:0] = [str(Path(agent_env.__file__).parent / "utils" / folder) for folder in ("agent_clock", "faketime_clock")] +agent_egress = importlib.import_module("agent_egress") +clockpage = importlib.import_module("clockpage") + +PROXY = "http://127.0.0.1:3128" +LOOPBACK = ["localhost", "127.0.0.1", "::1", "host.docker.internal"] + + +def _direct(environ: dict[str, str] | None = None, urls: list[str] | None = None) -> list[str]: + env = agent_egress.egress_env({agent_egress.EGRESS_PROXY_ENV: PROXY, **(environ or {})}, urls or [], armed=True) + assert env["NO_PROXY"] == env["no_proxy"] + return env["NO_PROXY"].split(",") + + +def test_on_an_armed_clock_everything_goes_through_the_proxy_but_loopback(): + env = agent_egress.egress_env({agent_egress.EGRESS_PROXY_ENV: PROXY}, armed=True) + + assert {env[name] for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", + "all_proxy")} == {PROXY} + assert env["NODE_USE_ENV_PROXY"] == "1" + assert _direct() == LOOPBACK + + +@pytest.mark.parametrize("url, direct", [ + ("http://gateway:8080/mcp", True), + ("http://10.0.0.5:4000", True), + ("HTTP://Gateway.Internal/mcp", True), + ("https://abc-123.w.modal.host/mcp", False), + ("https://10.0.0.5:4000", False), + ("https://[fd00::5]:4000", False), + ("https://litellm.example.com", False), +]) +def test_a_plain_http_host_goes_direct_and_a_tls_one_through_the_proxy(url, direct): + assert (urlsplit(url).hostname in _direct(urls=[url])) is direct + + +@pytest.mark.parametrize("urls", [ + ["http://tools.example.com:8080/mcp", "https://tools.example.com/mcp"], + ["https://tools.example.com/mcp", "http://tools.example.com:8080/mcp"], +]) +def test_a_host_also_reached_over_tls_stays_behind_the_proxy_since_no_proxy_goes_by_host(urls): + assert _direct(urls=urls) == LOOPBACK + + +def test_the_hosts_the_deploy_names_in_no_proxy_stay_direct(): + hosts = _direct({"NO_PROXY": "registry.internal, .corp", "no_proxy": "mirror.internal"}) + + assert hosts == [*LOOPBACK, "registry.internal", ".corp", "mirror.internal"] + + +def test_empty_and_duplicate_hosts_are_dropped(): + hosts = _direct({"NO_PROXY": "gateway,, localhost,", "no_proxy": " gateway "}, + ["http://gateway/mcp", "", "http:///mcp", "http://gateway:9000", "http://127.0.0.1:8000/peers/mcp"]) + + assert hosts == [*LOOPBACK, "gateway"] + + +@pytest.mark.parametrize("environ, armed", [ + pytest.param({agent_egress.EGRESS_PROXY_ENV: PROXY}, False, id="unarmed"), + pytest.param({}, True, id="no-proxy-in-the-image"), + pytest.param({agent_egress.EGRESS_PROXY_ENV: ""}, True, id="empty-proxy"), +]) +def test_unarmed_or_without_the_proxy_nothing_changes(environ, armed): + assert agent_egress.egress_env({**environ, "NO_PROXY": "registry.internal"}, ["http://gateway/mcp"], + armed=armed) == {} + + +def _page(armed: int, seq: int = 2) -> bytes: + return clockpage.encode(seq, 1_900_000_000 * 10**9 if armed else 0, 10**18 if armed else 0, 1.0, armed, 3, 0, 0) + + +def _torn(page: bytes) -> bytes: + """``page`` with a field changed after its check was written.""" + return page[:16] + bytes([page[16] ^ 1]) + page[17:] + + +@pytest.mark.parametrize("page, armed", [ + pytest.param(_page(1), True, id="armed"), + pytest.param(_page(0), False, id="unarmed"), + pytest.param(_page(1, seq=3), True, id="its-writer-died-mid-write"), + pytest.param(_torn(_page(1)), True, id="torn-armed"), + pytest.param(_torn(_page(0)), False, id="torn-unarmed"), + pytest.param(_page(1)[:40], False, id="short"), + pytest.param(b"\0" * 80, False, id="no-magic"), + pytest.param(bytes(range(80)), False, id="garbage"), + pytest.param(None, False, id="no-page"), +]) +def test_the_clock_is_armed_when_the_page_libfaketime_reads_says_so(tmp_path, page, armed): + path = tmp_path / "agentenv-clock" + if page is not None: + path.write_bytes(page) + assert agent_egress.clock_armed(str(path)) is armed + + +def test_it_reads_the_page_clockpage_writes(monkeypatch, tmp_path): + monkeypatch.setattr(clockpage, "PAGE", str(tmp_path / "agentenv-clock")) + monkeypatch.setattr(clockpage, "FILE", str(tmp_path / "clock")) + + clockpage.write(1, 1_900_000_000.0, 1.0, time.time()) + assert agent_egress.clock_armed() + clockpage.write(0, 0.0, 1.0, time.time()) + assert not agent_egress.clock_armed()