diff --git a/src/agent_env/agents/agents/hermes/Dockerfile b/src/agent_env/agents/agents/hermes/Dockerfile index 5b143517f..597066f6d 100644 --- a/src/agent_env/agents/agents/hermes/Dockerfile +++ b/src/agent_env/agents/agents/hermes/Dockerfile @@ -1,3 +1,17 @@ +# The env's clock: the egress proxy, one static Go binary, and agent-env's libfaketime and its TLS shim, taken from +# agent-env's sources as clock/ and egress/ (shared_sources.toml) and built for this image's libc (Debian bookworm) in +# stages of their own, so no toolchain is left in the image. +FROM public.ecr.aws/docker/library/golang:1.26.11-bookworm@sha256:dc40a13b61304647ed7e5d581d246b86413dfc29d63c6d018310feb93cd98fc0 AS egress +COPY egress/ /egress/ +RUN cd /egress && go vet ./... && go test -count=1 ./... \ + && CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags=-s -o /out/egress . + +FROM public.ecr.aws/docker/library/python:3.11-slim@sha256:c20888b6acdd1e63e1c433a185bf3ad162c0288fe484616ce062e0d28add2900 AS clock +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential patch curl ca-certificates +COPY clock/ /clock/ +RUN sh /clock/build.sh /out + FROM public.ecr.aws/docker/library/python:3.11-slim@sha256:c20888b6acdd1e63e1c433a185bf3ad162c0288fe484616ce062e0d28add2900 # Hermes Agent at a pinned release, installed by its own installer, which the image checks against its checksum. The # installer brings Hermes' runtime (its Python, Node and ripgrep among them) under its home, /data/.hermes; git and xz @@ -23,12 +37,36 @@ WORKDIR /opt/agent COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY a2a_server.py hermes_bridge.py hermes_kit.py entrypoint.sh ./ +# Every clocked agent's egress rule, which the bridge imports, and the clock page's format it reads (shared_sources.toml). +COPY agent_clock/agent_egress.py clock/clockpage.py ./ COPY agentenv_plugin ./agentenv_plugin RUN chmod 0755 entrypoint.sh -# Each context's Hermes profile is under /data/.hermes, Hermes' home, whatever the uid: Modal runs the image as root. +# The env's clock: the follower and the egress proxy, each as its own user, under root's supervisord (root.conf), and +# agent-env's libfaketime and its TLS shim preloaded into every process, faking only the agent's (clock.conf). The +# follower's config is the agent's user's to write (agentenv_protocol.a2a_agent.clock) and the follower's to read. +RUN pip install --no-cache-dir supervisor==4.2.5 \ + && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-clock \ + && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-egress \ + && mkdir -p /usr/local/lib/agentenv-clock /usr/local/lib/agentenv-egress /etc/agentenv \ + && install -d -o agent -g agentenv-clock -m 2750 /etc/agentenv/clock \ + && touch /etc/agentenv/ca.pem \ + && cp /etc/ssl/certs/ca-certificates.crt /etc/agentenv/ca-bundle.pem \ + && cp /etc/ssl/certs/ca-certificates.crt /usr/local/lib/agentenv-egress/roots.pem +COPY --from=clock /out/libfaketimeMT.so.1 /out/libtls_realtime.so /usr/local/lib/agentenv-clock/ +COPY --from=egress /out/egress /usr/local/lib/agentenv-egress/egress +COPY clock/clockpage.py clock/clock-follower /usr/local/lib/agentenv-clock/ +COPY clock/clock.conf /etc/agentenv/clock.conf +COPY root.conf /etc/agentenv/root.conf +COPY start-agent /usr/local/bin/start-agent +RUN sed -i -e "s/^AGENTENV_SESSION_UID=.*/AGENTENV_SESSION_UID=$(id -u agent)/" \ + -e "s/^AGENTENV_CLOCK_PAGE_UID=.*/AGENTENV_CLOCK_PAGE_UID=$(id -u agentenv-clock)/" /etc/agentenv/clock.conf \ + && chmod 0755 /usr/local/lib/agentenv-clock/clock-follower /usr/local/bin/start-agent \ + && printf '%s\n' /usr/local/lib/agentenv-clock/libfaketimeMT.so.1 /usr/local/lib/agentenv-clock/libtls_realtime.so \ + > /etc/ld.so.preload +# Each context's Hermes profile is under /data/.hermes, Hermes' home, whatever the uid. ENV A2A_PORT=8000 PYTHONPATH=/opt/agent HOME=/data HERMES_ROOT=/data/.hermes -USER agent WORKDIR /app EXPOSE 8000 -# CMD rather than ENTRYPOINT: Modal passes a sandbox's env to the image's CMD only. -CMD ["/opt/agent/entrypoint.sh"] +# Starts as root, which start-agent needs for the clock; the A2A server and the CLI run as the agent's user. CMD rather +# than ENTRYPOINT: Modal passes a sandbox's env to the image's CMD only. +CMD ["/usr/local/bin/start-agent"] diff --git a/src/agent_env/agents/agents/hermes/hermes_bridge.py b/src/agent_env/agents/agents/hermes/hermes_bridge.py index acc5cd9ae..91f14f76a 100644 --- a/src/agent_env/agents/agents/hermes/hermes_bridge.py +++ b/src/agent_env/agents/agents/hermes/hermes_bridge.py @@ -27,6 +27,7 @@ from jsonschema import Draft202012Validator, SchemaError, ValidationError +from agent_egress import clock_armed, egress_env from hermes_kit import in_thread_to_its_end logger = logging.getLogger(__name__) @@ -104,6 +105,19 @@ def get_last_attribution_snapshot() -> dict[str, str | None]: return dict(_last_attribution) +def cli_env(environ: Mapping[str, str], home: Path, *, events_fd: int | None = None, + mcp_servers: dict[str, dict] | None = None, clock_armed: bool = False) -> dict[str, str]: + """The turn's process's environment: this process's, with the context's profile as Hermes' home and the event + pipe's descriptor for the plugin, and on an armed clock the egress proxy (``egress_env``) for the model endpoint and + the turn's MCP servers, which Hermes connects to itself.""" + env = {**environ, "HERMES_HOME": str(home)} + if events_fd is not None: + env[_EVENTS_FD_ENV] = str(events_fd) + urls = [environ.get("LITELLM_BASE_URL", ""), *(str(info.get("url") or "") for info in (mcp_servers or {}).values())] + env.update(egress_env(environ, urls, armed=clock_armed)) + return env + + def _attribution_headers(project_id: str | None, task_id: str | None) -> dict[str, str]: """LiteLLM cost-attribution headers, x-litellm-customer-id + x-litellm-tags; {} when nothing to attribute.""" headers: dict[str, str] = {} @@ -338,7 +352,7 @@ async def run_hermes( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=workspace, - env={**os.environ, "HERMES_HOME": str(home), _EVENTS_FD_ENV: str(write_end)}, + env=cli_env(os.environ, home, events_fd=write_end, mcp_servers=mcp_servers, clock_armed=clock_armed()), pass_fds=(write_end,), start_new_session=True, )) diff --git a/src/agent_env/agents/agents/hermes/root.conf b/src/agent_env/agents/agents/hermes/root.conf new file mode 100644 index 000000000..08f0c90c1 --- /dev/null +++ b/src/agent_env/agents/agents/hermes/root.conf @@ -0,0 +1,63 @@ +; Root's supervisord (start-agent, FD-5916): the clock follower and the egress proxy, each as its own user, on real +; time, and the A2A server as the agent's user. The server starts with the real-time marker, which agent-env's +; libfaketime takes out of its environment, so the server's own records keep real timestamps and everything it starts, +; the CLI and the model's commands, reads the env's time. Its socket and logs are root's alone, so the agent's user can +; neither stop nor restart any of them. The follower follows nothing until the server writes its config, which +; sync_env_clock has it do, then picks it up within a quarter second. +[supervisord] +nodaemon=true +user=root +logfile=/var/log/agentenv-root/supervisord.log +pidfile=/run/agentenv-root/supervisord.pid +childlogdir=/var/log/agentenv-root + +[unix_http_server] +file=/run/agentenv-root/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/agentenv-root/supervisor.sock + +[program:clock] +command=/usr/local/bin/python3 -I /usr/local/lib/agentenv-clock/clock-follower +user=agentenv-clock +directory=/ +priority=3 +autorestart=true +startsecs=1 +startretries=1000 +redirect_stderr=true +stdout_logfile=/var/log/agentenv-root/clock.log +environment=HOME="/nonexistent",USER="agentenv-clock",LOGNAME="agentenv-clock" + +[program:egress] +command=/usr/local/lib/agentenv-egress/egress serve +user=agentenv-egress +directory=/ +priority=3 +autorestart=true +startsecs=1 +startretries=1000 +redirect_stderr=true +stdout_logfile=/var/log/agentenv-root/egress.log +environment=HOME="/nonexistent",USER="agentenv-egress",LOGNAME="agentenv-egress",AGENTENV_EGRESS_PRIVATE="allow" + +; uvicorn itself, not entrypoint.sh: the marker holds for the process it's given to alone, and a shell that execs +; uvicorn would hand uvicorn an environment without it. HOME is Hermes' install's, /data, as the image sets it; its +; HERMES_ROOT and PYTHONPATH come from the image's environment. +[program:agent] +command=/usr/local/bin/python3 -m uvicorn a2a_server:app --host 0.0.0.0 --port %(ENV_A2A_PORT)s --timeout-graceful-shutdown 5 +user=agent +directory=/app +priority=10 +autorestart=true +stopasgroup=true +killasgroup=true +stopwaitsecs=30 +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +environment=HOME="/data",USER="agent",LOGNAME="agent",AGENTENV_REAL_TIME="1",AGENTENV_EGRESS_PROXY="http://127.0.0.1:3128",NODE_EXTRA_CA_CERTS="/etc/agentenv/ca.pem",SSL_CERT_FILE="/etc/agentenv/ca-bundle.pem",REQUESTS_CA_BUNDLE="/etc/agentenv/ca-bundle.pem" diff --git a/src/agent_env/agents/agents/hermes/shared_sources.toml b/src/agent_env/agents/agents/hermes/shared_sources.toml new file mode 100644 index 000000000..6076f3540 --- /dev/null +++ b/src/agent_env/agents/agents/hermes/shared_sources.toml @@ -0,0 +1,5 @@ +# What this image's build context takes from agent-env's own sources, at the name its Dockerfile copies it from: one +# copy of each, shared with the other images that take it (utils/docker/context.py). Paths are under src/agent_env. +clock = "utils/faketime_clock" +egress = "utils/egress_proxy" +agent_clock = "utils/agent_clock" diff --git a/src/agent_env/agents/agents/hermes/start-agent b/src/agent_env/agents/agents/hermes/start-agent new file mode 100755 index 000000000..b813e50d9 --- /dev/null +++ b/src/agent_env/agents/agents/hermes/start-agent @@ -0,0 +1,30 @@ +#!/bin/sh +# The image's start (FD-5916). As root, it makes what the agent's user must not touch, the clock page and file (the +# clock user's from their first byte) and the egress proxy's CA (its key the egress user's alone, trusted in the +# system store, agent-env's trust files and every certifi bundle), then starts root's supervisord (root.conf), which +# runs the clock follower and the egress proxy, each as its own user on real time, and the A2A server as the agent's +# user on real time, everything it starts reading the env's time from the page. Until sync_env_clock has the agent +# follow a clock, the page holds real time and the proxy relays TLS untouched. Started as another user, the image runs +# the A2A server alone, as before, on real time. +set -eu +if [ "$(id -u)" != 0 ]; then + exec /opt/agent/entrypoint.sh +fi +install -d -o root -g root -m 0700 /var/log/agentenv-root /run/agentenv-root +install -d -o agentenv-clock -g agentenv-clock -m 0755 /run/agentenv-clock +python3 -I /usr/local/lib/agentenv-clock/clockpage.py create agentenv-clock +AGENTENV_SESSION_USER=agent AGENTENV_EGRESS_NSS=off \ + /usr/local/lib/agentenv-egress/egress boot > /var/log/agentenv-root/egress-boot.log 2>&1 \ + || { cat /var/log/agentenv-root/egress-boot.log >&2; echo "the egress proxy's boot failed" >&2; exit 2; } +# A provider's own CA bundle stays trusted: Sail, for one, points SSL_CERT_FILE and the like at the bundle it signs +# model requests under, and root.conf points the server's trust at agent-env's bundles instead, so the provider's goes +# into them, and into the egress proxy's roots, which carry those requests once the clock is armed. +for bundle in "${SSL_CERT_FILE:-}" "${REQUESTS_CA_BUNDLE:-}" "${NODE_EXTRA_CA_CERTS:-}" "${CURL_CA_BUNDLE:-}"; do + case "$bundle" in ""|/etc/agentenv/*) continue ;; esac + [ -r "$bundle" ] || continue + for target in /etc/agentenv/ca-bundle.pem /etc/agentenv/ca.pem /usr/local/lib/agentenv-egress/roots.pem; do + grep -qxF "# agentenv: $bundle" "$target" 2>/dev/null && continue + { printf '\n# agentenv: %s\n' "$bundle"; cat "$bundle"; } >> "$target" + done +done +exec supervisord -c /etc/agentenv/root.conf diff --git a/tst/integration/task_step/test_agent_env_clock_local.py b/tst/integration/task_step/test_agent_env_clock_local.py index 87468d613..53dbdf54a 100644 --- a/tst/integration/task_step/test_agent_env_clock_local.py +++ b/tst/integration/task_step/test_agent_env_clock_local.py @@ -89,6 +89,7 @@ class Agent: AGENTS = { "claude-code": Agent(bridge="claude_bridge", version=("claude", "--version"), shows="(Claude Code)", turn=_claude_turn), + "hermes": Agent(bridge="hermes_bridge", version=("hermes", "--version"), shows="Hermes Agent v"), "opencode": Agent(bridge="opencode_bridge", version=("opencode", "--version"), shows="1.17.18"), } diff --git a/tst/unit/agents/hermes/agent.py b/tst/unit/agents/hermes/agent.py index 774e1061e..b0f11cb31 100644 --- a/tst/unit/agents/hermes/agent.py +++ b/tst/unit/agents/hermes/agent.py @@ -1,17 +1,23 @@ -"""The Hermes agent's modules, imported from its folder as its image runs them: by their own names, from the folder on -the path.""" +"""The Hermes agent's modules, imported as its image runs them: by their own names, from its folder and from the folders +its build context takes as agent_clock/ and clock/, whose modules the image copies beside the bridge.""" import importlib import importlib.util +import os import sys from pathlib import Path import agent_env +from tst.util.build_context import sent AGENT_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "hermes" -sys.path.insert(0, str(AGENT_DIR)) +IMPORTED = [str(AGENT_DIR), str(sent(AGENT_DIR, "agent_clock/agent_egress.py").parent), + str(sent(AGENT_DIR, "clock/clockpage.py").parent)] +PYTHONPATH = os.pathsep.join(IMPORTED) +sys.path[:0] = IMPORTED hermes_bridge = importlib.import_module("hermes_bridge") hermes_kit = importlib.import_module("hermes_kit") +agent_egress = importlib.import_module("agent_egress") # By its path: another agent's a2a_server, imported by the same name, may be loaded already. _spec = importlib.util.spec_from_file_location("hermes_a2a_server", AGENT_DIR / "a2a_server.py") a2a_server = importlib.util.module_from_spec(_spec) diff --git a/tst/unit/agents/hermes/test_card_contract.py b/tst/unit/agents/hermes/test_card_contract.py index e89f706eb..126efe93b 100644 --- a/tst/unit/agents/hermes/test_card_contract.py +++ b/tst/unit/agents/hermes/test_card_contract.py @@ -10,7 +10,7 @@ import pytest -from tst.unit.agents.hermes.agent import AGENT_DIR, hermes_bridge +from tst.unit.agents.hermes.agent import AGENT_DIR, PYTHONPATH, hermes_bridge def _methods(card: dict) -> dict[str, list[str]]: @@ -57,7 +57,8 @@ async def test_the_attribution_probe_reads_back_what_the_last_turn_sent(monkeypa def test_the_card_is_read_by_importing_agent_card(): """agent-env imports AGENT_CARD from the image with no agent environment, and parses its stdout as the card.""" script = "import json; from a2a_server import AGENT_CARD; print(json.dumps(AGENT_CARD.model_dump(mode='json')))" - env = {key: value for key, value in os.environ.items() if not key.startswith("LITELLM_")} + env = {**{key: value for key, value in os.environ.items() if not key.startswith("LITELLM_")}, + "PYTHONPATH": PYTHONPATH} out = subprocess.run([sys.executable, "-c", script], cwd=AGENT_DIR, env=env, capture_output=True, text=True, check=True) diff --git a/tst/unit/agents/hermes/test_env_clock.py b/tst/unit/agents/hermes/test_env_clock.py new file mode 100644 index 000000000..661a7b3e5 --- /dev/null +++ b/tst/unit/agents/hermes/test_env_clock.py @@ -0,0 +1,72 @@ +"""Hermes on the env's clock: its bridge hands the shared egress rule (tst/unit/agents/test_agent_egress.py) the model +endpoint's URL and every MCP server's, which Hermes connects to itself, so on an armed clock the turn's process goes out +through the image's egress proxy, and to loopback, the hosts NO_PROXY names and plain-HTTP hosts directly; unarmed, it +goes out as before. What every clocked image shares is checked in tst/unit/agents/test_agent_clock_images.py.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tst.unit.agents.hermes.agent import AGENT_DIR, agent_egress, hermes_bridge + +PROXY = "http://127.0.0.1:3128" + +HOME = Path("/data/.hermes/profiles/ctx-0") + + +def test_on_an_armed_clock_the_turn_goes_out_through_the_proxy(): + env = hermes_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY, "NO_PROXY": "registry.internal"}, HOME, + events_fd=7, clock_armed=True) + + assert {env[name] for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "https_proxy")} == {PROXY} + assert {"localhost", "host.docker.internal", "registry.internal"} <= set(env["NO_PROXY"].split(",")) + assert (env["HERMES_HOME"], env["AGENTENV_EVENTS_FD"]) == (str(HOME), "7") + + +@pytest.mark.parametrize("environ, armed", [({}, True), ({agent_egress.EGRESS_PROXY_ENV: PROXY}, False)]) +def test_without_the_proxy_or_an_armed_clock_the_turn_goes_out_directly_as_before(environ, armed): + env = hermes_bridge.cli_env(environ, HOME, events_fd=7, clock_armed=armed) + + assert env == {**environ, "HERMES_HOME": str(HOME), "AGENTENV_EVENTS_FD": "7"} + + +@pytest.mark.parametrize("base_url, direct", [("http://10.0.0.5:4000", True), ("https://litellm.example.com", False)]) +def test_a_model_endpoint_over_plain_http_is_reached_directly_on_an_armed_clock(base_url, direct): + env = hermes_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url}, HOME, + clock_armed=True) + assert (base_url.split("//")[1].split(":")[0] in env["NO_PROXY"].split(",")) is direct + + +def test_the_turns_plain_http_mcp_servers_are_reached_directly_and_https_ones_through_the_proxy_on_an_armed_clock(): + """The proxy checks an HTTPS server's certificate on real time, and reaches a private address in this image; a host + reached over HTTPS too stays behind it, as NO_PROXY goes by host.""" + servers = {"env": {"url": "http://gateway.internal:8080/mcp"}, "docs": {"url": "https://10.1.2.3/mcp"}, + "remote": {"url": "https://abc-123.w.modal.host/mcp"}, "both": {"url": "http://abc-123.w.modal.host/sse"}} + env = hermes_bridge.cli_env({agent_egress.EGRESS_PROXY_ENV: PROXY}, HOME, mcp_servers=servers, clock_armed=True) + + direct = set(env["NO_PROXY"].split(",")) + assert "gateway.internal" in direct and not {"10.1.2.3", "abc-123.w.modal.host"} & direct + + +def test_start_agent_keeps_a_providers_ca_bundle_trusted_by_the_server_and_the_proxy(tmp_path): + """Sail points SSL_CERT_FILE and the like at the bundle it signs model requests under; root.conf points the server + at agent-env's bundles, so start-agent adds the provider's to them and to the proxy's roots, once each.""" + start = (AGENT_DIR / "start-agent").read_text() + merge = start[start.index('for bundle in "${SSL_CERT_FILE'):start.index("exec supervisord")] + targets = {"/etc/agentenv/ca-bundle.pem": tmp_path / "ca-bundle.pem", "/etc/agentenv/ca.pem": tmp_path / "ca.pem", + "/usr/local/lib/agentenv-egress/roots.pem": tmp_path / "roots.pem"} + for real, fake in targets.items(): + fake.write_text("SYSTEM ROOTS\n") + merge = merge.replace(real, str(fake)) + sail = tmp_path / "sailbox-ca-bundle.crt" + sail.write_text("-----BEGIN CERTIFICATE-----\nSAIL\n-----END CERTIFICATE-----\n") + + subprocess.run(["sh", "-c", merge], check=True, env={"PATH": "/usr/bin:/bin", "SSL_CERT_FILE": str(sail), + "NODE_EXTRA_CA_CERTS": str(sail), "REQUESTS_CA_BUNDLE": "/etc/agentenv/ca-bundle.pem"}) + + for fake in targets.values(): + text = fake.read_text() + assert text.startswith("SYSTEM ROOTS\n") and text.count("SAIL") == 1, (fake.name, text)