Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions src/agent_env/env/gateway/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,15 +190,26 @@ def sources(names: list[str]) -> str:
TOOL_DISABLE_ACTION = "disable"
TOOL_ENABLE_ACTION = "enable"
# The harness's routes that take the deploy's control token: the ones that change tool access, triggers and
# notifications. The gateway also takes it for the data plane and for an MCP server's own endpoint behind its proxy
# (Gateway._takes_control_token). Every other route stays open for now, the clock's writes included: callers outside
# agent-env set the clock over plain HTTP and don't send the token yet.
# notifications. The gateway also takes it, behind its proxy, for the data plane, for an MCP server's own endpoint and
# for the LEGACY_STATE_ROUTES (Gateway._takes_control_token). Every other route stays open for now, the clock's writes
# included: callers outside agent-env set the clock over plain HTTP and don't send the token yet.
CONTROL_ROUTES = frozenset({
f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}",
"/triggers/register", "/triggers/remove", "/triggers/clear",
"/notifications/config", "/notifications/take", "/notifications/clear",
CLOCK_FOLLOWERS_PATH,
})
# The older REST routes a backing server serves beside its data plane, which reset, load or export its state.
# Behind the proxy, /svc/<key>/ takes the control token for each, and anything under it, on a gateway that has one.
# A site's /website route serves none of them, since a site with no frontend of its own serves its backend there and
# an older frontend passes them on.
LEGACY_STATE_ROUTES = (
"/api/reset", "/api/add", "/api/bridge/apply",
"/export-state", "/export-snapshot", "/snapshot-dirty",
"/reset-memory", "/validate-rows",
"/api/_state", "/api/_reset", "/api/debug/seed",
)
SITE_LEGACY_STATE_ROUTES = frozenset(LEGACY_STATE_ROUTES)


class GatewayMode(str, Enum):
Expand Down
30 changes: 20 additions & 10 deletions src/agent_env/env/gateway/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@
WELL_KNOWN_PATH,
WILDCARD,
GatewayMode,
LEGACY_STATE_ROUTES,
SITE_LEGACY_STATE_ROUTES,
tool_name_clashes_text,
)
from .browser_proxy import PING_INTERVAL_S, BrowserRoutes, TokenRedaction
Expand Down Expand Up @@ -641,8 +643,9 @@ async def app(scope, receive, send):

def _takes_control_token(self, scope: dict) -> bool:
"""Whether a request takes the deploy's control token: a control route, the env's data plane, or one of
``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``reaches``),
after dot-segment normalisation. A site's /website route never reaches its data plane."""
``token_paths`` behind /svc (a backing server's data plane, its MCP endpoint and its LEGACY_STATE_ROUTES), by
the path as routed or by the one the backing server receives (``reaches``), after dot-segment normalisation.
A site's /website route never reaches its data plane."""
path = scope["path"]
if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH):
return True
Expand Down Expand Up @@ -909,12 +912,13 @@ async def _proxy_website_request(self, request: Request) -> Response:
"""Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers.
A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are
never reached through its route, so an agent browsing the site can't read its data or what grades it; agentenv
reaches them through /svc/<name>/."""
reaches them through /svc/<name>/. On a gateway with a token, neither is SITE_LEGACY_STATE_ROUTES
(``_site_reserved``)."""
app = request.path_params["service_name"]
card = await self._site_card(app)
if card is None:
return self._json_response({"error": "website card unavailable"}, 503)
if reserved_site_request(request, card.reserved):
if reserved_site_request(request, self._site_reserved(card)):
return self._json_response({"error": "not found"}, 404)
response = await self._proxy_rest_request(request, url_map=self.website_urls, site_names=self._names(app, card))
engine = self._trigger_engine
Expand All @@ -932,14 +936,14 @@ async def _proxy_website_request(self, request: Request) -> Response:

async def _proxy_website_socket(self, websocket: WebSocket) -> None:
"""Relay a WebSocket on a site's route to its server, as its HTTP requests are: its data plane and card, and
every path its card advertises for agentenv (``site_reserved_paths``), are refused, and it goes upstream with
the role, role key and forwarded address an HTTP request would carry."""
every path its card advertises for agentenv (``site_reserved_paths``), are refused, as ``_site_reserved``
says, and it goes upstream with the role, role key and forwarded address an HTTP request would carry."""
service_name = websocket.path_params["service_name"]
card = await self._site_card(service_name)
if card is None:
await refuse_websocket(websocket, self._json_response({"error": "website card unavailable"}, 503))
return
if reserved_site_request(websocket, card.reserved):
if reserved_site_request(websocket, self._site_reserved(card)):
await refuse_websocket(websocket, self._json_response({"error": "not found"}, 404))
return
base_url = self.website_urls.get(service_name)
Expand All @@ -950,6 +954,11 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None:
open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld,
site_names=self._names(service_name, card))

def _site_reserved(self, card: SiteCard) -> frozenset[str]:
"""The paths a site's route never reaches: those its card advertises for agentenv, and on a gateway with a
token SITE_LEGACY_STATE_ROUTES, which an older site's frontend passes on to its backend."""
return card.reserved | SITE_LEGACY_STATE_ROUTES if self._control_bearer else card.reserved

async def _fire_website_triggers(self, request: Request, app: str,
status: int) -> tuple[list[asyncio.Task], float | None]:
"""Feed an answered website request to the trigger engine (fail-open)."""
Expand Down Expand Up @@ -1881,10 +1890,11 @@ def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str]
def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str],
rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]:
"""The paths under /svc/<key>/ that take the gateway's control token, by key: every backing server's data plane,
extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and
triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here."""
extensions included, its LEGACY_STATE_ROUTES, and an MCP server's own endpoint (``mcp_endpoints``), which skips
the gateway's trajectory and triggers. A child env registered under its own name, a computer, takes its own token,
so none of its paths are here."""
endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls)
return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ()))
return {key: (RPC_PATH, *LEGACY_STATE_ROUTES, *((endpoints[key],) if key in endpoints else ()))
for key, base in rest_proxy_urls.items()
if key in website_urls or route_environment_name(key, base, servers, website=False) != key
or not any(server.name == key for server in servers)}
Expand Down
6 changes: 4 additions & 2 deletions tst/integration/cli/run_bundle_built_envs_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,17 +397,19 @@ def test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_thro


# Checks each items server CONFIG names holds exactly the items loaded into it, read from its export-state route behind
# the multi's gateway. CONFIG goes above it.
# the multi's gateway, with the control token the run's grant gives. CONFIG goes above it.
VERIFY_SEEDED = '''
import httpx
from agentenv_protocol.client import request_headers


async def verify(mcp_url):
gateway = mcp_url.rsplit("/mcp", 1)[0]
results = []
async with httpx.AsyncClient(timeout=30) as client:
for server, items in CONFIG["servers"].items():
response = await client.get(f"{gateway}/svc/mcp-{server}/export-state")
url = f"{gateway}/svc/mcp-{server}/export-state"
response = await client.get(url, headers=request_headers(url))
listed = response.json() if response.status_code == 200 else {}
description = f"{server} holds what it was seeded with, {items}: {response.status_code} {listed}"
results.append({"id": server, "result": listed.get("items") == items, "description": description})
Expand Down
60 changes: 50 additions & 10 deletions tst/unit/env/gateway/control_token_test.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""The gateway's control routes, the data plane and an MCP server's own endpoint behind its proxy take the deploy's
control token when it has one; every other route stays open."""
"""The gateway's control routes, and behind its proxy the data plane, an MCP server's own endpoint and the older REST
routes that reset, load or export state, take the deploy's control token when it has one; every other route stays
open."""
from __future__ import annotations

import asyncio
Expand All @@ -12,7 +13,7 @@

from agent_env.env.gateway import InternalMCPServer
from agent_env.env.gateway import rest_proxy as rest_proxy_module
from agent_env.env.gateway.constants import CONTROL_ROUTES, WELL_KNOWN_PATH
from agent_env.env.gateway.constants import CONTROL_ROUTES, LEGACY_STATE_ROUTES, WELL_KNOWN_PATH
from agent_env.env.gateway.gateway import Gateway

TOKEN = "s3cret"
Expand All @@ -27,17 +28,37 @@
"/clock/set-time": "PUT", "/clock/advance": "PUT", "/clock/clear": "POST"}


# Behind the proxy: the data plane, its extensions, each service's included, and an MCP server's own endpoint, however
# the path is spelt on its way to the gateway.
LEGACY = {"/api/reset": "POST", "/api/add": "POST", "/api/bridge/apply": "POST", "/export-state": "GET",
"/export-snapshot": "GET", "/snapshot-dirty": "GET", "/reset-memory": "POST", "/validate-rows": "POST",
"/api/_state": "GET", "/api/_reset": "POST", "/api/debug/seed": "POST"}

# Behind the proxy: the data plane, its extensions, each service's included, an MCP server's own endpoint, and the
# older REST routes on an MCP server's route and a site's, however the path is spelt on its way to the gateway.
GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"),
("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"),
("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"),
("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"),
("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"),
("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")]
("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP"),
*((method, f"/svc/{key}{path}") for key in ("mcp-svc", "shop") for path, method in LEGACY.items()),
("GET", "/svc/mcp-svc/export-snapshot?stream=1"), ("POST", "/svc/shop/api/reset/x"),
("GET", "/svc/mcp-svc/x/../export-state"), ("GET", "/svc/mcp-svc/%65xport-state"),
("POST", "/svc/mcp-svc/public%2Fx/../api/reset"), ("POST", "/svc/shop/public%2Fx/../api/add"),
("POST", "/svc/mcp-svc/API/Reset"), ("GET", "/svc/shop/Export-Snapshot"), ("POST", "/svc/mcp-svc/api//reset"),
("POST", "/svc/mcp-svc/api/bridge/../bridge/apply"), ("GET", "/svc/shop/api/_state/"),
("GET", "/svc/shop/API/_State"), ("GET", "/svc/shop/x/../api/_state"), ("GET", "/svc/shop/api/%5Fstate")]
PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"),
("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"),
("GET", "/website/shop/index.html")]
("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), ("GET", "/website/shop/index.html"),
("GET", "/svc/mcp-svc/export-statex"), ("POST", "/svc/mcp-svc/api/resetx"),
("POST", "/svc/shop/api/address"), ("GET", "/svc/mcp-svc/x/export-state"),
("POST", "/svc/mcp-svc/api/bridge"), ("GET", "/website/shop/export-statex"),
("GET", "/svc/shop/api/_statex"), ("GET", "/svc/shop/api/state"), ("GET", "/website/shop/api/state"),
("GET", "/website/shop/api/_statex"), ("GET", "/svc/shop/api/debug/inspect")]
# A site's route never reaches these on a gateway with a token, however they are spelt, with the token or without.
SITE_REFUSED = [*(f"/website/shop{path}" for path in LEGACY), "/website/shop/api/reset/x",
"/website/shop/export-snapshot?stream=1", "/website/shop/x/../export-state",
"/website/shop/x/../api/add", "/website/shop/public%2Fx/../api/reset",
"/website/shop/x/../api/_state", "/website/shop/api/%5Fstate"]
_RealAsyncClient = httpx.AsyncClient


Expand All @@ -48,6 +69,7 @@ def _client(token: str | None) -> httpx.AsyncClient:

def test_the_routes_under_test_are_the_ones_the_gateway_covers():
assert set(CONTROL) == CONTROL_ROUTES
assert set(LEGACY) == set(LEGACY_STATE_ROUTES)


@pytest.mark.asyncio
Expand Down Expand Up @@ -151,7 +173,7 @@ async def _proxied(token: str | None, method: str, path: str, headers: dict[str,
"website_urls": {"shop": "http://shop-backend"}, **children})
url = urlsplit(path)
scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method,
"scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"",
"scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": url.query.encode(),
"root_path": "", "server": ("gateway", 80), "client": ("agent", 1),
"headers": [(b"content-type", b"application/json"),
*((name.lower().encode(), value.encode()) for name, value in headers.items())]}
Expand Down Expand Up @@ -182,7 +204,8 @@ async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse
caplog.set_level(logging.WARNING, logger="agent_env.env.gateway.gateway")
assert await _proxied(TOKEN, method, path, headers) == 401
assert backing.paths == []
assert [r.getMessage() for r in caplog.records] == [f"refused {method} {unquote(path)}: it takes the env's control token"]
assert [r.getMessage() for r in caplog.records] == [
f"refused {method} {unquote(urlsplit(path).path)}: it takes the env's control token"]


@pytest.mark.asyncio
Expand All @@ -208,6 +231,23 @@ async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy
assert await _proxied(None, method, path, {}) != 401


@pytest.mark.asyncio
@pytest.mark.parametrize("path", SITE_REFUSED)
@pytest.mark.parametrize("headers", [{}, {"Authorization": f"Bearer {TOKEN}"}], ids=["none", "token"])
async def test_a_sites_route_never_reaches_its_reset_or_add_on_a_gateway_with_a_token(monkeypatch, path, headers):
backing = _Backing(monkeypatch)
assert await _proxied(TOKEN, "POST", path, headers) == 404
assert [at for at in backing.paths if at != WELL_KNOWN_PATH] == []


@pytest.mark.asyncio
@pytest.mark.parametrize("path", SITE_REFUSED)
async def test_a_sites_route_passes_its_reset_and_add_on_a_gateway_without_a_token(monkeypatch, path):
backing = _Backing(monkeypatch)
assert await _proxied(None, "POST", path, {}) == 200
assert [at for at in backing.paths if at != WELL_KNOWN_PATH]


@pytest.mark.asyncio
@pytest.mark.parametrize("method, path", [("POST", "/svc/mcp-x/rpc"), ("GET", "/svc/mcp-x/rpc/"), ("POST", "/svc/mcp-x/agentenv")])
async def test_an_mcp_server_whose_endpoint_is_elsewhere_is_guarded_at_its_endpoint(monkeypatch, method, path):
Expand Down
Loading