diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 26e3133d9..87b6e07f1 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -190,15 +190,26 @@ def sources(names: list[str]) -> str: TOOL_DISABLE_ACTION = "disable" TOOL_ENABLE_ACTION = "enable" # The harness's routes that take the deploy's control token: the ones that change tool access, triggers and -# notifications. The gateway also takes it for the data plane and for an MCP server's own endpoint behind its proxy -# (Gateway._takes_control_token). Every other route stays open for now, the clock's writes included: callers outside -# agent-env set the clock over plain HTTP and don't send the token yet. +# notifications. The gateway also takes it, behind its proxy, for the data plane, for an MCP server's own endpoint and +# for the LEGACY_STATE_ROUTES (Gateway._takes_control_token). Every other route stays open for now, the clock's writes +# included: callers outside agent-env set the clock over plain HTTP and don't send the token yet. CONTROL_ROUTES = frozenset({ f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}", "/triggers/register", "/triggers/remove", "/triggers/clear", "/notifications/config", "/notifications/take", "/notifications/clear", CLOCK_FOLLOWERS_PATH, }) +# The older REST routes a backing server serves beside its data plane, which reset, load or export its state. +# Behind the proxy, /svc// takes the control token for each, and anything under it, on a gateway that has one. +# A site's /website route serves none of them, since a site with no frontend of its own serves its backend there and +# an older frontend passes them on. +LEGACY_STATE_ROUTES = ( + "/api/reset", "/api/add", "/api/bridge/apply", + "/export-state", "/export-snapshot", "/snapshot-dirty", + "/reset-memory", "/validate-rows", + "/api/_state", "/api/_reset", "/api/debug/seed", +) +SITE_LEGACY_STATE_ROUTES = frozenset(LEGACY_STATE_ROUTES) class GatewayMode(str, Enum): diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 32021167f..0bc3511da 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -73,6 +73,8 @@ WELL_KNOWN_PATH, WILDCARD, GatewayMode, + LEGACY_STATE_ROUTES, + SITE_LEGACY_STATE_ROUTES, tool_name_clashes_text, ) from .browser_proxy import PING_INTERVAL_S, BrowserRoutes, TokenRedaction @@ -641,8 +643,9 @@ async def app(scope, receive, send): def _takes_control_token(self, scope: dict) -> bool: """Whether a request takes the deploy's control token: a control route, the env's data plane, or one of - ``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``reaches``), - after dot-segment normalisation. A site's /website route never reaches its data plane.""" + ``token_paths`` behind /svc (a backing server's data plane, its MCP endpoint and its LEGACY_STATE_ROUTES), by + the path as routed or by the one the backing server receives (``reaches``), after dot-segment normalisation. + A site's /website route never reaches its data plane.""" path = scope["path"] if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): return True @@ -909,12 +912,13 @@ async def _proxy_website_request(self, request: Request) -> Response: """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are never reached through its route, so an agent browsing the site can't read its data or what grades it; agentenv - reaches them through /svc//.""" + reaches them through /svc//. On a gateway with a token, neither is SITE_LEGACY_STATE_ROUTES + (``_site_reserved``).""" app = request.path_params["service_name"] card = await self._site_card(app) if card is None: return self._json_response({"error": "website card unavailable"}, 503) - if reserved_site_request(request, card.reserved): + if reserved_site_request(request, self._site_reserved(card)): return self._json_response({"error": "not found"}, 404) response = await self._proxy_rest_request(request, url_map=self.website_urls, site_names=self._names(app, card)) engine = self._trigger_engine @@ -932,14 +936,14 @@ async def _proxy_website_request(self, request: Request) -> Response: async def _proxy_website_socket(self, websocket: WebSocket) -> None: """Relay a WebSocket on a site's route to its server, as its HTTP requests are: its data plane and card, and - every path its card advertises for agentenv (``site_reserved_paths``), are refused, and it goes upstream with - the role, role key and forwarded address an HTTP request would carry.""" + every path its card advertises for agentenv (``site_reserved_paths``), are refused, as ``_site_reserved`` + says, and it goes upstream with the role, role key and forwarded address an HTTP request would carry.""" service_name = websocket.path_params["service_name"] card = await self._site_card(service_name) if card is None: await refuse_websocket(websocket, self._json_response({"error": "website card unavailable"}, 503)) return - if reserved_site_request(websocket, card.reserved): + if reserved_site_request(websocket, self._site_reserved(card)): await refuse_websocket(websocket, self._json_response({"error": "not found"}, 404)) return base_url = self.website_urls.get(service_name) @@ -950,6 +954,11 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None: open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld, site_names=self._names(service_name, card)) + def _site_reserved(self, card: SiteCard) -> frozenset[str]: + """The paths a site's route never reaches: those its card advertises for agentenv, and on a gateway with a + token SITE_LEGACY_STATE_ROUTES, which an older site's frontend passes on to its backend.""" + return card.reserved | SITE_LEGACY_STATE_ROUTES if self._control_bearer else card.reserved + async def _fire_website_triggers(self, request: Request, app: str, status: int) -> tuple[list[asyncio.Task], float | None]: """Feed an answered website request to the trigger engine (fail-open).""" @@ -1881,10 +1890,11 @@ def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str] def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, - extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and - triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here.""" + extensions included, its LEGACY_STATE_ROUTES, and an MCP server's own endpoint (``mcp_endpoints``), which skips + the gateway's trajectory and triggers. A child env registered under its own name, a computer, takes its own token, + so none of its paths are here.""" endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls) - return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ())) + return {key: (RPC_PATH, *LEGACY_STATE_ROUTES, *((endpoints[key],) if key in endpoints else ())) for key, base in rest_proxy_urls.items() if key in website_urls or route_environment_name(key, base, servers, website=False) != key or not any(server.name == key for server in servers)} diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index a2d862105..9e2e17d52 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -397,9 +397,10 @@ def test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_thro # Checks each items server CONFIG names holds exactly the items loaded into it, read from its export-state route behind -# the multi's gateway. CONFIG goes above it. +# the multi's gateway, with the control token the run's grant gives. CONFIG goes above it. VERIFY_SEEDED = ''' import httpx +from agentenv_protocol.client import request_headers async def verify(mcp_url): @@ -407,7 +408,8 @@ async def verify(mcp_url): results = [] async with httpx.AsyncClient(timeout=30) as client: for server, items in CONFIG["servers"].items(): - response = await client.get(f"{gateway}/svc/mcp-{server}/export-state") + url = f"{gateway}/svc/mcp-{server}/export-state" + response = await client.get(url, headers=request_headers(url)) listed = response.json() if response.status_code == 200 else {} description = f"{server} holds what it was seeded with, {items}: {response.status_code} {listed}" results.append({"id": server, "result": listed.get("items") == items, "description": description}) diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 6a9a22771..b968aaa54 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -1,5 +1,6 @@ -"""The gateway's control routes, the data plane and an MCP server's own endpoint behind its proxy take the deploy's -control token when it has one; every other route stays open.""" +"""The gateway's control routes, and behind its proxy the data plane, an MCP server's own endpoint and the older REST +routes that reset, load or export state, take the deploy's control token when it has one; every other route stays +open.""" from __future__ import annotations import asyncio @@ -12,7 +13,7 @@ from agent_env.env.gateway import InternalMCPServer from agent_env.env.gateway import rest_proxy as rest_proxy_module -from agent_env.env.gateway.constants import CONTROL_ROUTES, WELL_KNOWN_PATH +from agent_env.env.gateway.constants import CONTROL_ROUTES, LEGACY_STATE_ROUTES, WELL_KNOWN_PATH from agent_env.env.gateway.gateway import Gateway TOKEN = "s3cret" @@ -27,17 +28,37 @@ "/clock/set-time": "PUT", "/clock/advance": "PUT", "/clock/clear": "POST"} -# Behind the proxy: the data plane, its extensions, each service's included, and an MCP server's own endpoint, however -# the path is spelt on its way to the gateway. +LEGACY = {"/api/reset": "POST", "/api/add": "POST", "/api/bridge/apply": "POST", "/export-state": "GET", + "/export-snapshot": "GET", "/snapshot-dirty": "GET", "/reset-memory": "POST", "/validate-rows": "POST", + "/api/_state": "GET", "/api/_reset": "POST", "/api/debug/seed": "POST"} + +# Behind the proxy: the data plane, its extensions, each service's included, an MCP server's own endpoint, and the +# older REST routes on an MCP server's route and a site's, however the path is spelt on its way to the gateway. GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"), ("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"), ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"), ("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"), - ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")] + ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP"), + *((method, f"/svc/{key}{path}") for key in ("mcp-svc", "shop") for path, method in LEGACY.items()), + ("GET", "/svc/mcp-svc/export-snapshot?stream=1"), ("POST", "/svc/shop/api/reset/x"), + ("GET", "/svc/mcp-svc/x/../export-state"), ("GET", "/svc/mcp-svc/%65xport-state"), + ("POST", "/svc/mcp-svc/public%2Fx/../api/reset"), ("POST", "/svc/shop/public%2Fx/../api/add"), + ("POST", "/svc/mcp-svc/API/Reset"), ("GET", "/svc/shop/Export-Snapshot"), ("POST", "/svc/mcp-svc/api//reset"), + ("POST", "/svc/mcp-svc/api/bridge/../bridge/apply"), ("GET", "/svc/shop/api/_state/"), + ("GET", "/svc/shop/API/_State"), ("GET", "/svc/shop/x/../api/_state"), ("GET", "/svc/shop/api/%5Fstate")] PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), - ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), - ("GET", "/website/shop/index.html")] + ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), ("GET", "/website/shop/index.html"), + ("GET", "/svc/mcp-svc/export-statex"), ("POST", "/svc/mcp-svc/api/resetx"), + ("POST", "/svc/shop/api/address"), ("GET", "/svc/mcp-svc/x/export-state"), + ("POST", "/svc/mcp-svc/api/bridge"), ("GET", "/website/shop/export-statex"), + ("GET", "/svc/shop/api/_statex"), ("GET", "/svc/shop/api/state"), ("GET", "/website/shop/api/state"), + ("GET", "/website/shop/api/_statex"), ("GET", "/svc/shop/api/debug/inspect")] +# A site's route never reaches these on a gateway with a token, however they are spelt, with the token or without. +SITE_REFUSED = [*(f"/website/shop{path}" for path in LEGACY), "/website/shop/api/reset/x", + "/website/shop/export-snapshot?stream=1", "/website/shop/x/../export-state", + "/website/shop/x/../api/add", "/website/shop/public%2Fx/../api/reset", + "/website/shop/x/../api/_state", "/website/shop/api/%5Fstate"] _RealAsyncClient = httpx.AsyncClient @@ -48,6 +69,7 @@ def _client(token: str | None) -> httpx.AsyncClient: def test_the_routes_under_test_are_the_ones_the_gateway_covers(): assert set(CONTROL) == CONTROL_ROUTES + assert set(LEGACY) == set(LEGACY_STATE_ROUTES) @pytest.mark.asyncio @@ -151,7 +173,7 @@ async def _proxied(token: str | None, method: str, path: str, headers: dict[str, "website_urls": {"shop": "http://shop-backend"}, **children}) url = urlsplit(path) scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method, - "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"", + "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": url.query.encode(), "root_path": "", "server": ("gateway", 80), "client": ("agent", 1), "headers": [(b"content-type", b"application/json"), *((name.lower().encode(), value.encode()) for name, value in headers.items())]} @@ -182,7 +204,8 @@ async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse caplog.set_level(logging.WARNING, logger="agent_env.env.gateway.gateway") assert await _proxied(TOKEN, method, path, headers) == 401 assert backing.paths == [] - assert [r.getMessage() for r in caplog.records] == [f"refused {method} {unquote(path)}: it takes the env's control token"] + assert [r.getMessage() for r in caplog.records] == [ + f"refused {method} {unquote(urlsplit(path).path)}: it takes the env's control token"] @pytest.mark.asyncio @@ -208,6 +231,23 @@ async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy assert await _proxied(None, method, path, {}) != 401 +@pytest.mark.asyncio +@pytest.mark.parametrize("path", SITE_REFUSED) +@pytest.mark.parametrize("headers", [{}, {"Authorization": f"Bearer {TOKEN}"}], ids=["none", "token"]) +async def test_a_sites_route_never_reaches_its_reset_or_add_on_a_gateway_with_a_token(monkeypatch, path, headers): + backing = _Backing(monkeypatch) + assert await _proxied(TOKEN, "POST", path, headers) == 404 + assert [at for at in backing.paths if at != WELL_KNOWN_PATH] == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", SITE_REFUSED) +async def test_a_sites_route_passes_its_reset_and_add_on_a_gateway_without_a_token(monkeypatch, path): + backing = _Backing(monkeypatch) + assert await _proxied(None, "POST", path, {}) == 200 + assert [at for at in backing.paths if at != WELL_KNOWN_PATH] + + @pytest.mark.asyncio @pytest.mark.parametrize("method, path", [("POST", "/svc/mcp-x/rpc"), ("GET", "/svc/mcp-x/rpc/"), ("POST", "/svc/mcp-x/agentenv")]) async def test_an_mcp_server_whose_endpoint_is_elsewhere_is_guarded_at_its_endpoint(monkeypatch, method, path):