From 3e1fb06b513c6bad17c6f4a6ead095e40c47ef81 Mon Sep 17 00:00:00 2001 From: "scale-avr[bot]" <340582274+scale-avr[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:15:24 +0000 Subject: [PATCH 1/2] fix(gateway): take the control token on the older REST data routes behind /svc/mcp-/ --- src/agent_env/env/envs/_deployment.py | 4 +- src/agent_env/env/envs/mcp_server.py | 3 +- src/agent_env/env/envs/website.py | 5 ++- src/agent_env/env/gateway/gateway.py | 12 +++++- src/agent_env/env/legacy_protocol.py | 41 ++++++++++++------- .../task_step/task_steps/snapshot_env.py | 6 ++- tst/unit/env/gateway/control_token_test.py | 10 ++++- tst/unit/env/test_legacy_protocol.py | 40 +++++++++++++++++- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 426dfa45c..dafc14948 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -174,8 +174,8 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo return False else: try: - await legacy_protocol.reset_via_rest(legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp), - timeout=60) + legacy_url = legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp) + await legacy_protocol.reset_via_rest(legacy_url, timeout=60, headers=data_plane_headers(env, legacy_url)) except httpx.HTTPStatusError as e: if e.response.status_code != 404: raise diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 44f5c02dd..64dd3dc58 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -296,7 +296,8 @@ async def _exec_in_server(self, *command: str) -> None: async def _load_environment_artifact_legacy(self, file_artifact) -> None: container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename) await legacy_protocol.reset( - self._gateway_url, self.environment_name, container_path, max_retries=self._MCP_MAX_RETRIES + self._gateway_url, self.environment_name, container_path, max_retries=self._MCP_MAX_RETRIES, + deployed=self._deployed, ) async def validate(self, on_progress: Optional[Callable[[str], None]] = None) -> str: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index b7a01c7d2..b7ddb3958 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -371,9 +371,10 @@ async def _load_environment_artifact_legacy(self, file_artifact) -> None: container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename) base_url = legacy_protocol.environment_base_url(self._gateway_url, self.environment_name, mcp=False) from agent_env.env.gateway.constants import DATA_PLANE_LOAD_TIMEOUT_S - await legacy_protocol.reset_via_rest(base_url, timeout=60) + headers = data_plane_headers(self, base_url) + await legacy_protocol.reset_via_rest(base_url, timeout=60, headers=headers) try: - await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S) + await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers) except Exception as e: logger.error( "load_environment_artifact: /api/add failed after /api/reset succeeded; " diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 5c42fdb82..6968bfdab 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -1852,13 +1852,21 @@ def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str] return endpoints +# The older REST routes a BaseService server serves beside its MCP endpoint, behind the same /svc/mcp-/ proxy: +# they reset, export and validate the state the verifiers grade, as the data plane's do, so they take the token too. +LEGACY_DATA_PATHS = ("/api/reset", "/export-state", "/export-snapshot", "/snapshot-dirty", "/reset-memory", + "/validate-rows") + + def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and - triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here.""" + triggers, and for an ``mcp-`` route the older REST routes (``LEGACY_DATA_PATHS``). A child env registered under its + own name, a computer, takes its own token, so none of its paths are here.""" endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls) - return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ())) + return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ()), + *(LEGACY_DATA_PATHS if key.startswith("mcp-") and key not in website_urls else ())) for key, base in rest_proxy_urls.items() if key in website_urls or route_environment_name(key, base, servers, website=False) != key or not any(server.name == key for server in servers)} diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 18d78e234..585286884 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -209,21 +209,24 @@ async def reset_via_rest( mock_data_path: Optional[str] = None, timeout: int = 30, verify: bool = True, + headers: Optional[dict[str, str]] = None, ) -> dict: - """POST ``{base_url}/api/reset``. Sends the seed path when given. Raises on non-2xx.""" + """POST ``{base_url}/api/reset``. Sends the seed path when given, and ``headers`` (the deployment's control token, + which a gateway that has one takes on this route). Raises on non-2xx.""" body = {"mock_data_path": mock_data_path} if mock_data_path is not None else None async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout) + response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout, headers=headers) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: REST reset succeeded: {result}") return result -async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True) -> dict: - """POST ``{base_url}/api/add`` with a backend file path. Raises on non-2xx.""" +async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True, + headers: Optional[dict[str, str]] = None) -> dict: + """POST ``{base_url}/api/add`` with a backend file path, and ``headers``. Raises on non-2xx.""" async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout) + response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout, headers=headers) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: add response: {result}") @@ -231,19 +234,25 @@ async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify async def export_state( - gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True + gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True, + headers: Optional[dict[str, str]] = None, ) -> dict: - """GET ``/svc/mcp-{name}/export-state`` — the legacy state snapshot for one service.""" - return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify) + """GET ``/svc/mcp-{name}/export-state`` — the legacy state snapshot for one service, with ``headers``.""" + return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify, headers) -async def _export_state_at(base_url: str, timeout: int, verify: bool) -> dict: +async def _export_state_at(base_url: str, timeout: int, verify: bool, headers: Optional[dict[str, str]] = None) -> dict: async with httpx.AsyncClient(verify=verify) as client: - response = await client.get(f"{base_url}/export-state", timeout=timeout) + response = await client.get(f"{base_url}/export-state", timeout=timeout, headers=headers) response.raise_for_status() return response.json() +def _control_headers(deployed: Optional[DeployedEnv], url: str) -> Optional[dict[str, str]]: + """The headers a call to ``url`` sends of ``deployed``'s control token; None when it has none to send.""" + return (deployed.control_headers_for(url) or None) if deployed is not None else None + + async def service_state(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str) -> dict: """An MCP service's state as JSON: its v1 ``data/get`` answer when that is data, asked with the deployment's control token, else ``GET /export-state``. @@ -253,14 +262,16 @@ async def service_state(deployed: Optional[DeployedEnv], gateway_url: Optional[s """ base_url = await v1_base_url(deployed, gateway_url, environment_name, mcp=True) if base_url is None: - return await export_state(gateway_url, environment_name) + return await export_state(gateway_url, environment_name, headers=_control_headers(deployed, environment_base_url( + gateway_url or "", environment_name, mcp=True))) response = await protocol_v1.get_data(base_url, headers=deployed.control_headers_for(base_url) if deployed is not None else None) if not response.parts: return {} if isinstance(response.parts[0], DataPart): return response.parts[0].data - return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True) + return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True, + headers=_control_headers(deployed, base_url)) async def offers_tool(mcp_url: str, name: str) -> bool: @@ -322,11 +333,13 @@ async def reset( environment_name: str, mock_data_path: str, max_retries: int = DEFAULT_MCP_MAX_RETRIES, + deployed: Optional[DeployedEnv] = None, ) -> Optional[dict]: - """Reset an MCP service to a seed: try REST ``/api/reset``, fall back to the MCP tool.""" + """Reset an MCP service to a seed: try REST ``/api/reset`` (with ``deployed``'s control token), fall back to the + MCP tool.""" base_url = environment_base_url(gateway_url, environment_name, mcp=True) try: - return await reset_via_rest(base_url, mock_data_path) + return await reset_via_rest(base_url, mock_data_path, headers=_control_headers(deployed, base_url)) except Exception as rest_err: # TODO remove MCP tool fallback once all usages of MCP servers are on REST logger.info(f"load_environment_artifact: REST reset failed ({type(rest_err).__name__}), falling back to MCP tool") diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index d42d70943..4959d108a 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -585,6 +585,7 @@ async def _export_environment_to_file( async with client.stream( "GET", uri, + headers=headers, timeout=httpx.Timeout( timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS ), @@ -599,11 +600,14 @@ async def _export_environment_to_file( json.dump(state, f, default=str) return ".json" - legacy_url = f"{legacy_protocol.environment_base_url(gateway_url, environment_name, mcp=True)}/export-state" + legacy_base = legacy_protocol.environment_base_url(gateway_url, environment_name, mcp=True) + legacy_url = f"{legacy_base}/export-state" + legacy_headers = (deployed.control_headers_for(legacy_base) or None) if deployed is not None else None async with httpx.AsyncClient() as client: async with client.stream( "GET", legacy_url, + headers=legacy_headers, timeout=httpx.Timeout(timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS), ) as resp: resp.raise_for_status() diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 6a9a22771..66f85a9c0 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -34,9 +34,15 @@ ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"), ("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"), - ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")] + ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP"), + # the older REST routes of a BaseService server, which reset and export what the verifiers grade + ("POST", "/svc/mcp-svc/api/reset"), ("GET", "/svc/mcp-svc/./export-state"), ("GET", "/svc/mcp-svc/export-state"), + ("GET", "/svc/mcp-svc/export-snapshot"), ("GET", "/svc/mcp-svc/x/../export-snapshot"), + ("GET", "/svc/mcp-svc/snapshot-dirty"), ("POST", "/svc/mcp-svc/reset-memory"), + ("POST", "/svc/mcp-svc/validate-rows"), ("POST", "/svc/mcp-svc/API/Reset"), + ("POST", "/svc/mcp-svc/%61pi/reset")] PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), - ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), + ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/api/reset"), ("POST", "/svc/shop/mcp"), ("GET", "/website/shop/index.html")] _RealAsyncClient = httpx.AsyncClient diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index d426e3259..af619aa31 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -44,7 +44,7 @@ async def test_reset_uses_rest_on_success(monkeypatch): result = await legacy_protocol.reset("http://gw", "email", "/data/seed.json") assert result == {"ok": True} - rest.assert_awaited_once_with("http://gw/svc/mcp-email", "/data/seed.json") + rest.assert_awaited_once_with("http://gw/svc/mcp-email", "/data/seed.json", headers=None) mcp.assert_not_called() @@ -345,3 +345,41 @@ async def test_backing_servers_use_a_records_computers_only_for_its_own_gateway( servers = await legacy_protocol.backing_servers(record, "http://another-gw", SimpleNamespace(mcp_server_envs=[], website_envs=[])) assert [(s.environment_name, s.address, s.record) for s in servers] == [("desktop", "http://another-gw", None)] + + +@pytest.mark.asyncio +async def test_the_legacy_rest_routes_send_the_deployments_control_token(monkeypatch): + seen, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + seen.append((request.method, request.url.path, request.headers.get("authorization"))) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + deployed = DeployedSandboxEnv(env_id="e", env_version=1, sandbox_id="sb-1", + environment_card_url=f"http://gw{WELL_KNOWN_PATH}", control_token="t0ken") + base = "http://gw/svc/mcp-email" + + assert await legacy_protocol.reset("http://gw", "email", "/data/seed.json", deployed=deployed) == {"ok": True} + headers = deployed.control_headers_for(base) + await legacy_protocol.add_via_rest(base, "/data/seed.json", headers=headers) + await legacy_protocol.export_state("http://gw", "email", headers=headers) + assert seen == [("POST", "/svc/mcp-email/api/reset", "Bearer t0ken"), ("POST", "/svc/mcp-email/api/add", "Bearer t0ken"), + ("GET", "/svc/mcp-email/export-state", "Bearer t0ken")] + + +@pytest.mark.asyncio +async def test_a_legacy_service_state_is_exported_with_the_deployments_control_token(monkeypatch): + seen, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + seen.append((request.url.path, request.headers.get("authorization"))) + if request.url.path.endswith(WELL_KNOWN_PATH): + return httpx.Response(404) + return httpx.Response(200, json={"emails": []}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + deployed = DeployedSandboxEnv(env_id="e", env_version=1, sandbox_id="sb-1", control_token="t0ken", + environment_card_url=f"http://gw{WELL_KNOWN_PATH}") + assert await legacy_protocol.service_state(deployed, "http://gw", "email") == {"emails": []} + assert seen[-1] == ("/svc/mcp-email/export-state", "Bearer t0ken") From 3699dc868421a32fc041331032b8c816e4bcde9f Mon Sep 17 00:00:00 2001 From: "scale-avr[bot]" <340582274+scale-avr[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:58:11 +0000 Subject: [PATCH 2/2] fix(gateway): send the granted control token on REST reset, add and export-state calls --- src/agent_env/env/legacy_protocol.py | 9 ++++++--- .../env/envs/multi_env_local_test.py | 4 ++-- tst/unit/env/test_legacy_protocol.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 585286884..24e54d6a3 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -215,7 +215,8 @@ async def reset_via_rest( which a gateway that has one takes on this route). Raises on non-2xx.""" body = {"mock_data_path": mock_data_path} if mock_data_path is not None else None async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout, headers=headers) + response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout, + headers=protocol_v1.request_headers(f"{base_url}/api/reset", headers)) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: REST reset succeeded: {result}") @@ -226,7 +227,8 @@ async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify headers: Optional[dict[str, str]] = None) -> dict: """POST ``{base_url}/api/add`` with a backend file path, and ``headers``. Raises on non-2xx.""" async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout, headers=headers) + response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout, + headers=protocol_v1.request_headers(f"{base_url}/api/add", headers)) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: add response: {result}") @@ -243,7 +245,8 @@ async def export_state( async def _export_state_at(base_url: str, timeout: int, verify: bool, headers: Optional[dict[str, str]] = None) -> dict: async with httpx.AsyncClient(verify=verify) as client: - response = await client.get(f"{base_url}/export-state", timeout=timeout, headers=headers) + response = await client.get(f"{base_url}/export-state", timeout=timeout, + headers=protocol_v1.request_headers(f"{base_url}/export-state", headers)) response.raise_for_status() return response.json() diff --git a/tst/integration/env/envs/multi_env_local_test.py b/tst/integration/env/envs/multi_env_local_test.py index 50c369c60..06385a0e4 100644 --- a/tst/integration/env/envs/multi_env_local_test.py +++ b/tst/integration/env/envs/multi_env_local_test.py @@ -55,13 +55,13 @@ async def test_a_multi_env_deploys_restores_loads_and_tears_down_on_local_compos base_url = await legacy_protocol.v1_base_url(deployed, deployed.gateway_url, "items") assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} slack_state = f"{legacy_protocol.environment_base_url(deployed.gateway_url, 'slack')}/export-state" - assert len(httpx.get(slack_state, timeout=30).json()["channels"]) == 10 + assert len(httpx.get(slack_state, timeout=30, headers=deployed.control_headers_for(slack_state)).json()["channels"]) == 10 # A multi env without computers resets its servers, each as it resets alone, and fails if one doesn't. await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( TaskStepContext(deployed_envs=[deployed])) assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": []} - assert httpx.get(slack_state, timeout=30).json()["channels"] == [] + assert httpx.get(slack_state, timeout=30, headers=deployed.control_headers_for(slack_state)).json()["channels"] == [] await restored.close() assert _stack(work_dir) == "" diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index af619aa31..0dae2f7ac 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -383,3 +383,21 @@ def handle(request: httpx.Request) -> httpx.Response: environment_card_url=f"http://gw{WELL_KNOWN_PATH}") assert await legacy_protocol.service_state(deployed, "http://gw", "email") == {"emails": []} assert seen[-1] == ("/svc/mcp-email/export-state", "Bearer t0ken") + + +@pytest.mark.asyncio +async def test_a_rest_call_without_headers_takes_the_token_its_env_was_granted(): + from agentenv_protocol import client as protocol_v1 + resp = MagicMock(); resp.raise_for_status = MagicMock(); resp.json = MagicMock(return_value={"ok": True}) + client = _fake_client(resp) + client.get = AsyncMock(return_value=resp) + granted = {"X-Control-Token": "placeholder"} + with patch("agent_env.env.legacy_protocol.httpx.AsyncClient", return_value=client): + with protocol_v1.authorized({"http://gw": granted}): + await legacy_protocol.reset_via_rest("http://gw/svc/mcp-email", "/data/seed.json") + await legacy_protocol.export_state("http://gw", "email") + await legacy_protocol.reset_via_rest("http://gw/svc/mcp-email") + + assert client.post.call_args_list[0].kwargs["headers"] == granted + assert client.get.call_args.kwargs["headers"] == granted + assert not client.post.call_args_list[1].kwargs["headers"]