Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions packages/agentenv-protocol/src/agentenv_protocol/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,28 @@ def request_headers(url: str, headers: dict[str, str] | None = None) -> dict[str

def granted_headers(url: str, grants: dict[str | None, dict[str, str]]) -> dict[str, str]:
"""What ``grants``, env addresses and the headers that carry their control tokens, give a call to ``url``: those of
the address it is or is under, and none anywhere else."""
return next((sent for base, sent in grants.items()
if base and (url == base.rstrip("/") or url.startswith(f"{base.rstrip('/')}/"))), {})
the address it resolves to or under, and none anywhere else."""
where = _resolved(url)
if where is None:
return {}
for base, sent in grants.items():
address = _resolved(base) if base else None
if address is not None and where[0] == address[0] and where[1][:len(address[1])] == address[1]:
return sent
return {}


def _resolved(url: str) -> tuple[tuple[str, str, int | None], list[str]] | None:
"""``url``'s origin and path segments as its server routes them, dot segments resolved; None for a relative url or
one whose path still holds a dot segment once percent-decoded, which a server may resolve out of its address."""
try:
parsed = httpx.URL(url)
except httpx.InvalidURL:
return None
segments = parsed.path.rstrip("/").split("/")
if not parsed.is_absolute_url or any(segment in (".", "..") for segment in segments):
return None
return (parsed.scheme, parsed.host, parsed.port), segments


async def _rpc(base_url: str, method: str, params: dict, timeout: int, verify: bool,
Expand Down
18 changes: 18 additions & 0 deletions packages/agentenv-protocol/tests/test_control_token.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,24 @@ def test_an_agents_token_travels_in_its_own_header_not_authorization():
assert client.control_token_headers(None) == {} and client.control_token_headers("") == {}


@pytest.mark.parametrize("url, granted", [
("http://proxy/sandbox/A/svc/mcp-x/agentenv", True),
("http://PROXY:80/sandbox/A/./svc/x", True),
("http://proxy/sandbox/A/svc/x/../y", True),
("http://proxy/sandbox/A/svc/mcp-x/../../../B/x", False),
("http://proxy/sandbox/A/../B/x", False),
("http://proxy/sandbox/A/%2e%2e/B/x", False),
("http://proxy/sandbox/AB/x", False),
("https://proxy/sandbox/A/x", False),
("../sandbox/A/x", False),
])
def test_a_grant_covers_only_urls_that_resolve_under_its_address(url, granted):
"""The server resolves dot segments before it routes, so a dotted path that climbs out of the env's address
reaches another env behind the same proxy."""
assert client.granted_headers(url, {"http://proxy/sandbox/A/": client.control_headers("s3cret")}) == (
{"Authorization": "Bearer s3cret"} if granted else {})


@pytest.mark.asyncio
async def test_invoke_extension_and_the_data_plane_send_the_token(monkeypatch):
seen = []
Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/env/envs/_deployment.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo
return False
else:
try:
await legacy_protocol.reset_via_rest(legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp),
timeout=60)
legacy_url = legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp)
await legacy_protocol.reset_via_rest(legacy_url, timeout=60, headers=data_plane_headers(env, legacy_url))
except httpx.HTTPStatusError as e:
if e.response.status_code != 404:
raise
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/env/envs/mcp_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,9 +295,9 @@ async def _exec_in_server(self, *command: str) -> None:

async def _load_environment_artifact_legacy(self, file_artifact) -> None:
container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename)
await legacy_protocol.reset(
self._gateway_url, self.environment_name, container_path, max_retries=self._MCP_MAX_RETRIES
)
base_url = legacy_protocol.environment_base_url(self._gateway_url, self.environment_name, mcp=True)
await legacy_protocol.reset(self._gateway_url, self.environment_name, container_path,
max_retries=self._MCP_MAX_RETRIES, headers=data_plane_headers(self, base_url))

async def validate(self, on_progress: Optional[Callable[[str], None]] = None) -> str:
"""Run the env validator task and return the task instance ID.
Expand Down
5 changes: 3 additions & 2 deletions src/agent_env/env/envs/website.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,9 +371,10 @@ async def _load_environment_artifact_legacy(self, file_artifact) -> None:
container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename)
base_url = legacy_protocol.environment_base_url(self._gateway_url, self.environment_name, mcp=False)
from agent_env.env.gateway.constants import DATA_PLANE_LOAD_TIMEOUT_S
await legacy_protocol.reset_via_rest(base_url, timeout=60)
headers = data_plane_headers(self, base_url)
await legacy_protocol.reset_via_rest(base_url, timeout=60, headers=headers)
try:
await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S)
await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers)
except Exception as e:
logger.error(
"load_environment_artifact: /api/add failed after /api/reset succeeded; "
Expand Down
53 changes: 35 additions & 18 deletions src/agent_env/env/legacy_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,58 +209,70 @@ async def reset_via_rest(
mock_data_path: Optional[str] = None,
timeout: int = 30,
verify: bool = True,
*,
headers: Optional[dict[str, str]] = None,
) -> dict:
"""POST ``{base_url}/api/reset``. Sends the seed path when given. Raises on non-2xx."""
"""POST ``{base_url}/api/reset`` with ``headers``, else the run's grant (``request_headers``). Sends the seed path
when given. Raises on non-2xx."""
body = {"mock_data_path": mock_data_path} if mock_data_path is not None else None
url = f"{base_url}/api/reset"
async with httpx.AsyncClient(verify=verify) as client:
response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout)
response = await client.post(url, json=body, timeout=timeout, headers=protocol_v1.request_headers(url, headers))
response.raise_for_status()
result = response.json()
logger.info(f"load_environment_artifact: REST reset succeeded: {result}")
return result


async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True) -> dict:
"""POST ``{base_url}/api/add`` with a backend file path. Raises on non-2xx."""
async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True, *,
headers: Optional[dict[str, str]] = None) -> dict:
"""POST ``{base_url}/api/add`` with a backend file path and ``headers``, else the run's grant. Raises on non-2xx."""
url = f"{base_url}/api/add"
async with httpx.AsyncClient(verify=verify) as client:
response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout)
response = await client.post(url, json={"file_path": file_path}, timeout=timeout,
headers=protocol_v1.request_headers(url, headers))
response.raise_for_status()
result = response.json()
logger.info(f"load_environment_artifact: add response: {result}")
return result


async def export_state(
gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True
gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True, *,
headers: Optional[dict[str, str]] = None,
) -> dict:
"""GET ``/svc/mcp-{name}/export-state`` — the legacy state snapshot for one service."""
return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify)
"""GET ``/svc/mcp-{name}/export-state``, the legacy state snapshot for one service, with ``headers``, else the
run's grant."""
return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify, headers)


async def _export_state_at(base_url: str, timeout: int, verify: bool) -> dict:
async def _export_state_at(base_url: str, timeout: int, verify: bool, headers: Optional[dict[str, str]]) -> dict:
url = f"{base_url}/export-state"
async with httpx.AsyncClient(verify=verify) as client:
response = await client.get(f"{base_url}/export-state", timeout=timeout)
response = await client.get(url, timeout=timeout, headers=protocol_v1.request_headers(url, headers))
response.raise_for_status()
return response.json()


async def service_state(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str) -> dict:
"""An MCP service's state as JSON: its v1 ``data/get`` answer when that is data, asked with the deployment's
control token, else ``GET /export-state``.
"""An MCP service's state as JSON: its v1 ``data/get`` answer when that is data, else ``GET /export-state``, each
asked with the deployment's control token.

A service that exports a file from ``data/get`` (a bundle of its database, to reload from) still serves its state
as JSON at ``/export-state``.
"""
base_url = await v1_base_url(deployed, gateway_url, environment_name, mcp=True)
if base_url is None:
return await export_state(gateway_url, environment_name)
response = await protocol_v1.get_data(base_url,
headers=deployed.control_headers_for(base_url) if deployed is not None else None)
legacy_url = environment_base_url(gateway_url, environment_name, mcp=True)
return await export_state(gateway_url, environment_name,
headers=(deployed.control_headers_for(legacy_url) or None) if deployed is not None else None)
headers = (deployed.control_headers_for(base_url) or None) if deployed is not None else None
response = await protocol_v1.get_data(base_url, headers=headers)
if not response.parts:
return {}
if isinstance(response.parts[0], DataPart):
return response.parts[0].data
return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True)
return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True, headers=headers)


async def offers_tool(mcp_url: str, name: str) -> bool:
Expand Down Expand Up @@ -322,12 +334,17 @@ async def reset(
environment_name: str,
mock_data_path: str,
max_retries: int = DEFAULT_MCP_MAX_RETRIES,
*,
headers: Optional[dict[str, str]] = None,
) -> Optional[dict]:
"""Reset an MCP service to a seed: try REST ``/api/reset``, fall back to the MCP tool."""
"""Reset an MCP service to a seed: try REST ``/api/reset`` with ``headers``, fall back to the MCP tool. A 401 or
403 raises: it means the token was dropped or wrong, and the tool would reset the service without one."""
base_url = environment_base_url(gateway_url, environment_name, mcp=True)
try:
return await reset_via_rest(base_url, mock_data_path)
return await reset_via_rest(base_url, mock_data_path, headers=headers)
except Exception as rest_err:
if isinstance(rest_err, httpx.HTTPStatusError) and rest_err.response.status_code in (401, 403):
raise
# TODO remove MCP tool fallback once all usages of MCP servers are on REST
logger.info(f"load_environment_artifact: REST reset failed ({type(rest_err).__name__}), falling back to MCP tool")
await reset_via_mcp_tool(f"{gateway_url}/mcp", environment_name, mock_data_path, max_retries=max_retries)
Expand Down
7 changes: 6 additions & 1 deletion src/agent_env/task_step/task_steps/snapshot_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ async def _export_environment_to_file(
if base_url is None and not mcp:
return None
if base_url is not None:
headers = deployed.control_headers_for(base_url) if deployed is not None else None
headers = (deployed.control_headers_for(base_url) or None) if deployed is not None else None
if returns_file_tree(card or {}):
upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds,
getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS)
Expand Down Expand Up @@ -582,13 +582,16 @@ async def _export_environment_to_file(
raise ValueError(
f"FilePart for '{environment_name}' names {uri!r}, which is not on its service's address {base_url}"
)
# The service names this url, so it gets the token only where it resolves under the service.
uri_headers = protocol_v1.granted_headers(uri, {base_url: protocol_v1.request_headers(base_url, headers)})
async with httpx.AsyncClient() as client:
async with client.stream(
"GET",
uri,
timeout=httpx.Timeout(
timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS
),
headers=uri_headers,
) as resp_file:
resp_file.raise_for_status()
with open(tmp_path, "wb") as f:
Expand All @@ -601,11 +604,13 @@ async def _export_environment_to_file(
return ".json"

legacy_url = f"{legacy_protocol.environment_base_url(gateway_url, environment_name, mcp=True)}/export-state"
legacy_headers = (deployed.control_headers_for(legacy_url) or None) if deployed is not None else None
async with httpx.AsyncClient() as client:
async with client.stream(
"GET",
legacy_url,
timeout=httpx.Timeout(timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS),
headers=protocol_v1.request_headers(legacy_url, legacy_headers),
) as resp:
resp.raise_for_status()
seen_first_bytes = False
Expand Down
4 changes: 2 additions & 2 deletions tst/integration/env/envs/multi_env_local_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,13 @@ async def test_a_multi_env_deploys_restores_loads_and_tears_down_on_local_compos
base_url = await legacy_protocol.v1_base_url(deployed, deployed.gateway_url, "items")
assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]}
slack_state = f"{legacy_protocol.environment_base_url(deployed.gateway_url, 'slack')}/export-state"
assert len(httpx.get(slack_state, timeout=30).json()["channels"]) == 10
assert len(httpx.get(slack_state, headers=deployed.control_headers, timeout=30).json()["channels"]) == 10

# A multi env without computers resets its servers, each as it resets alone, and fails if one doesn't.
await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute(
TaskStepContext(deployed_envs=[deployed]))
assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": []}
assert httpx.get(slack_state, timeout=30).json()["channels"] == []
assert httpx.get(slack_state, headers=deployed.control_headers, timeout=30).json()["channels"] == []

await restored.close()
assert _stack(work_dir) == ""
Expand Down
19 changes: 19 additions & 0 deletions tst/unit/env/envs/test_env_card_loads.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,25 @@ async def test_a_child_env_missing_from_the_card_takes_the_legacy_path(sent):
assert json.loads(request.content) == {"mock_data_path": "/data/email.json"}


@pytest.mark.asyncio
@pytest.mark.parametrize("token, sent_as", [("s3cret", "Bearer s3cret"), (None, None)], ids=["token", "no-token"])
async def test_the_legacy_path_sends_its_records_control_token_when_it_has_one(sent, token, sent_as):
# Neither email nor shop serves a card, so the gateway's card, read at its own address, lists neither.
deployed = DeployedGatewayEnv(
env_id="multi", env_version=1, gateway_url=_GW, mcp_url=f"{_GW}/mcp", db_web_url=None, sandbox_id="sb-1",
environment_card_url=f"{_GW}{WELL_KNOWN_PATH}", environment_card=await _card("mcp-slack"), control_token=token)
web = _website()
web._sandbox, web._gateway_url, web._deployed = MagicMock(), _GW, deployed
web._copy_artifact_into_container = AsyncMock(return_value="/tmp/data/shop.json")
web._env_provider = MagicMock(spec=EnvironmentGatewayProvider, install_changelog_triggers=AsyncMock())

await _mcp(deployed).load_environment_artifact(_artifact("email"))
await web.load_environment_artifact(_artifact("shop"))

assert [(r.url.path, r.headers.get("authorization")) for r in sent] == [
("/svc/mcp-email/api/reset", sent_as), ("/svc/shop/api/reset", sent_as), ("/svc/shop/api/add", sent_as)]


@pytest.mark.asyncio
async def test_the_legacy_path_refuses_an_environment_of_several_files_before_sending_anything(sent):
several = SimpleNamespace(environment_name="email", id="mail-seed", file_artifact_refs={"a.json": None, "b.json": None})
Expand Down
8 changes: 4 additions & 4 deletions tst/unit/env/envs/test_mcp_server_plugin_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI

from agent_env.config import get_config, reset_config
from agent_env.env.env import DeployedEnv, DeployedSandboxEnv
from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv
from agent_env.env.envs._deployment import LOAD_OPERATIONS
from agent_env.env.envs.mcp_server import MCPServerEnv
from agent_env.env.envs.multi_env import MultiEnv
Expand Down Expand Up @@ -228,9 +228,9 @@ async def test_close_closes_the_plugin_that_deployed():
async def test_a_multi_env_child_of_a_plugin_type_is_still_the_gateways():
child = _env()
multi = MultiEnv(id="multi", version=1, mcp_server_envs=[child])
record = SimpleNamespace(env_id="multi", env_version=1, sandbox_id="gw", sandbox_type="modal", sandbox_ids={},
env_state_instance_ids=[], instance_id="i", mcp_server_name="m", environment_card=None,
environment_card_url=None, gateway_url="https://gw.example")
record = DeployedGatewayEnv(env_id="multi", env_version=1, sandbox_id="gw", sandbox_type="modal", instance_id="i",
mcp_server_name="m", gateway_url="https://gw.example", mcp_url="https://gw.example/mcp",
db_web_url=None)
gw = MagicMock(sandbox_id="gw", mode="vm", type="modal")
with patch("agent_env.env.env.Env.get", return_value=multi), \
patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(return_value=gw))), \
Expand Down
11 changes: 11 additions & 0 deletions tst/unit/env/envs/test_server_reset.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,17 @@ async def test_an_older_server_is_emptied_through_its_api_reset_and_one_answerin
assert [call.args for call in offers.await_args_list] == ([] if reset else [(f"{GATEWAY}/mcp", "email_reset")])


@pytest.mark.asyncio
@pytest.mark.parametrize("token, sent", [("s3cret", "Bearer s3cret"), (None, None)], ids=["token", "no-token"])
async def test_an_older_servers_api_reset_sends_its_records_control_token_when_it_has_one(server, token, sent):
env = _server(None)
env._deployed.control_token = token
with patch("agent_env.env.legacy_protocol.v1_base_url", AsyncMock(return_value=None)):
assert await _deployment.reset_server_data(env, mcp=True) is True
assert [(request.url.path, request.headers.get("authorization")) for request in server.requests] == [
("/svc/mcp-email/api/reset", sent)]


@pytest.mark.asyncio
async def test_an_older_server_answering_404_resets_through_its_tool_with_no_seed_to_the_data_it_boots_with(server):
server.status = 404
Expand Down
Loading
Loading