Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 4 additions & 12 deletions src/agent_env/env/envs/computer/utils/sites.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of
from agent_env.env.envs.computer.utils.site_proxy import is_local, matches
from agent_env.env.gateway.constants import AGENT_ENV_COMPUTER_HEADER
from agent_env.env.gateway.rest_proxy import fallback_name, served_names
from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV
from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox
from agent_env.store.base import NotFoundError
Expand Down Expand Up @@ -165,13 +166,6 @@ def covers(strong: str, weak: str) -> bool:
and weak.partition(".")[2] == strong[2:])


def fallback_name(site: str) -> str:
"""The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a
``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name
are refused as any two claiming it."""
return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost"


def claims(sites: Iterable[Site], allow_hosts: Iterable[str] = ()) -> list[Claim]:
"""Each name claimed: a site's, its own or ``fallback_name`` when it declares none, and the network's allowlist."""
found = [Claim(name, SYNTHETIC, site.name) for site in sites
Expand Down Expand Up @@ -594,11 +588,9 @@ def serves_websites(record: DeployedEnv) -> bool:
def _served_names(sites: list[tuple[str, dict]]) -> list[dict]:
"""One env's ``sites``, each with the name it's served as: named by their cards, unless two cards share a name, as
two websites of one image whose cards name it can; then each is named as it's served, its environment_name, which
the env's gateway gives each a route of its own by."""
names = [site["name"] for _, site in sites]
if len(set(names)) == len(names):
return [site for _, site in sites]
return [{**site, "name": service} for service, site in sites]
the env's gateway gives each a route of its own by (``served_names``, which its gateway lists them by too)."""
named = served_names({service: site["name"] for service, site in sites})
return [{**site, "name": named[service]} for service, site in sites]


def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]:
Expand Down
44 changes: 33 additions & 11 deletions src/agent_env/env/gateway/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@
WebsiteRoute,
declared_site_names,
called_tools,
fallback_name,
served_names,
website_card_name,
fetch_child_card,
forward,
is_child_mcp_path,
Expand Down Expand Up @@ -121,11 +124,13 @@


class SiteCard(NamedTuple):
"""What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, and
the names it declares."""
"""What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, the
names it declares (hostnames), and the card's own name (``website_card_name``), which needn't be the name the
gateway serves the site as."""

reserved: frozenset[str]
names: tuple[str, ...]
card_name: str | None = None


class ToolNameClash(ValueError):
Expand Down Expand Up @@ -246,7 +251,7 @@ def __init__(
if self.website_urls:
@self._mcp.tool(name="list_website_urls", description="List available website URLs that can be browsed.")
async def list_website_urls() -> list[str]:
return [await self._listed_url(site, url) for site, url in self.website_urls.items()]
return await self._listed_urls()
self._server_tools.setdefault("gateway", []).append(
MCPTool(
name="list_website_urls",
Expand Down Expand Up @@ -1027,21 +1032,38 @@ async def _site_card(self, app: str) -> SiteCard | None:
except ValueError:
card = None
address = self.website_urls.get(app) or base_url
self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card))
self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card),
website_card_name(card))
if isinstance(card, dict) else SiteCard(frozenset(), ()))
return self._site_cards[app]

def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]:
"""The names website ``app`` is reached at: those its env stores, else those its ``card`` declares."""
return self._site_names.get(app) or (card.names if card else ())

async def _listed_url(self, site: str, url: str) -> str:
"""The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name,
its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``."""
if site not in self._listed_by_name:
return url
card = None if site in self._site_names else await self._site_card(site)
return listed_url(self._names(site, card)) or url
async def _listed_urls(self) -> list[str]:
"""The address ``list_website_urls`` gives each site, at its url behind the gateway: for a site listed by name,
its first name (``listed_url``), those its env stores, else those its card declares, else, for a site declaring
none, the ``fallback_name`` its browser's site proxy routes it at, of the name it goes by (``served_names``:
its card's, unless two sites' cards give one); else, or while its card can't be read, its url. Cards are read
together, and those of sites with stored names only when some site needs a fallback."""
by_name = [site for site in self.website_urls if site in self._listed_by_name]
cards = await self._site_cards_of([site for site in by_name if site not in self._site_names])
if any(card is not None and not card.names for card in cards.values()):
cards |= await self._site_cards_of([site for site in by_name if site in self._site_names])
goes_by = served_names({site: card.card_name or site for site, card in cards.items() if card is not None})
listed = []
for site, url in self.website_urls.items():
card = cards.get(site)
names = self._names(site, card) if site in self._listed_by_name else ()
if not names and card is not None:
names = (fallback_name(goes_by[site]),)
listed.append(listed_url(names) or url)
return listed

async def _site_cards_of(self, sites: list[str]) -> dict[str, SiteCard | None]:
"""Each of ``sites``' cards (``_site_card``), read at once."""
return dict(zip(sites, await asyncio.gather(*(self._site_card(site) for site in sites)), strict=True))

def _backing_keys(self) -> list[str]:
# rest_proxy_urls entries are the env's backing servers: mcp-<svc> (MCP) + <svc> (website
Expand Down
25 changes: 24 additions & 1 deletion src/agent_env/env/gateway/rest_proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import logging
import posixpath
import re
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping
from typing import Any, NamedTuple
from urllib.parse import unquote, urlsplit, urlunsplit

Expand Down Expand Up @@ -240,6 +240,29 @@ def declared_site_names(card: dict) -> tuple[str, ...]:
return ()


def website_card_name(card: dict) -> str | None:
"""The name a site's card gives it, as a computer's site proxy names the site: the card's ``name`` when it
declares ``website/v1``; None otherwise, and the site goes by the name it's served as."""
extensions = (card.get("capabilities") or {}).get("extensions") or []
name = card.get("name")
declares = any(isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI for ext in extensions)
return name if declares and isinstance(name, str) and name else None


def served_names(sites: Mapping[str, str]) -> dict[str, str]:
"""The name each of one env's sites goes by, by the name it's served as: the name it gives, its card's or else the
one it's served as, unless two give one name, and then each the name it's served as."""
given = list(sites.values())
return dict(sites) if len(set(given)) == len(given) else {served: served for served in sites}


def fallback_name(site: str) -> str:
"""The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a
``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name
are refused as any two claiming it."""
return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost"


def listed_url(names: tuple[str, ...]) -> str | None:
"""The address a browser computer opens a site with ``names`` at: its first name that isn't a ``*.`` one, over
HTTPS but for a ``.localhost`` one, as its site proxy serves them. None with no such name."""
Expand Down
54 changes: 50 additions & 4 deletions tst/unit/env/gateway/rest_proxy_stream_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -826,14 +826,60 @@ async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_a
transport = _card_site([], {"carded": ["carded.test"]})
monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient",
lambda **kw: _RealAsyncClient(transport=transport, **kw))
sites = ("shop", "notes", "wild", "bare", "carded", "other")
sites = ("shop", "notes", "wild", "bare", "Old_Notes", "carded", "other")
gw = _gateway(website_urls={site: f"http://{site}" for site in sites},
site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"],
"wild": ["*.wild.test"], "other": ["other.test"]},
listed_by_name=["shop", "notes", "wild", "bare", "carded"])
listed_by_name=["shop", "notes", "wild", "bare", "Old_Notes", "carded"])
listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn()
assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare",
"https://carded.test/", "http://other"]
assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare.localhost/",
"http://old-notes.localhost/", "https://carded.test/", "http://other"]


@pytest.mark.asyncio
async def test_a_site_declaring_no_names_is_listed_at_the_fallback_of_the_name_its_card_gives_it_as_a_proxy_routes_it(
monkeypatch):
named = {"notes": "NotesApp", "left": "Shop", "right": "Shop", "plain": None}

def handler(request: httpx.Request) -> httpx.Response:
card = {"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {}}]}}
return httpx.Response(200, json={**card, **({"name": name} if (name := named[request.url.host]) else {})})
transport = httpx.MockTransport(handler)
monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient",
lambda **kw: _RealAsyncClient(transport=transport, **kw))
gw = _gateway(website_urls={site: f"http://{site}" for site in named}, listed_by_name=list(named))
listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn()
# Named by their cards, but for the two cards giving one name, each named as it's served instead.
assert listed == ["http://notes.localhost/", "http://left.localhost/", "http://right.localhost/",
"http://plain.localhost/"]
gw = _gateway(website_urls={"notes": "http://notes", "plain": "http://plain"}, listed_by_name=["notes", "plain"])
assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notesapp.localhost/",
"http://plain.localhost/"]


@pytest.mark.asyncio
async def test_list_website_urls_reads_no_card_when_every_site_listed_by_name_stores_its_names(monkeypatch):
asked: list[str] = []

def handler(request: httpx.Request) -> httpx.Response:
asked.append(str(request.url))
return httpx.Response(404)
transport = httpx.MockTransport(handler)
monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient",
lambda **kw: _RealAsyncClient(transport=transport, **kw))
gw = _gateway(website_urls={"shop": "http://shop", "notes": "http://notes"},
site_names={"shop": ["shop.test"], "notes": ["notes.localhost"]}, listed_by_name=["shop", "notes"])
listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn()
assert (listed, asked) == (["https://shop.test/", "http://notes.localhost/"], [])


@pytest.mark.asyncio
async def test_list_website_urls_gives_a_site_listed_by_name_its_address_while_its_card_cant_be_read(monkeypatch):
transport = httpx.MockTransport(lambda request: httpx.Response(503))
monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient",
lambda **kw: _RealAsyncClient(transport=transport, **kw))
gw = _gateway(website_urls={"notes": "http://notes"}, listed_by_name=["notes"])
assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notes"]


@pytest.mark.asyncio
Expand Down
Loading