diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index e529c5fb8..b24fc67f8 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -45,6 +45,7 @@ from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of from agent_env.env.envs.computer.utils.site_proxy import is_local, matches from agent_env.env.gateway.constants import AGENT_ENV_COMPUTER_HEADER +from agent_env.env.gateway.rest_proxy import fallback_name, served_names from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox from agent_env.store.base import NotFoundError @@ -165,13 +166,6 @@ def covers(strong: str, weak: str) -> bool: and weak.partition(".")[2] == strong[2:]) -def fallback_name(site: str) -> str: - """The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a - ``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name - are refused as any two claiming it.""" - return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost" - - def claims(sites: Iterable[Site], allow_hosts: Iterable[str] = ()) -> list[Claim]: """Each name claimed: a site's, its own or ``fallback_name`` when it declares none, and the network's allowlist.""" found = [Claim(name, SYNTHETIC, site.name) for site in sites @@ -594,11 +588,9 @@ def serves_websites(record: DeployedEnv) -> bool: def _served_names(sites: list[tuple[str, dict]]) -> list[dict]: """One env's ``sites``, each with the name it's served as: named by their cards, unless two cards share a name, as two websites of one image whose cards name it can; then each is named as it's served, its environment_name, which - the env's gateway gives each a route of its own by.""" - names = [site["name"] for _, site in sites] - if len(set(names)) == len(names): - return [site for _, site in sites] - return [{**site, "name": service} for service, site in sites] + the env's gateway gives each a route of its own by (``served_names``, which its gateway lists them by too).""" + named = served_names({service: site["name"] for service, site in sites}) + return [{**site, "name": named[service]} for service, site in sites] def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]: diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 5c42fdb82..a503d8c84 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -87,6 +87,9 @@ WebsiteRoute, declared_site_names, called_tools, + fallback_name, + served_names, + website_card_name, fetch_child_card, forward, is_child_mcp_path, @@ -121,11 +124,13 @@ class SiteCard(NamedTuple): - """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, and - the names it declares.""" + """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, the + names it declares (hostnames), and the card's own name (``website_card_name``), which needn't be the name the + gateway serves the site as.""" reserved: frozenset[str] names: tuple[str, ...] + card_name: str | None = None class ToolNameClash(ValueError): @@ -246,7 +251,7 @@ def __init__( if self.website_urls: @self._mcp.tool(name="list_website_urls", description="List available website URLs that can be browsed.") async def list_website_urls() -> list[str]: - return [await self._listed_url(site, url) for site, url in self.website_urls.items()] + return await self._listed_urls() self._server_tools.setdefault("gateway", []).append( MCPTool( name="list_website_urls", @@ -1027,7 +1032,8 @@ async def _site_card(self, app: str) -> SiteCard | None: except ValueError: card = None address = self.website_urls.get(app) or base_url - self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card)) + self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card), + website_card_name(card)) if isinstance(card, dict) else SiteCard(frozenset(), ())) return self._site_cards[app] @@ -1035,13 +1041,29 @@ def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]: """The names website ``app`` is reached at: those its env stores, else those its ``card`` declares.""" return self._site_names.get(app) or (card.names if card else ()) - async def _listed_url(self, site: str, url: str) -> str: - """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, - its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``.""" - if site not in self._listed_by_name: - return url - card = None if site in self._site_names else await self._site_card(site) - return listed_url(self._names(site, card)) or url + async def _listed_urls(self) -> list[str]: + """The address ``list_website_urls`` gives each site, at its url behind the gateway: for a site listed by name, + its first name (``listed_url``), those its env stores, else those its card declares, else, for a site declaring + none, the ``fallback_name`` its browser's site proxy routes it at, of the name it goes by (``served_names``: + its card's, unless two sites' cards give one); else, or while its card can't be read, its url. Cards are read + together, and those of sites with stored names only when some site needs a fallback.""" + by_name = [site for site in self.website_urls if site in self._listed_by_name] + cards = await self._site_cards_of([site for site in by_name if site not in self._site_names]) + if any(card is not None and not card.names for card in cards.values()): + cards |= await self._site_cards_of([site for site in by_name if site in self._site_names]) + goes_by = served_names({site: card.card_name or site for site, card in cards.items() if card is not None}) + listed = [] + for site, url in self.website_urls.items(): + card = cards.get(site) + names = self._names(site, card) if site in self._listed_by_name else () + if not names and card is not None: + names = (fallback_name(goes_by[site]),) + listed.append(listed_url(names) or url) + return listed + + async def _site_cards_of(self, sites: list[str]) -> dict[str, SiteCard | None]: + """Each of ``sites``' cards (``_site_card``), read at once.""" + return dict(zip(sites, await asyncio.gather(*(self._site_card(site) for site in sites)), strict=True)) def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 4cea659dc..75a87191f 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -37,7 +37,7 @@ import logging import posixpath import re -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from typing import Any, NamedTuple from urllib.parse import unquote, urlsplit, urlunsplit @@ -240,6 +240,29 @@ def declared_site_names(card: dict) -> tuple[str, ...]: return () +def website_card_name(card: dict) -> str | None: + """The name a site's card gives it, as a computer's site proxy names the site: the card's ``name`` when it + declares ``website/v1``; None otherwise, and the site goes by the name it's served as.""" + extensions = (card.get("capabilities") or {}).get("extensions") or [] + name = card.get("name") + declares = any(isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI for ext in extensions) + return name if declares and isinstance(name, str) and name else None + + +def served_names(sites: Mapping[str, str]) -> dict[str, str]: + """The name each of one env's sites goes by, by the name it's served as: the name it gives, its card's or else the + one it's served as, unless two give one name, and then each the name it's served as.""" + given = list(sites.values()) + return dict(sites) if len(set(given)) == len(given) else {served: served for served in sites} + + +def fallback_name(site: str) -> str: + """The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a + ``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name + are refused as any two claiming it.""" + return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost" + + def listed_url(names: tuple[str, ...]) -> str | None: """The address a browser computer opens a site with ``names`` at: its first name that isn't a ``*.`` one, over HTTPS but for a ``.localhost`` one, as its site proxy serves them. None with no such name.""" diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 5df7f530e..38743e8a0 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -826,14 +826,60 @@ async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_a transport = _card_site([], {"carded": ["carded.test"]}) monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) - sites = ("shop", "notes", "wild", "bare", "carded", "other") + sites = ("shop", "notes", "wild", "bare", "Old_Notes", "carded", "other") gw = _gateway(website_urls={site: f"http://{site}" for site in sites}, site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"], "wild": ["*.wild.test"], "other": ["other.test"]}, - listed_by_name=["shop", "notes", "wild", "bare", "carded"]) + listed_by_name=["shop", "notes", "wild", "bare", "Old_Notes", "carded"]) listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() - assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare", - "https://carded.test/", "http://other"] + assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare.localhost/", + "http://old-notes.localhost/", "https://carded.test/", "http://other"] + + +@pytest.mark.asyncio +async def test_a_site_declaring_no_names_is_listed_at_the_fallback_of_the_name_its_card_gives_it_as_a_proxy_routes_it( + monkeypatch): + named = {"notes": "NotesApp", "left": "Shop", "right": "Shop", "plain": None} + + def handler(request: httpx.Request) -> httpx.Response: + card = {"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {}}]}} + return httpx.Response(200, json={**card, **({"name": name} if (name := named[request.url.host]) else {})}) + transport = httpx.MockTransport(handler) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={site: f"http://{site}" for site in named}, listed_by_name=list(named)) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + # Named by their cards, but for the two cards giving one name, each named as it's served instead. + assert listed == ["http://notes.localhost/", "http://left.localhost/", "http://right.localhost/", + "http://plain.localhost/"] + gw = _gateway(website_urls={"notes": "http://notes", "plain": "http://plain"}, listed_by_name=["notes", "plain"]) + assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notesapp.localhost/", + "http://plain.localhost/"] + + +@pytest.mark.asyncio +async def test_list_website_urls_reads_no_card_when_every_site_listed_by_name_stores_its_names(monkeypatch): + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(404) + transport = httpx.MockTransport(handler) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"shop": "http://shop", "notes": "http://notes"}, + site_names={"shop": ["shop.test"], "notes": ["notes.localhost"]}, listed_by_name=["shop", "notes"]) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + assert (listed, asked) == (["https://shop.test/", "http://notes.localhost/"], []) + + +@pytest.mark.asyncio +async def test_list_website_urls_gives_a_site_listed_by_name_its_address_while_its_card_cant_be_read(monkeypatch): + transport = httpx.MockTransport(lambda request: httpx.Response(503)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"notes": "http://notes"}, listed_by_name=["notes"]) + assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notes"] @pytest.mark.asyncio