From ae0d6bbdaff3bb691540a21c67e8cfb8029a7381 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 09:33:32 -0800 Subject: [PATCH 1/4] fix(gateway): list_website_urls gives a site that declares no names its .localhost name, as a browser computer opens it A multi env whose website browser is a computer lists its sites by name, since the browser can't reach their containers. A site declaring no names was listed at its backend address, which the browser can't resolve, while the computer's site proxy routes it at fallback_name(site) and its start tabs open it there. The gateway now lists that name too; a site whose card can't be read yet keeps its address until it can. fallback_name moves next to listed_url so the gateway and the site proxy share one rule. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/sites.py | 8 +------- src/agent_env/env/gateway/gateway.py | 8 ++++++-- src/agent_env/env/gateway/rest_proxy.py | 7 +++++++ tst/unit/env/gateway/rest_proxy_stream_test.py | 17 +++++++++++++---- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index e529c5fb8..0d2b01b1a 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -45,6 +45,7 @@ from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of from agent_env.env.envs.computer.utils.site_proxy import is_local, matches from agent_env.env.gateway.constants import AGENT_ENV_COMPUTER_HEADER +from agent_env.env.gateway.rest_proxy import fallback_name from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox from agent_env.store.base import NotFoundError @@ -165,13 +166,6 @@ def covers(strong: str, weak: str) -> bool: and weak.partition(".")[2] == strong[2:]) -def fallback_name(site: str) -> str: - """The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a - ``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name - are refused as any two claiming it.""" - return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost" - - def claims(sites: Iterable[Site], allow_hosts: Iterable[str] = ()) -> list[Claim]: """Each name claimed: a site's, its own or ``fallback_name`` when it declares none, and the network's allowlist.""" found = [Claim(name, SYNTHETIC, site.name) for site in sites diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 5c42fdb82..7912d3030 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -87,6 +87,7 @@ WebsiteRoute, declared_site_names, called_tools, + fallback_name, fetch_child_card, forward, is_child_mcp_path, @@ -1037,11 +1038,14 @@ def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]: async def _listed_url(self, site: str, url: str) -> str: """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, - its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``.""" + its first name (``listed_url``), its card read for a site whose env stores none, and ``fallback_name`` for one + whose card declares none either, as its browser's site proxy routes it; else, or while its card can't be read, + ``url``.""" if site not in self._listed_by_name: return url card = None if site in self._site_names else await self._site_card(site) - return listed_url(self._names(site, card)) or url + names = self._names(site, card) or ((fallback_name(site),) if card is not None else ()) + return listed_url(names) or url def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 4cea659dc..0a4eeac39 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -240,6 +240,13 @@ def declared_site_names(card: dict) -> tuple[str, ...]: return () +def fallback_name(site: str) -> str: + """The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a + ``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name + are refused as any two claiming it.""" + return f"{site.lower().replace('_', '-')[:63].strip('-')}.localhost" + + def listed_url(names: tuple[str, ...]) -> str | None: """The address a browser computer opens a site with ``names`` at: its first name that isn't a ``*.`` one, over HTTPS but for a ``.localhost`` one, as its site proxy serves them. None with no such name.""" diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 5df7f530e..6b216bb9f 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -826,14 +826,23 @@ async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_a transport = _card_site([], {"carded": ["carded.test"]}) monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) - sites = ("shop", "notes", "wild", "bare", "carded", "other") + sites = ("shop", "notes", "wild", "bare", "Old_Notes", "carded", "other") gw = _gateway(website_urls={site: f"http://{site}" for site in sites}, site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"], "wild": ["*.wild.test"], "other": ["other.test"]}, - listed_by_name=["shop", "notes", "wild", "bare", "carded"]) + listed_by_name=["shop", "notes", "wild", "bare", "Old_Notes", "carded"]) listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() - assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare", - "https://carded.test/", "http://other"] + assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare.localhost/", + "http://old-notes.localhost/", "https://carded.test/", "http://other"] + + +@pytest.mark.asyncio +async def test_list_website_urls_gives_a_site_listed_by_name_its_address_while_its_card_cant_be_read(monkeypatch): + transport = httpx.MockTransport(lambda request: httpx.Response(503)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"notes": "http://notes"}, listed_by_name=["notes"]) + assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notes"] @pytest.mark.asyncio From b546fbea47e037e8542d26e557e7bfc9e54f9233 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 09:41:55 -0800 Subject: [PATCH 2/4] fix(gateway): a nameless site is listed at the fallback of the name its card gives it, as the site proxy routes it The site proxy names a site by its website/v1 card's name, unless two of the env's cards give one name, and then each by the name it's served as; a site declaring no names is routed at fallback_name of that. The gateway now keeps each card's name (SiteCard.name, website_card_name) and lists the sites by the same rule (served_names, shared with sites._served_names), instead of by the name each is served as. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/sites.py | 10 ++--- src/agent_env/env/gateway/gateway.py | 37 ++++++++++++------- src/agent_env/env/gateway/rest_proxy.py | 18 ++++++++- .../env/gateway/rest_proxy_stream_test.py | 21 +++++++++++ 4 files changed, 65 insertions(+), 21 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 0d2b01b1a..b24fc67f8 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -45,7 +45,7 @@ from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of from agent_env.env.envs.computer.utils.site_proxy import is_local, matches from agent_env.env.gateway.constants import AGENT_ENV_COMPUTER_HEADER -from agent_env.env.gateway.rest_proxy import fallback_name +from agent_env.env.gateway.rest_proxy import fallback_name, served_names from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox from agent_env.store.base import NotFoundError @@ -588,11 +588,9 @@ def serves_websites(record: DeployedEnv) -> bool: def _served_names(sites: list[tuple[str, dict]]) -> list[dict]: """One env's ``sites``, each with the name it's served as: named by their cards, unless two cards share a name, as two websites of one image whose cards name it can; then each is named as it's served, its environment_name, which - the env's gateway gives each a route of its own by.""" - names = [site["name"] for _, site in sites] - if len(set(names)) == len(names): - return [site for _, site in sites] - return [{**site, "name": service} for service, site in sites] + the env's gateway gives each a route of its own by (``served_names``, which its gateway lists them by too).""" + named = served_names({service: site["name"] for service, site in sites}) + return [{**site, "name": named[service]} for service, site in sites] def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]: diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 7912d3030..4920db480 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -88,6 +88,8 @@ declared_site_names, called_tools, fallback_name, + served_names, + website_card_name, fetch_child_card, forward, is_child_mcp_path, @@ -122,11 +124,12 @@ class SiteCard(NamedTuple): - """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, and - the names it declares.""" + """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, the + names it declares, and the name it gives the site (``website_card_name``).""" reserved: frozenset[str] names: tuple[str, ...] + name: str | None = None class ToolNameClash(ValueError): @@ -247,7 +250,7 @@ def __init__( if self.website_urls: @self._mcp.tool(name="list_website_urls", description="List available website URLs that can be browsed.") async def list_website_urls() -> list[str]: - return [await self._listed_url(site, url) for site, url in self.website_urls.items()] + return await self._listed_urls() self._server_tools.setdefault("gateway", []).append( MCPTool( name="list_website_urls", @@ -1028,7 +1031,8 @@ async def _site_card(self, app: str) -> SiteCard | None: except ValueError: card = None address = self.website_urls.get(app) or base_url - self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card)) + self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card), + website_card_name(card)) if isinstance(card, dict) else SiteCard(frozenset(), ())) return self._site_cards[app] @@ -1036,16 +1040,21 @@ def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]: """The names website ``app`` is reached at: those its env stores, else those its ``card`` declares.""" return self._site_names.get(app) or (card.names if card else ()) - async def _listed_url(self, site: str, url: str) -> str: - """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, - its first name (``listed_url``), its card read for a site whose env stores none, and ``fallback_name`` for one - whose card declares none either, as its browser's site proxy routes it; else, or while its card can't be read, - ``url``.""" - if site not in self._listed_by_name: - return url - card = None if site in self._site_names else await self._site_card(site) - names = self._names(site, card) or ((fallback_name(site),) if card is not None else ()) - return listed_url(names) or url + async def _listed_urls(self) -> list[str]: + """The address ``list_website_urls`` gives each site, at its url behind the gateway: for a site listed by name, + its first name (``listed_url``), those its env stores, else those its card declares, else, for a site declaring + none, the ``fallback_name`` its browser's site proxy routes it at, of the name it goes by (``served_names``: + its card's, unless two sites' cards give one); else, or while its card can't be read, its url.""" + cards = {site: await self._site_card(site) for site in self.website_urls if site in self._listed_by_name} + goes_by = served_names({site: card.name or site for site, card in cards.items() if card is not None}) + listed = [] + for site, url in self.website_urls.items(): + card = cards.get(site) + names = self._names(site, card) if site in cards else () + if not names and card is not None: + names = (fallback_name(goes_by[site]),) + listed.append(listed_url(names) or url) + return listed def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 0a4eeac39..75a87191f 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -37,7 +37,7 @@ import logging import posixpath import re -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from typing import Any, NamedTuple from urllib.parse import unquote, urlsplit, urlunsplit @@ -240,6 +240,22 @@ def declared_site_names(card: dict) -> tuple[str, ...]: return () +def website_card_name(card: dict) -> str | None: + """The name a site's card gives it, as a computer's site proxy names the site: the card's ``name`` when it + declares ``website/v1``; None otherwise, and the site goes by the name it's served as.""" + extensions = (card.get("capabilities") or {}).get("extensions") or [] + name = card.get("name") + declares = any(isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI for ext in extensions) + return name if declares and isinstance(name, str) and name else None + + +def served_names(sites: Mapping[str, str]) -> dict[str, str]: + """The name each of one env's sites goes by, by the name it's served as: the name it gives, its card's or else the + one it's served as, unless two give one name, and then each the name it's served as.""" + given = list(sites.values()) + return dict(sites) if len(set(given)) == len(given) else {served: served for served in sites} + + def fallback_name(site: str) -> str: """The name a synthetic site declaring none is reached by: its name on ``.localhost``, lowercase, each ``_`` a ``-``, and no ``-`` at either end of the label (``My_Site`` is ``my-site.localhost``). Two sites it gives one name diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 6b216bb9f..b8a05ec2e 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -836,6 +836,27 @@ async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_a "http://old-notes.localhost/", "https://carded.test/", "http://other"] +@pytest.mark.asyncio +async def test_a_site_declaring_no_names_is_listed_at_the_fallback_of_the_name_its_card_gives_it_as_a_proxy_routes_it( + monkeypatch): + named = {"notes": "NotesApp", "left": "Shop", "right": "Shop", "plain": None} + + def handler(request: httpx.Request) -> httpx.Response: + card = {"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {}}]}} + return httpx.Response(200, json={**card, **({"name": name} if (name := named[request.url.host]) else {})}) + transport = httpx.MockTransport(handler) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={site: f"http://{site}" for site in named}, listed_by_name=list(named)) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + # Named by their cards, but for the two cards giving one name, each named as it's served instead. + assert listed == ["http://notes.localhost/", "http://left.localhost/", "http://right.localhost/", + "http://plain.localhost/"] + gw = _gateway(website_urls={"notes": "http://notes", "plain": "http://plain"}, listed_by_name=["notes", "plain"]) + assert await gw._mcp._tool_manager.get_tool("list_website_urls").fn() == ["http://notesapp.localhost/", + "http://plain.localhost/"] + + @pytest.mark.asyncio async def test_list_website_urls_gives_a_site_listed_by_name_its_address_while_its_card_cant_be_read(monkeypatch): transport = httpx.MockTransport(lambda request: httpx.Response(503)) From d70cdf4a95121a9b9524cae14359433a5c71ac7f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 09:47:07 -0800 Subject: [PATCH 3/4] fix(gateway): list_website_urls reads cards together, and stored-name sites' only when a fallback is needed Greptile r2: reading every listed site's card one after another made a stalled card endpoint delay the whole list, even when every site stores its names. Cards of sites without stored names are now read at once, and those of sites with stored names only when some site needs a fallback, the one case their names can change. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 14 +++++++++++--- tst/unit/env/gateway/rest_proxy_stream_test.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 4920db480..b5d75eade 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -1044,18 +1044,26 @@ async def _listed_urls(self) -> list[str]: """The address ``list_website_urls`` gives each site, at its url behind the gateway: for a site listed by name, its first name (``listed_url``), those its env stores, else those its card declares, else, for a site declaring none, the ``fallback_name`` its browser's site proxy routes it at, of the name it goes by (``served_names``: - its card's, unless two sites' cards give one); else, or while its card can't be read, its url.""" - cards = {site: await self._site_card(site) for site in self.website_urls if site in self._listed_by_name} + its card's, unless two sites' cards give one); else, or while its card can't be read, its url. Cards are read + together, and those of sites with stored names only when some site needs a fallback.""" + by_name = [site for site in self.website_urls if site in self._listed_by_name] + cards = await self._site_cards_of([site for site in by_name if site not in self._site_names]) + if any(card is not None and not card.names for card in cards.values()): + cards |= await self._site_cards_of([site for site in by_name if site in self._site_names]) goes_by = served_names({site: card.name or site for site, card in cards.items() if card is not None}) listed = [] for site, url in self.website_urls.items(): card = cards.get(site) - names = self._names(site, card) if site in cards else () + names = self._names(site, card) if site in self._listed_by_name else () if not names and card is not None: names = (fallback_name(goes_by[site]),) listed.append(listed_url(names) or url) return listed + async def _site_cards_of(self, sites: list[str]) -> dict[str, SiteCard | None]: + """Each of ``sites``' cards (``_site_card``), read at once.""" + return dict(zip(sites, await asyncio.gather(*(self._site_card(site) for site in sites)), strict=True)) + def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website # backends). Non-v1 backers (e.g. the injected website_browser) self-omit from the card via 404. diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index b8a05ec2e..38743e8a0 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -857,6 +857,22 @@ def handler(request: httpx.Request) -> httpx.Response: "http://plain.localhost/"] +@pytest.mark.asyncio +async def test_list_website_urls_reads_no_card_when_every_site_listed_by_name_stores_its_names(monkeypatch): + asked: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(404) + transport = httpx.MockTransport(handler) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"shop": "http://shop", "notes": "http://notes"}, + site_names={"shop": ["shop.test"], "notes": ["notes.localhost"]}, listed_by_name=["shop", "notes"]) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + assert (listed, asked) == (["https://shop.test/", "http://notes.localhost/"], []) + + @pytest.mark.asyncio async def test_list_website_urls_gives_a_site_listed_by_name_its_address_while_its_card_cant_be_read(monkeypatch): transport = httpx.MockTransport(lambda request: httpx.Response(503)) From 3e82602ab8d2f057d82c0ae46f5d6a74b402c669 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 10:25:40 -0800 Subject: [PATCH 4/4] refactor(gateway): SiteCard.card_name, so names keeps meaning the hostnames a site is reached at Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index b5d75eade..a503d8c84 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -125,11 +125,12 @@ class SiteCard(NamedTuple): """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, the - names it declares, and the name it gives the site (``website_card_name``).""" + names it declares (hostnames), and the card's own name (``website_card_name``), which needn't be the name the + gateway serves the site as.""" reserved: frozenset[str] names: tuple[str, ...] - name: str | None = None + card_name: str | None = None class ToolNameClash(ValueError): @@ -1050,7 +1051,7 @@ async def _listed_urls(self) -> list[str]: cards = await self._site_cards_of([site for site in by_name if site not in self._site_names]) if any(card is not None and not card.names for card in cards.values()): cards |= await self._site_cards_of([site for site in by_name if site in self._site_names]) - goes_by = served_names({site: card.name or site for site, card in cards.items() if card is not None}) + goes_by = served_names({site: card.card_name or site for site, card in cards.items() if card is not None}) listed = [] for site, url in self.website_urls.items(): card = cards.get(site)