Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/agent_env/cli/env/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,11 @@ def deploy(env_id: str, env_version: int | None, ttl_seconds: int, gateway_mode:
# A record without a gateway has none of the gateway's URLs.
if isinstance(deployed_env, DeployedGatewayEnv):
click.echo("Env Gateway Url: " + click.style(deployed_env.gateway_url, fg="yellow"))
if deployed_env.db_web_url:
click.echo("Env DB Web Url: " + click.style(deployed_env.db_web_url, fg="yellow"))
if deployed_env.db_mcp_url:
click.echo("Env DB MCP Url: " + click.style(deployed_env.db_mcp_url, fg="yellow"))
# The SQL consoles take the env's control token, as an Authorization: Bearer header.
if deployed_env.db_web_gateway_url:
click.echo("Env DB Web Url: " + click.style(deployed_env.db_web_gateway_url, fg="yellow"))
if deployed_env.db_mcp_gateway_url:
click.echo("Env DB MCP Url: " + click.style(deployed_env.db_mcp_gateway_url, fg="yellow"))
if deployed_env.vnc_url:
click.echo("VNC Url: " + click.style(deployed_env.vnc_url, fg="yellow"))
for svc_name, url in (deployed_env.website_frontend_urls or {}).items():
Expand Down
8 changes: 4 additions & 4 deletions src/agent_env/cli/env/get_instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ def get_instance(instance_id: str):
if gateway:
click.echo(f"Env Base Url: {gateway.gateway_url}")
click.echo(f"Env MCP Url: {deployed_env.mcp_url}")
if gateway and gateway.db_web_url:
click.echo(f"Env DB Web Url: {gateway.db_web_url}")
if gateway and gateway.db_mcp_url:
click.echo(f"Env DB MCP Url: {gateway.db_mcp_url}")
if gateway and gateway.db_web_gateway_url:
click.echo(f"Env DB Web Url: {gateway.db_web_gateway_url}")
if gateway and gateway.db_mcp_gateway_url:
click.echo(f"Env DB MCP Url: {gateway.db_mcp_gateway_url}")
for svc_name, url in ((gateway.website_frontend_urls if gateway else None) or {}).items():
click.echo(f"Website Frontend ({svc_name}): {url}")
if isinstance(deployed_env, DeployedSandboxEnv):
Expand Down
2 changes: 1 addition & 1 deletion src/agent_env/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

_COLORS = {Outcome.PASSED: "green", Outcome.BELOW_ONE: "yellow", Outcome.UNSCORED: "yellow", Outcome.FAILED: "red",
Outcome.CANCELLED: "yellow"}
_ENV_ENDPOINTS = (("mcp", "mcp_url"), ("gateway", "gateway_url"), ("pgweb", "db_web_url"), ("db-mcp", "db_mcp_url"),
_ENV_ENDPOINTS = (("mcp", "mcp_url"), ("gateway", "gateway_url"), ("pgweb", "db_web_gateway_url"), ("db-mcp", "db_mcp_gateway_url"),
("vnc", "vnc_url"), ("expires", "expires_at_utc"))
_DRY_RUN = "Dry run: nothing is written or run."

Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/cli/task/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ def _format_env(env) -> str:
f" env_version: {env.env_version}",
*([f" gateway_url: {env.gateway_url}"] if fronted else []),
f" mcp_url: {env.mcp_url}",
*([f" db_web_url: {env.db_web_url}", f" db_mcp_url: {env.db_mcp_url}"] if fronted else []),
*([f" db_web_gateway_url: {env.db_web_gateway_url}", f" db_mcp_gateway_url: {env.db_mcp_gateway_url}"]
if fronted else []),
]
if isinstance(env, DeployedSandboxEnv): # an env outside our sandboxes has no sandbox to name
lines.append(f" sandbox_id: {env.sandbox_id}")
Expand Down
10 changes: 8 additions & 2 deletions src/agent_env/env/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ class DeployedGatewayEnv(DeployedSandboxEnv):
gateway_mode: str = GatewayMode.PERFORMANCE.value
db_web_url: Optional[str] = None
db_mcp_url: Optional[str] = None
# pgweb and db-mcp behind the gateway, which take the control token (``control_headers_for``); None when the deploy
# renders no such console, has no control token, or its gateway doesn't guard and reach the console.
db_web_gateway_url: Optional[str] = None
db_mcp_gateway_url: Optional[str] = None
website_frontend_urls: Optional[dict[str, str]] = None
vnc_url: Optional[str] = None
# Ids of the EnvStateInstance record(s) this deploy's state was provisioned into
Expand All @@ -251,6 +255,8 @@ def _fields_from(cls, data: dict) -> dict:
"gateway_mode": data.get("gateway_mode", GatewayMode.PERFORMANCE.value),
"db_web_url": data.get("db_web_url"),
"db_mcp_url": data.get("db_mcp_url"),
"db_web_gateway_url": data.get("db_web_gateway_url"),
"db_mcp_gateway_url": data.get("db_mcp_gateway_url"),
"website_frontend_urls": data.get("website_frontend_urls"),
"vnc_url": data.get("vnc_url"),
"env_state_instance_ids": data.get("env_state_instance_ids") or [],
Expand Down Expand Up @@ -587,5 +593,5 @@ def _record_class(data: dict) -> type[DeployedEnv]:


# A record carrying any of these is a gateway's. Not gateway_mode: an older kernel writes its default into any record it round-trips.
_GATEWAY_ONLY_FIELDS = ("gateway_url", "db_web_url", "db_mcp_url", "website_frontend_urls", "vnc_url", "env_state_instance_ids",
"child_envs", "stack_layout")
_GATEWAY_ONLY_FIELDS = ("gateway_url", "db_web_url", "db_mcp_url", "db_web_gateway_url", "db_mcp_gateway_url",
"website_frontend_urls", "vnc_url", "env_state_instance_ids", "child_envs", "stack_layout")
7 changes: 5 additions & 2 deletions src/agent_env/env/envs/_multi_computers.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, Env, gateway_url_of
from agent_env.env.envs.computer.utils import clock as computer_clock, sites
from agent_env.env.envs.service_db import CONSOLE_SERVICE_NAMES
from agent_env.env.gateway import ExternalChild
from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider, get_env_sandbox_provider

Expand All @@ -29,8 +30,8 @@
def name_problems(computers: Iterable[str], *, mcp_servers: Iterable[str] = (), websites: Iterable[str] = ()) -> list[str]:
"""What's wrong with ``computers``, the environment names of a multi env's computers, beside the names of its
``mcp_servers`` and ``websites``: a computer serves its tools under its name and ``_``, which must make a tool prefix,
and the gateway knows a computer by its name alone, which no other child env and no route of the gateway's to an
MCP server (``mcp-<server>``) may have."""
and the gateway knows a computer by its name alone, which no other child env, no route of the gateway's to an MCP
server (``mcp-<server>``) and no SQL console's /svc key may have."""
computers, mcp_servers = list(computers), list(dict.fromkeys(mcp_servers))
problems = [f"computer {name!r} serves its tools under its name and '_': {problem}"
for name in dict.fromkeys(computers) if (problem := tool_prefix_problem(f"{name}_"))]
Expand All @@ -40,6 +41,8 @@ def name_problems(computers: Iterable[str], *, mcp_servers: Iterable[str] = (),
for name in dict.fromkeys(computers) if counts[name] > 1]
problems += [f"{name!r} is the gateway's route to MCP server {routes[name]!r}, and the gateway knows a computer by its "
"name alone" for name in dict.fromkeys(computers) if name in routes]
problems += [f"{name!r} is the gateway's route to a SQL console, and the gateway knows a computer by its name alone"
for name in dict.fromkeys(computers) if name in CONSOLE_SERVICE_NAMES]
return problems


Expand Down
2 changes: 2 additions & 0 deletions src/agent_env/env/envs/service_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
DB_MCP_SERVICE_NAME = "db-mcp"
DB_MCP_PORT = 18767 # db-mcp host/tunnel port
DB_MCP_CONTAINER_PORT = 8000 # db-mcp's in-container listen port (tunneled as DB_MCP_PORT:DB_MCP_CONTAINER_PORT)
# The env's SQL consoles, each served by the gateway at /svc/<name>/, so no child env may take one's name.
CONSOLE_SERVICE_NAMES = frozenset({PGWEB_SERVICE_NAME, DB_MCP_SERVICE_NAME})


@dataclass
Expand Down
3 changes: 3 additions & 0 deletions src/agent_env/env/gateway/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int:
# The gateway's computers, comma-separated: a multi env's, each a child on a sandbox of its own, whose input calls set
# who drives it, the role its browser's requests take.
COMPUTERS_ENV = "COMPUTERS"
# The env's SQL consoles (pgweb, db-mcp), each as `key=url` on the env's network, comma-separated. The gateway serves
# each at /svc/<key>/, every path of it behind its control token, and none of them when it has no token.
CONSOLE_URLS_ENV = "CONSOLE_URLS"
# The route a clock follower long-polls, with its read token.
CLOCK_WATCH_PATH = "/clock/watch"
CLOCK_FOLLOWERS_PATH = "/clock/followers"
Expand Down
20 changes: 11 additions & 9 deletions src/agent_env/env/gateway/entrypoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import httpx

from . import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode, InternalMCPServer
from .constants import (CLOCK_FOLLOWERS_ENV, COMPUTERS_ENV, CONTROL_TOKEN_CHILDREN_ENV, CONTROL_TOKEN_ENV, DEFAULT_MCP_SERVER_NAME,
from .constants import (CLOCK_FOLLOWERS_ENV, COMPUTERS_ENV, CONSOLE_URLS_ENV, CONTROL_TOKEN_CHILDREN_ENV, CONTROL_TOKEN_ENV, DEFAULT_MCP_SERVER_NAME,
GATEWAY_PROVIDER_HEADERS_ENV, MCP_TRANSPORT, WEBSITE_LISTED_BY_NAME_ENV,
WEBSITE_NAMES_ENV, WELL_KNOWN_PATH)
from .gateway import Gateway, card_url
Expand Down Expand Up @@ -165,8 +165,9 @@ async def _site_card(client: httpx.AsyncClient, base: str, deadline: float) -> d
await asyncio.sleep(_INTERNAL_SERVER_POLL_S)


def parse_name_url_map(env_str: str) -> dict[str, str]:
# Expected format: 'slack=http://slack-website-backend:8000,email=http://email-website-backend:8000'
def parse_name_url_map(variable: str) -> dict[str, str]:
"""The environment variable ``variable`` as a name-to-URL map, from 'slack=http://slack-website-backend:8000,...'."""
env_str = os.environ.get(variable, "")
urls: dict[str, str] = {}
if not env_str.strip():
return urls
Expand All @@ -175,16 +176,16 @@ def parse_name_url_map(env_str: str) -> dict[str, str]:
if not entry:
continue
if "=" not in entry:
raise ValueError(f"Invalid REST_PROXY_URLS entry: '{entry}' (expected 'name=url')")
raise ValueError(f"Invalid {variable} entry: '{entry}' (expected 'name=url')")
name, url = entry.split("=", 1)
name = name.strip()
url = url.strip()
if not name:
raise ValueError(f"Empty service name in REST_PROXY_URLS entry: '{entry}'")
raise ValueError(f"Empty service name in {variable} entry: '{entry}'")
if not url.startswith(("http://", "https://")):
raise ValueError(f"Invalid URL in REST_PROXY_URLS: '{url}' (must start with http:// or https://)")
raise ValueError(f"Invalid URL in {variable}: '{url}' (must start with http:// or https://)")
if name in urls:
raise ValueError(f"Duplicate service name in REST_PROXY_URLS: '{name}'")
raise ValueError(f"Duplicate service name in {variable}: '{name}'")
urls[name] = url
return urls

Expand Down Expand Up @@ -223,8 +224,8 @@ def parse_site_names(env_str: str) -> dict[str, tuple[str, ...]]:
spec=os.environ.get("INTERNAL_MCP_SERVERS", ""),
port=int(os.environ.get("INTERNAL_MCP_PORT", str(AGENT_ENV_GATEWAY_MCP_PORT))),
)
website_urls = parse_name_url_map(os.environ.get("WEBSITE_URLS", ""))
rest_proxy_urls = parse_name_url_map(os.environ.get("REST_PROXY_URLS", ""))
website_urls = parse_name_url_map("WEBSITE_URLS")
rest_proxy_urls = parse_name_url_map("REST_PROXY_URLS")
internal_mcp_servers += asyncio.run(website_mcp_servers(
{name: rest_proxy_urls.get(name, url) for name, url in website_urls.items()},
taken=frozenset(server.name for server in internal_mcp_servers)))
Expand All @@ -239,6 +240,7 @@ def parse_site_names(env_str: str) -> dict[str, tuple[str, ...]]:
internal_mcp_servers=internal_mcp_servers,
website_urls=website_urls,
rest_proxy_urls=rest_proxy_urls,
console_urls=parse_name_url_map(CONSOLE_URLS_ENV),
gateway_mode=gateway_mode,
service_db_url=service_db_url,
control_token=os.environ.get(CONTROL_TOKEN_ENV),
Expand Down
28 changes: 23 additions & 5 deletions src/agent_env/env/gateway/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ def __init__(
clock_followers: Mapping[str, str] | None = None,
site_names: dict[str, Iterable[str]] | None = None,
listed_by_name: Iterable[str] = (),
console_urls: Mapping[str, str] | None = None,
):
"""`provider_headers` names the headers the sandbox provider in front of the gateway is sent, its credentials,
which the gateway takes off every request it forwards to a service behind it. `clock_followers` names the
Expand All @@ -191,7 +192,8 @@ def __init__(
computer opens it, and the others at their ``website_urls``. `control_token_children` are the children its
/svc routes pass its control token on to, a multi env's computers, which take it. `computers` are its children
that are computers, a multi env's: an input call to one's tools makes its caller the role driving it, which the
requests its browser sends take."""
requests its browser sends take. `console_urls` are the env's SQL consoles by key, each served at /svc/<key>/
only with the control token, and not at all without one."""
self.host = host
self.port = port
self.server_name = server_name
Expand All @@ -214,6 +216,9 @@ def __init__(
self._service_db_url = service_db_url
self._control_bearer = f"Bearer {control_token}".encode() if control_token else None
self._control_token_children = frozenset(control_token_children)
self._consoles = dict(console_urls or {}) if control_token else {}
if clashes := sorted(self._consoles.keys() & self.rest_proxy_urls.keys()):
raise ValueError(f"console keys {clashes} are also backing servers' /svc keys")
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
self._computers = frozenset(computers)
# The computer each computer tool is one of, the computer tools that take input rather than read, and the role
# driving each computer now: the last to make an input call to its tools.
Expand Down Expand Up @@ -267,14 +272,17 @@ async def list_website_urls() -> list[str]:
self._env_get_time_url_cache: str | None = None
self._install_role_filters()

if self.rest_proxy_urls:
if self.rest_proxy_urls or self._consoles:
@self._mcp.custom_route(
"/svc/{service_name}/{path:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
)
async def proxy_to_service(request: Request) -> Response:
if request.path_params["service_name"] in self._consoles:
return await self._proxy_console_request(request)
return await self._proxy_rest_request(request)

if self.rest_proxy_urls:
@self._mcp.custom_route(BROWSER_ROUTE, methods=["GET", "HEAD"])
async def browser_discovery(request: Request) -> Response:
return await self._browsers.discovery(request)
Expand Down Expand Up @@ -640,14 +648,17 @@ async def app(scope, receive, send):
return app

def _takes_control_token(self, scope: dict) -> bool:
"""Whether a request takes the deploy's control token: a control route, the env's data plane, or one of
``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``reaches``),
after dot-segment normalisation. A site's /website route never reaches its data plane."""
"""Whether a request takes the deploy's control token: a control route, the env's data plane, any path of a SQL
console behind /svc, or one of ``token_paths`` behind /svc, by the path as routed or by the one the backing
server receives (``reaches``), after dot-segment normalisation. A site's /website route never reaches its data
plane."""
path = scope["path"]
if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH):
return True
prefix, _, rest = path.lstrip("/").partition("/")
service, _, rest = rest.partition("/")
if prefix == "svc" and service in self._consoles:
return True
guarded = self._token_paths.get(service, ()) if prefix == "svc" else ()
raw, base = scope.get("raw_path") or path.encode(), self.rest_proxy_urls.get(service, "")
return any(reaches(base, raw, path, rest, endpoint) for endpoint in guarded)
Expand Down Expand Up @@ -905,6 +916,13 @@ async def log_write(status: int) -> None:
on_done=log_write, withheld=self._withheld, mcp_endpoint=mcp_endpoint,
site_names=site_names)

async def _proxy_console_request(self, request: Request) -> Response:
"""Forward a request to one of the env's SQL consoles. Only a holder of the control token reaches one, and what
it writes there is no role's, so it opens no write window."""
return await forward(request, self._consoles[request.path_params["service_name"]],
role=self._request_role(request), timeout=self.REST_PROXY_TIMEOUT, mcp_server=False,
withheld=self._withheld)

async def _proxy_website_request(self, request: Request) -> Response:
"""Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers.
A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are
Expand Down
Loading
Loading